add numeric tokenizer

1 files changed, 45 insertions, 19 deletions

diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
index 7f92240..6a91291 100644
--- a/src/z/parser/js/grammar/literal.zig
+++ b/src/z/parser/js/grammar/literal.zig
@@ -10,6 +10,8 @@ pub const Literal = enum {
     bigint,
     string,
 
+    const digits = "_0123456789abcdef";
+
     pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
         comptime {
             errdefer lexer.revert();
@@ -18,27 +20,50 @@ pub const Literal = enum {
 
             switch (try lexer.consume()) {
                 '0'...'9', '.' => |c| {
-                    token.kind.literal = .numeric;
+                    const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) {
+                        'x' => 16,
+                        'o' => 8,
+                        'b' => 2,
+                        else => 10,
+                    };
 
-                    var had_period = c == '.';
+                    if (base != 10) lexer.skip() catch unreachable;
 
-                    while (lexer.peekChar()) |ch| {
-                        switch (ch) {
-                            '0'...'9' => {},
-                            '.' => if (had_period) {
-                                return Lexer.Error.UnexpectedToken;
-                            } else {
-                                had_period = true;
-                            },
-                            else => return Lexer.Error.UnexpectedToken,
-                        }
+                    {
+                        const index = std.mem.findNone(
+                            u8,
+                            lexer.buffer,
+                            digits[0 .. base + 1],
+                        ) orelse lexer.buffer.len;
+                        try lexer.skipTo(index);
+
+
+                        if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken;
+                    }
 
+                    if (base == 10 and lexer.peekChar() == '.') {
                         try lexer.skip();
+                        const index = std.mem.findNone(
+                            u8,
+                            lexer.buffer,
+                            digits[0 .. base + 1],
+                        ) orelse lexer.buffer.len;
+                        try lexer.skipTo(index);
+                    }
+
+                    if (lexer.peekChar() == 'n') {
+                        lexer.skip() catch unreachable;
+                        token.kind.literal = .bigint;
+                    } else {
+                        token.kind.literal = .numeric;
                     }
                 },
-                '\'', '"' => |quote| while (try lexer.consume() != quote) {
-                    if (quote == '\\') {
-                        try lexer.consume();
+                '\'', '"' => |quote| {
+                    token.kind.literal = .string;
+                    while (try lexer.consume() != quote) {
+                        if (quote == '\\') {
+                            try lexer.consume();
+                        }
                     }
                 },
                 else => return Lexer.Error.UnexpectedToken,
@@ -61,10 +86,11 @@ test "numeric" {
 
     try Simple.run("1234", 4);
     try Simple.run("1.234", 5);
-    try std.testing.expectError(
-        Lexer.Error.UnexpectedToken,
-        Simple.run("12.3.4", 0),
-    );
+    try Simple.run("1_2_3_4", 7);
+    try Simple.run("0b101011", 8);
+    try Simple.run("0xabcf01", 8);
+    try Simple.run("0o731734", 8);
+    try Simple.run("0o731734n", 9);
 }
 
 test "string" {