add tokenizer for all other variants

1 files changed, 81 insertions, 0 deletions

diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
index a2ef97a..7f92240 100644
--- a/src/z/parser/js/grammar/literal.zig
+++ b/src/z/parser/js/grammar/literal.zig
@@ -1,5 +1,86 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Literal = enum {
+    // NOTE: currently only decimal literals are tokenized
     numeric,
     bigint,
     string,
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            errdefer lexer.revert();
+
+            var token = lexer.start(.{ .literal = undefined });
+
+            switch (try lexer.consume()) {
+                '0'...'9', '.' => |c| {
+                    token.kind.literal = .numeric;
+
+                    var had_period = c == '.';
+
+                    while (lexer.peekChar()) |ch| {
+                        switch (ch) {
+                            '0'...'9' => {},
+                            '.' => if (had_period) {
+                                return Lexer.Error.UnexpectedToken;
+                            } else {
+                                had_period = true;
+                            },
+                            else => return Lexer.Error.UnexpectedToken,
+                        }
+
+                        try lexer.skip();
+                    }
+                },
+                '\'', '"' => |quote| while (try lexer.consume() != quote) {
+                    if (quote == '\\') {
+                        try lexer.consume();
+                    }
+                },
+                else => return Lexer.Error.UnexpectedToken,
+            }
+
+            lexer.commit(token);
+        }
+    }
 };
+
+test "numeric" {
+    const Simple = struct {
+        pub fn run(comptime buffer: []const u8, n: usize) !void {
+            comptime var lexer: Lexer = .init(buffer);
+            try Literal.tokenize(&lexer);
+            try std.testing.expectEqual(1, lexer.tokens.len);
+            try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+        }
+    };
+
+    try Simple.run("1234", 4);
+    try Simple.run("1.234", 5);
+    try std.testing.expectError(
+        Lexer.Error.UnexpectedToken,
+        Simple.run("12.3.4", 0),
+    );
+}
+
+test "string" {
+    const Simple = struct {
+        pub fn run(comptime buffer: []const u8, n: usize) !void {
+            comptime var lexer: Lexer = .init(buffer);
+            try Literal.tokenize(&lexer);
+            try std.testing.expectEqual(1, lexer.tokens.len);
+            try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+        }
+    };
+
+    try Simple.run("'hello'", 7);
+    try Simple.run("\"hello\"", 7);
+    try std.testing.expectError(
+        Lexer.Error.EndOfBuffer,
+        Simple.run("'hello", 0),
+    );
+}