add tokenizer for all other variants

1 files changed, 54 insertions, 0 deletions

diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig
index 81b2e6e..a72ab8a 100644
--- a/src/z/parser/js/grammar/punctuator.zig
+++ b/src/z/parser/js/grammar/punctuator.zig
@@ -1,3 +1,9 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Punctuator = enum {
     @"+",
     @"-",
@@ -58,4 +64,52 @@ pub const Punctuator = enum {
     @"?.",
     @"#",
     @"@",
+
+    const Self = @This();
+
+    const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: {
+        var punctuators: []const struct { []const u8, Self } = &.{};
+
+        for (@typeInfo(Self).@"enum".fields) |field| {
+            const name: []const u8 = field.name;
+            const value: Self = @field(Self, field.name);
+
+            punctuators = punctuators ++ .{.{ name, value }};
+        }
+
+        break :kws punctuators;
+    });
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            @setEvalBranchQuota(10_000);
+            defer lexer.revert();
+
+            if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| {
+                const token = lexer.start(.{ .punctuator = pair.value });
+                try lexer.skipTo(pair.key.len);
+                lexer.commit(token);
+            } else {
+                return Lexer.Error.UnexpectedToken;
+            }
+        }
+    }
 };
+
+test "simple" {
+    inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key);
+        try Punctuator.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
+
+test "ending" {
+    inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key ++ "a");
+        try Punctuator.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}