add tokenizer for all other variants

1 files changed, 65 insertions, 0 deletions

diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig
index 14c3272..c3b32ac 100644
--- a/src/z/parser/js/grammar/keyword.zig
+++ b/src/z/parser/js/grammar/keyword.zig
@@ -1,3 +1,9 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Keyword = enum {
     @"break",
     case,
@@ -45,4 +51,63 @@ pub const Keyword = enum {
     null,
     true,
     false,
+
+    const Self = @This();
+
+    const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: {
+        var keywords: []const struct { []const u8, Self } = &.{};
+
+        for (@typeInfo(Self).@"enum".fields) |field| {
+            const name: []const u8 = field.name;
+            const value: Self = @field(Self, field.name);
+
+            keywords = keywords ++ .{ .{ name, value } };
+        }
+
+        break :kws keywords;
+    });
+
+    pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            @setEvalBranchQuota(10000);
+            errdefer lexer.revert();
+
+            const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len;
+
+            if (index == 0) return Lexer.Error.UnexpectedToken;
+
+            if (keywordmap.get(lexer.buffer[0..index])) |tag| {
+                const token = lexer.start(.{ .keyword = tag });
+                try lexer.skipTo(index);
+                lexer.commit(token);
+            } else {
+                return Lexer.Error.UnexpectedToken;
+            }
+        }
+    }
 };
+
+test "simple" {
+    inline for (comptime Keyword.keywordmap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key);
+        try Keyword.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
+
+test "ending" {
+    inline for (comptime Keyword.keywordmap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key ++ ".");
+        try Keyword.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
+
+test "subword" {
+    inline for (comptime Keyword.keywordmap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key ++ "a");
+        try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer));
+    }
+}