add tokenizer for all other variants

6 files changed, 285 insertions, 19 deletions

diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig
index 038dbc5..905f937 100644
--- a/src/z/parser/js/grammar/identifier.zig
+++ b/src/z/parser/js/grammar/identifier.zig
@@ -1,4 +1,58 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Identifier = enum {
     public,
     private,
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            defer lexer.revert();
+
+            const token = tok: {
+                if (lexer.peekChar() == '#') {
+                    const t = lexer.start(.{ .identifier = .private });
+                    lexer.skip() catch unreachable;
+                    break :tok t;
+                } else {
+                    break :tok lexer.start(.{ .identifier = .public });
+                }
+            };
+
+            switch (try lexer.consume()) {
+                'a'...'z', 'A'...'Z', '_' => {},
+                else => return Lexer.Error.UnexpectedToken,
+            }
+
+            while (lexer.peekChar()) |ch| {
+                switch (ch) {
+                    'a'...'z', 'A'...'Z', '0'...'9', '_' => {},
+                    else => break,
+                }
+
+                try lexer.skip();
+            }
+
+            lexer.commit(token);
+        }
+    }
 };
+
+test "simple" {
+    {
+        comptime var lexer: Lexer = .init("some_literal9");
+        try Identifier.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(13, lexer.tokens[0].slice.len);
+    }
+
+    {
+        comptime var lexer: Lexer = .init("#some_literal9");
+        try Identifier.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(14, lexer.tokens[0].slice.len);
+    }
+}
diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig
index 14c3272..c3b32ac 100644
--- a/src/z/parser/js/grammar/keyword.zig
+++ b/src/z/parser/js/grammar/keyword.zig
@@ -1,3 +1,9 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Keyword = enum {
     @"break",
     case,
@@ -45,4 +51,63 @@ pub const Keyword = enum {
     null,
     true,
     false,
+
+    const Self = @This();
+
+    const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: {
+        var keywords: []const struct { []const u8, Self } = &.{};
+
+        for (@typeInfo(Self).@"enum".fields) |field| {
+            const name: []const u8 = field.name;
+            const value: Self = @field(Self, field.name);
+
+            keywords = keywords ++ .{ .{ name, value } };
+        }
+
+        break :kws keywords;
+    });
+
+    pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            @setEvalBranchQuota(10000);
+            errdefer lexer.revert();
+
+            const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len;
+
+            if (index == 0) return Lexer.Error.UnexpectedToken;
+
+            if (keywordmap.get(lexer.buffer[0..index])) |tag| {
+                const token = lexer.start(.{ .keyword = tag });
+                try lexer.skipTo(index);
+                lexer.commit(token);
+            } else {
+                return Lexer.Error.UnexpectedToken;
+            }
+        }
+    }
 };
+
+test "simple" {
+    inline for (comptime Keyword.keywordmap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key);
+        try Keyword.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
+
+test "ending" {
+    inline for (comptime Keyword.keywordmap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key ++ ".");
+        try Keyword.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
+
+test "subword" {
+    inline for (comptime Keyword.keywordmap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key ++ "a");
+        try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer));
+    }
+}
diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
index a2ef97a..7f92240 100644
--- a/src/z/parser/js/grammar/literal.zig
+++ b/src/z/parser/js/grammar/literal.zig
@@ -1,5 +1,86 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Literal = enum {
+    // NOTE: currently only decimal literals are tokenized
     numeric,
     bigint,
     string,
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            errdefer lexer.revert();
+
+            var token = lexer.start(.{ .literal = undefined });
+
+            switch (try lexer.consume()) {
+                '0'...'9', '.' => |c| {
+                    token.kind.literal = .numeric;
+
+                    var had_period = c == '.';
+
+                    while (lexer.peekChar()) |ch| {
+                        switch (ch) {
+                            '0'...'9' => {},
+                            '.' => if (had_period) {
+                                return Lexer.Error.UnexpectedToken;
+                            } else {
+                                had_period = true;
+                            },
+                            else => return Lexer.Error.UnexpectedToken,
+                        }
+
+                        try lexer.skip();
+                    }
+                },
+                '\'', '"' => |quote| while (try lexer.consume() != quote) {
+                    if (quote == '\\') {
+                        try lexer.consume();
+                    }
+                },
+                else => return Lexer.Error.UnexpectedToken,
+            }
+
+            lexer.commit(token);
+        }
+    }
 };
+
+test "numeric" {
+    const Simple = struct {
+        pub fn run(comptime buffer: []const u8, n: usize) !void {
+            comptime var lexer: Lexer = .init(buffer);
+            try Literal.tokenize(&lexer);
+            try std.testing.expectEqual(1, lexer.tokens.len);
+            try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+        }
+    };
+
+    try Simple.run("1234", 4);
+    try Simple.run("1.234", 5);
+    try std.testing.expectError(
+        Lexer.Error.UnexpectedToken,
+        Simple.run("12.3.4", 0),
+    );
+}
+
+test "string" {
+    const Simple = struct {
+        pub fn run(comptime buffer: []const u8, n: usize) !void {
+            comptime var lexer: Lexer = .init(buffer);
+            try Literal.tokenize(&lexer);
+            try std.testing.expectEqual(1, lexer.tokens.len);
+            try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+        }
+    };
+
+    try Simple.run("'hello'", 7);
+    try Simple.run("\"hello\"", 7);
+    try std.testing.expectError(
+        Lexer.Error.EndOfBuffer,
+        Simple.run("'hello", 0),
+    );
+}
diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig
index 81b2e6e..a72ab8a 100644
--- a/src/z/parser/js/grammar/punctuator.zig
+++ b/src/z/parser/js/grammar/punctuator.zig
@@ -1,3 +1,9 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
 pub const Punctuator = enum {
     @"+",
     @"-",
@@ -58,4 +64,52 @@ pub const Punctuator = enum {
     @"?.",
     @"#",
     @"@",
+
+    const Self = @This();
+
+    const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: {
+        var punctuators: []const struct { []const u8, Self } = &.{};
+
+        for (@typeInfo(Self).@"enum".fields) |field| {
+            const name: []const u8 = field.name;
+            const value: Self = @field(Self, field.name);
+
+            punctuators = punctuators ++ .{.{ name, value }};
+        }
+
+        break :kws punctuators;
+    });
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            @setEvalBranchQuota(10_000);
+            defer lexer.revert();
+
+            if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| {
+                const token = lexer.start(.{ .punctuator = pair.value });
+                try lexer.skipTo(pair.key.len);
+                lexer.commit(token);
+            } else {
+                return Lexer.Error.UnexpectedToken;
+            }
+        }
+    }
 };
+
+test "simple" {
+    inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key);
+        try Punctuator.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
+
+test "ending" {
+    inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+        comptime var lexer: Lexer = .init(key ++ "a");
+        try Punctuator.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+    }
+}
diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
index e1a1e96..03b386e 100644
--- a/src/z/parser/js/grammar/root.zig
+++ b/src/z/parser/js/grammar/root.zig
@@ -1,3 +1,13 @@
+//! This is a Javascript parser and tokenizer.
+//! It is and probably will never be complete,
+//! since js is a very strange language and I want
+//! to move forward and code on more interesting things
+//! I started something called mujs (micro-js). It's a
+//! subset of the javascript language. It does not support
+//! the regex literal and currently also not template strings.
+//! It is chosen like that to make parsing easier and also less
+//! context-dependent.
+
 const std = @import("std");
 
 const parser = @import("../../root.zig");
@@ -5,29 +15,35 @@ pub const Lexer = parser.Lexer(Grammar);
 pub const Token = parser.Token(Grammar);
 
 pub const Whitespace = @import("whitespace.zig").Whitespace;
+pub const Keyword = @import("keyword.zig").Keyword;
 pub const Literal = @import("literal.zig").Literal;
 pub const Punctuator = @import("punctuator.zig").Punctuator;
 pub const Identifier = @import("identifier.zig").Identifier;
 
 pub const Grammar = union(enum) {
     whitespace: Whitespace,
+    keyword: Keyword,
     literal: Literal,
     punctuator: Punctuator,
     identifier: Identifier,
     end_of_file: void,
 
-    pub fn tokenize(lexer: *Lexer) Lexer.Error!void {
-        while (lexer.peakChar()) |_| {
-            Whitespace.tokenize(lexer) catch {
-                Literal.tokenize(lexer) catch {
-                    Punctuator.tokenize(lexer) catch {
-                        try Identifier.tokenize(lexer);
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            while (lexer.peekChar()) |_| {
+                Whitespace.tokenize(lexer) catch {
+                    Keyword.tokenize(lexer) catch {
+                        Literal.tokenize(lexer) catch {
+                            Punctuator.tokenize(lexer) catch {
+                                try Identifier.tokenize(lexer);
+                            };
+                        };
                     };
                 };
-            };
-        }
+            }
 
-        lexer.push(lexer.token(0, .end_of_file));
+            lexer.commit(lexer.start(.end_of_file));
+        }
     }
 };
 
diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig
index c3435c4..08d031f 100644
--- a/src/z/parser/js/grammar/whitespace.zig
+++ b/src/z/parser/js/grammar/whitespace.zig
@@ -11,11 +11,11 @@ pub const Whitespace = enum {
         comptime {
             errdefer lexer.revert();
 
-            var token = lexer.start(undefined);
+            var token = lexer.start(.{ .whitespace = undefined });
 
-            switch (lexer.peekChar() orelse 0) {
+            switch (try lexer.consume()) {
                 ' ', '\t', '\r', '\n' => {
-                    token.kind = .{ .whitespace = .space };
+                    token.kind.whitespace = .space;
 
                     while (lexer.peekChar()) |ch| {
                         switch (ch) {
@@ -25,13 +25,13 @@ pub const Whitespace = enum {
                     }
                 },
                 '/' => {
-                    switch (lexer.peekCharAt(1) orelse 0) {
+                    switch (lexer.peekChar() orelse 0) {
                         '*' => {
-                            token.kind = .{ .whitespace = .block_comment };
+                            token.kind.whitespace = .block_comment;
                             try lexer.skipUntil("*/");
                         },
                         '/' => {
-                            token.kind = .{ .whitespace = .line_comment };
+                            token.kind.whitespace = .line_comment;
                             lexer.skipUntil("\n") catch lexer.skipToEnd();
                         },
                         else => return Lexer.Error.UnexpectedToken,
@@ -99,8 +99,4 @@ test "mixed" {
     } else |_| {}
 
     try std.testing.expectEqual(7, lexer.tokens.len);
-
-    inline for (lexer.tokens) |token| {
-        std.debug.print("{f}\n", .{token});
-    }
 }