add tokenizer for whitespaces in js

1 files changed, 36 insertions, 0 deletions

diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
new file mode 100644
index 0000000..e1a1e96
--- /dev/null
+++ b/src/z/parser/js/grammar/root.zig
@@ -0,0 +1,36 @@
+const std = @import("std");
+
+const parser = @import("../../root.zig");
+pub const Lexer = parser.Lexer(Grammar);
+pub const Token = parser.Token(Grammar);
+
+pub const Whitespace = @import("whitespace.zig").Whitespace;
+pub const Literal = @import("literal.zig").Literal;
+pub const Punctuator = @import("punctuator.zig").Punctuator;
+pub const Identifier = @import("identifier.zig").Identifier;
+
+pub const Grammar = union(enum) {
+    whitespace: Whitespace,
+    literal: Literal,
+    punctuator: Punctuator,
+    identifier: Identifier,
+    end_of_file: void,
+
+    pub fn tokenize(lexer: *Lexer) Lexer.Error!void {
+        while (lexer.peakChar()) |_| {
+            Whitespace.tokenize(lexer) catch {
+                Literal.tokenize(lexer) catch {
+                    Punctuator.tokenize(lexer) catch {
+                        try Identifier.tokenize(lexer);
+                    };
+                };
+            };
+        }
+
+        lexer.push(lexer.token(0, .end_of_file));
+    }
+};
+
+test {
+    _ = std.testing.refAllDecls(@This());
+}