1 files changed, 25 insertions, 9 deletions

diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
index e1a1e96..03b386e 100644
--- a/src/z/parser/js/grammar/root.zig
+++ b/src/z/parser/js/grammar/root.zig
@@ -1,3 +1,13 @@
+//! This is a Javascript parser and tokenizer.
+//! It is and probably will never be complete,
+//! since js is a very strange language and I want
+//! to move forward and code on more interesting things
+//! I started something called mujs (micro-js). It's a
+//! subset of the javascript language. It does not support
+//! the regex literal and currently also not template strings.
+//! It is chosen like that to make parsing easier and also less
+//! context-dependent.
+
 const std = @import("std");
 
 const parser = @import("../../root.zig");
@@ -5,29 +15,35 @@ pub const Lexer = parser.Lexer(Grammar);
 pub const Token = parser.Token(Grammar);
 
 pub const Whitespace = @import("whitespace.zig").Whitespace;
+pub const Keyword = @import("keyword.zig").Keyword;
 pub const Literal = @import("literal.zig").Literal;
 pub const Punctuator = @import("punctuator.zig").Punctuator;
 pub const Identifier = @import("identifier.zig").Identifier;
 
 pub const Grammar = union(enum) {
     whitespace: Whitespace,
+    keyword: Keyword,
     literal: Literal,
     punctuator: Punctuator,
     identifier: Identifier,
     end_of_file: void,
 
-    pub fn tokenize(lexer: *Lexer) Lexer.Error!void {
-        while (lexer.peakChar()) |_| {
-            Whitespace.tokenize(lexer) catch {
-                Literal.tokenize(lexer) catch {
-                    Punctuator.tokenize(lexer) catch {
-                        try Identifier.tokenize(lexer);
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            while (lexer.peekChar()) |_| {
+                Whitespace.tokenize(lexer) catch {
+                    Keyword.tokenize(lexer) catch {
+                        Literal.tokenize(lexer) catch {
+                            Punctuator.tokenize(lexer) catch {
+                                try Identifier.tokenize(lexer);
+                            };
+                        };
                     };
                 };
-            };
-        }
+            }
 
-        lexer.push(lexer.token(0, .end_of_file));
+            lexer.commit(lexer.start(.end_of_file));
+        }
     }
 };