add tokenizer for whitespaces in js

1 files changed, 106 insertions, 0 deletions

diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig
new file mode 100644
index 0000000..c3435c4
--- /dev/null
+++ b/src/z/parser/js/grammar/whitespace.zig
@@ -0,0 +1,106 @@
+const std = @import("std");
+const Lexer = @import("root.zig").Lexer;
+const Token = @import("root.zig").Token;
+
+pub const Whitespace = enum {
+    space,
+    line_comment,
+    block_comment,
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            errdefer lexer.revert();
+
+            var token = lexer.start(undefined);
+
+            switch (lexer.peekChar() orelse 0) {
+                ' ', '\t', '\r', '\n' => {
+                    token.kind = .{ .whitespace = .space };
+
+                    while (lexer.peekChar()) |ch| {
+                        switch (ch) {
+                            ' ', '\t', '\r', '\n' => try lexer.skip(),
+                            else => break,
+                        }
+                    }
+                },
+                '/' => {
+                    switch (lexer.peekCharAt(1) orelse 0) {
+                        '*' => {
+                            token.kind = .{ .whitespace = .block_comment };
+                            try lexer.skipUntil("*/");
+                        },
+                        '/' => {
+                            token.kind = .{ .whitespace = .line_comment };
+                            lexer.skipUntil("\n") catch lexer.skipToEnd();
+                        },
+                        else => return Lexer.Error.UnexpectedToken,
+                    }
+                },
+                else => return Lexer.Error.UnexpectedToken,
+            }
+
+            lexer.commit(token);
+        }
+    }
+};
+
+test "space" {
+    {
+        comptime var lexer: Lexer = .init(" ");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(1, lexer.tokens[0].slice.len);
+    }
+
+    {
+        comptime var lexer: Lexer = .init(" \t\t   \n\r  ");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(10, lexer.tokens[0].slice.len);
+    }
+}
+
+test "line-comment" {
+    {
+        comptime var lexer: Lexer = .init("// some comment");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice);
+    }
+
+    {
+        comptime var lexer: Lexer = .init("// some comment\n");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice);
+    }
+}
+
+test "block-comment" {
+    {
+        comptime var lexer: Lexer = .init("/* some comment */");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice);
+    }
+}
+
+test "mixed" {
+    comptime var lexer: Lexer = .init(
+        \\  // some comment
+        \\/* other comment here */     // comment or so
+        \\            /* multi
+        \\             * line
+        \\             * comment
+        \\             */
+    );
+    inline while (Whitespace.tokenize(&lexer)) {
+    } else |_| {}
+
+    try std.testing.expectEqual(7, lexer.tokens.len);
+
+    inline for (lexer.tokens) |token| {
+        std.debug.print("{f}\n", .{token});
+    }
+}