From 0bc6d195195d1e126b535554a4a4105468cd06d9 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Mon, 3 Aug 2026 17:07:05 +0200 Subject: add tokenizer for whitespaces in js --- src/z/parser/js/grammar/identifier.zig | 4 ++ src/z/parser/js/grammar/keyword.zig | 48 +++++++++++++++ src/z/parser/js/grammar/literal.zig | 5 ++ src/z/parser/js/grammar/punctuator.zig | 61 +++++++++++++++++++ src/z/parser/js/grammar/root.zig | 36 +++++++++++ src/z/parser/js/grammar/whitespace.zig | 106 +++++++++++++++++++++++++++++++++ src/z/parser/js/lexical_grammar.zig | 3 - src/z/parser/js/root.zig | 6 +- 8 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 src/z/parser/js/grammar/identifier.zig create mode 100644 src/z/parser/js/grammar/keyword.zig create mode 100644 src/z/parser/js/grammar/literal.zig create mode 100644 src/z/parser/js/grammar/punctuator.zig create mode 100644 src/z/parser/js/grammar/root.zig create mode 100644 src/z/parser/js/grammar/whitespace.zig delete mode 100644 src/z/parser/js/lexical_grammar.zig (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig new file mode 100644 index 0000000..038dbc5 --- /dev/null +++ b/src/z/parser/js/grammar/identifier.zig @@ -0,0 +1,4 @@ +pub const Identifier = enum { + public, + private, +}; diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig new file mode 100644 index 0000000..14c3272 --- /dev/null +++ b/src/z/parser/js/grammar/keyword.zig @@ -0,0 +1,48 @@ +pub const Keyword = enum { + @"break", + case, + @"catch", + class, + @"const", + @"continue", + debugger, + default, + delete, + do, + @"else", + @"export", + extends, + finally, + @"for", + function, + @"if", + import, + in, + instanceof, + new, + @"return", + super, + @"switch", + this, + throw, + @"try", + typeof, + @"var", + void, + @"while", + with, + yield, + let, + static, + await, + async, + of, + from, + get, + set, + meta, + @"enum", + null, + true, + false, +}; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..a2ef97a --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,5 @@ +pub const Literal = enum { + numeric, + bigint, + string, +}; diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig new file mode 100644 index 0000000..81b2e6e --- /dev/null +++ b/src/z/parser/js/grammar/punctuator.zig @@ -0,0 +1,61 @@ +pub const Punctuator = enum { + @"+", + @"-", + @"*", + @"/", + @"%", + @"**", + @"++", + @"--", + @"=", + @"+=", + @"-=", + @"*=", + @"/=", + @"%=", + @"**=", + @"&=", + @"|=", + @"^=", + @"<<=", + @">>=", + @">>>=", + @"&&=", + @"||=", + @"??=", + @"==", + @"===", + @"!=", + @"!==", + @"<", + @"<=", + @">", + @">=", + @"&&", + @"||", + @"??", + @"&", + @"|", + @"^", + @"~", + @"<<", + @">>", + @">>>", + @"!", + @"?", + @":", + @";", + @",", + @".", + @"...", + @"=>", + @"(", + @")", + @"[", + @"]", + @"{", + @"}", + @"?.", + @"#", + @"@", +}; diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig new file mode 100644 index 0000000..e1a1e96 --- /dev/null +++ b/src/z/parser/js/grammar/root.zig @@ -0,0 +1,36 @@ +const std = @import("std"); + +const parser = @import("../../root.zig"); +pub const Lexer = parser.Lexer(Grammar); +pub const Token = parser.Token(Grammar); + +pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Literal = @import("literal.zig").Literal; +pub const Punctuator = @import("punctuator.zig").Punctuator; +pub const Identifier = @import("identifier.zig").Identifier; + +pub const Grammar = union(enum) { + whitespace: Whitespace, + literal: Literal, + punctuator: Punctuator, + identifier: Identifier, + end_of_file: void, + + pub fn tokenize(lexer: *Lexer) Lexer.Error!void { + while (lexer.peakChar()) |_| { + Whitespace.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; + }; + } + + lexer.push(lexer.token(0, .end_of_file)); + } +}; + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..c3435c4 --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,106 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(undefined); + + switch (lexer.peekChar() orelse 0) { + ' ', '\t', '\r', '\n' => { + token.kind = .{ .whitespace = .space }; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekCharAt(1) orelse 0) { + '*' => { + token.kind = .{ .whitespace = .block_comment }; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind = .{ .whitespace = .line_comment }; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); + + inline for (lexer.tokens) |token| { + std.debug.print("{f}\n", .{token}); + } +} diff --git a/src/z/parser/js/lexical_grammar.zig b/src/z/parser/js/lexical_grammar.zig deleted file mode 100644 index 7060766..0000000 --- a/src/z/parser/js/lexical_grammar.zig +++ /dev/null @@ -1,3 +0,0 @@ -const parser = @import("root.zig"); -const Result = parser.Result; - diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig index a6be9ca..e604855 100644 --- a/src/z/parser/js/root.zig +++ b/src/z/parser/js/root.zig @@ -1,10 +1,6 @@ const std = @import("std"); -pub const lexical_grammar = @import("lexical_grammar.zig"); - -pub const Result = @import("result.zig").Result; -pub const Parser = @import("parser.zig").Parser; -pub const builtin = @import("builtin.zig"); +pub const grammar = @import("./grammar/root.zig"); test { _ = std.testing.refAllDecls(@This()); -- cgit v1.2.3-70-g09d2