From 0bc6d195195d1e126b535554a4a4105468cd06d9 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Mon, 3 Aug 2026 17:07:05 +0200 Subject: add tokenizer for whitespaces in js --- src/z/File.zig | 1 - src/z/parser/builtin.zig | 17 ----- src/z/parser/js/grammar/identifier.zig | 4 ++ src/z/parser/js/grammar/keyword.zig | 48 ++++++++++++++ src/z/parser/js/grammar/literal.zig | 5 ++ src/z/parser/js/grammar/punctuator.zig | 61 +++++++++++++++++ src/z/parser/js/grammar/root.zig | 36 ++++++++++ src/z/parser/js/grammar/whitespace.zig | 106 +++++++++++++++++++++++++++++ src/z/parser/js/lexical_grammar.zig | 3 - src/z/parser/js/root.zig | 6 +- src/z/parser/lexer.zig | 110 +++++++++++++++++++++++++++++++ src/z/parser/parser.zig | 117 --------------------------------- src/z/parser/result.zig | 4 -- src/z/parser/root.zig | 17 +++++ src/z/root.zig | 1 + 15 files changed, 389 insertions(+), 147 deletions(-) delete mode 100644 src/z/parser/builtin.zig create mode 100644 src/z/parser/js/grammar/identifier.zig create mode 100644 src/z/parser/js/grammar/keyword.zig create mode 100644 src/z/parser/js/grammar/literal.zig create mode 100644 src/z/parser/js/grammar/punctuator.zig create mode 100644 src/z/parser/js/grammar/root.zig create mode 100644 src/z/parser/js/grammar/whitespace.zig delete mode 100644 src/z/parser/js/lexical_grammar.zig create mode 100644 src/z/parser/lexer.zig delete mode 100644 src/z/parser/parser.zig delete mode 100644 src/z/parser/result.zig create mode 100644 src/z/parser/root.zig (limited to 'src/z') diff --git a/src/z/File.zig b/src/z/File.zig index cd7094f..cd19dba 100644 --- a/src/z/File.zig +++ b/src/z/File.zig @@ -15,4 +15,3 @@ pub fn load(comptime self: @This(), comptime content: []const u8) Element { return .transparent(.content(comptime self.minifier.minify(content))); } } - diff --git a/src/z/parser/builtin.zig b/src/z/parser/builtin.zig deleted file mode 100644 index fc96fc3..0000000 --- a/src/z/parser/builtin.zig +++ /dev/null @@ -1,17 +0,0 @@ -const std = @import("std"); - -const parser = @import("root.zig"); -const Parser = parser.Parser; -const Result = parser.Result; - -pub fn literal(lit: []const u8) Parser(void) { - return .fromType(struct { - pub fn parse(buffer: []const u8) !Result(void) { - if (std.mem.startsWith(u8, buffer, lit)) { - return .{ void{}, buffer[lit.len..] }; - } - - return error.Literal; - } - }); -} diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig new file mode 100644 index 0000000..038dbc5 --- /dev/null +++ b/src/z/parser/js/grammar/identifier.zig @@ -0,0 +1,4 @@ +pub const Identifier = enum { + public, + private, +}; diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig new file mode 100644 index 0000000..14c3272 --- /dev/null +++ b/src/z/parser/js/grammar/keyword.zig @@ -0,0 +1,48 @@ +pub const Keyword = enum { + @"break", + case, + @"catch", + class, + @"const", + @"continue", + debugger, + default, + delete, + do, + @"else", + @"export", + extends, + finally, + @"for", + function, + @"if", + import, + in, + instanceof, + new, + @"return", + super, + @"switch", + this, + throw, + @"try", + typeof, + @"var", + void, + @"while", + with, + yield, + let, + static, + await, + async, + of, + from, + get, + set, + meta, + @"enum", + null, + true, + false, +}; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..a2ef97a --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,5 @@ +pub const Literal = enum { + numeric, + bigint, + string, +}; diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig new file mode 100644 index 0000000..81b2e6e --- /dev/null +++ b/src/z/parser/js/grammar/punctuator.zig @@ -0,0 +1,61 @@ +pub const Punctuator = enum { + @"+", + @"-", + @"*", + @"/", + @"%", + @"**", + @"++", + @"--", + @"=", + @"+=", + @"-=", + @"*=", + @"/=", + @"%=", + @"**=", + @"&=", + @"|=", + @"^=", + @"<<=", + @">>=", + @">>>=", + @"&&=", + @"||=", + @"??=", + @"==", + @"===", + @"!=", + @"!==", + @"<", + @"<=", + @">", + @">=", + @"&&", + @"||", + @"??", + @"&", + @"|", + @"^", + @"~", + @"<<", + @">>", + @">>>", + @"!", + @"?", + @":", + @";", + @",", + @".", + @"...", + @"=>", + @"(", + @")", + @"[", + @"]", + @"{", + @"}", + @"?.", + @"#", + @"@", +}; diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig new file mode 100644 index 0000000..e1a1e96 --- /dev/null +++ b/src/z/parser/js/grammar/root.zig @@ -0,0 +1,36 @@ +const std = @import("std"); + +const parser = @import("../../root.zig"); +pub const Lexer = parser.Lexer(Grammar); +pub const Token = parser.Token(Grammar); + +pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Literal = @import("literal.zig").Literal; +pub const Punctuator = @import("punctuator.zig").Punctuator; +pub const Identifier = @import("identifier.zig").Identifier; + +pub const Grammar = union(enum) { + whitespace: Whitespace, + literal: Literal, + punctuator: Punctuator, + identifier: Identifier, + end_of_file: void, + + pub fn tokenize(lexer: *Lexer) Lexer.Error!void { + while (lexer.peakChar()) |_| { + Whitespace.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; + }; + } + + lexer.push(lexer.token(0, .end_of_file)); + } +}; + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..c3435c4 --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,106 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(undefined); + + switch (lexer.peekChar() orelse 0) { + ' ', '\t', '\r', '\n' => { + token.kind = .{ .whitespace = .space }; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekCharAt(1) orelse 0) { + '*' => { + token.kind = .{ .whitespace = .block_comment }; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind = .{ .whitespace = .line_comment }; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); + + inline for (lexer.tokens) |token| { + std.debug.print("{f}\n", .{token}); + } +} diff --git a/src/z/parser/js/lexical_grammar.zig b/src/z/parser/js/lexical_grammar.zig deleted file mode 100644 index 7060766..0000000 --- a/src/z/parser/js/lexical_grammar.zig +++ /dev/null @@ -1,3 +0,0 @@ -const parser = @import("root.zig"); -const Result = parser.Result; - diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig index a6be9ca..e604855 100644 --- a/src/z/parser/js/root.zig +++ b/src/z/parser/js/root.zig @@ -1,10 +1,6 @@ const std = @import("std"); -pub const lexical_grammar = @import("lexical_grammar.zig"); - -pub const Result = @import("result.zig").Result; -pub const Parser = @import("parser.zig").Parser; -pub const builtin = @import("builtin.zig"); +pub const grammar = @import("./grammar/root.zig"); test { _ = std.testing.refAllDecls(@This()); diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig new file mode 100644 index 0000000..d9a2868 --- /dev/null +++ b/src/z/parser/lexer.zig @@ -0,0 +1,110 @@ +const std = @import("std"); + +pub fn Token(TokenKind: type) type { + return struct { + const Self = @This(); + + kind: TokenKind, + slice: []const u8, + + pub fn start(comptime lexer: *const Lexer(TokenKind), kind: TokenKind) Self { + return .{ + .kind = kind, + .slice = lexer.buffer, + }; + } + + pub inline fn stop(comptime self: Self, comptime lexer: *const Lexer(TokenKind)) Self { + comptime { + const length = self.slice.len - lexer.buffer.len; + return .{ + .kind = self.kind, + .slice = self.slice[0..length], + }; + } + } + + pub fn format(self: *const Self, writer: *std.Io.Writer) !void { + try writer.print("Token{}(\"{s}\")", .{self.kind, self.slice}); + } + }; +} + +pub fn Lexer(TokenKind: type) type { + return struct { + const Self = @This(); + + pub const Error = error{ + UnexpectedToken, + EndOfBuffer, + }; + + buffer: []const u8, + last_buffer: []const u8, + tokens: []const Token(TokenKind) = &.{}, + + pub fn init(buffer: []const u8) Self { + return .{ + .buffer = buffer, + .last_buffer = buffer, + }; + } + + pub inline fn peekChar(self: *Self) ?u8 { + return self.peekCharAt(0); + } + + pub inline fn peekCharAt(self: *Self, n: usize) ?u8 { + return if (n < self.buffer.len) self.buffer[n] else null; + } + + pub fn peekSlice(self: *Self, length: usize) ?[]const u8 { + return self.peekSliceAt(length, 0); + } + + pub fn peekSliceAt(self: *Self, length: usize, offset: usize) ?[]const u8 { + if (length + offset <= self.buffer.len) { + return self.buffer[offset .. offset + length]; + } + return null; + } + + pub fn skip(self: *Self) !void { + return self.skipTo(1); + } + + pub fn skipTo(self: *Self, n: usize) Error!void { + if (n <= self.buffer.len) { + self.buffer = self.buffer[n..]; + } else { + return Error.EndOfBuffer; + } + } + + pub fn skipUntil(self: *Self, literal: []const u8) Error!void { + if (std.mem.find(u8, self.buffer, literal)) |index| { + self.buffer = self.buffer[index + literal.len..]; + } else { + return Error.EndOfBuffer; + } + } + + pub fn skipToEnd(self: *Self) void { + self.buffer = self.buffer[self.buffer.len..]; + } + + pub fn start(self: *Self, kind: TokenKind) Token(TokenKind) { + return .start(self, kind); + } + + pub fn revert(self: *Self) void { + self.buffer = self.last_buffer; + } + + pub fn commit(self: *Self, tok: Token(TokenKind)) void { + self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; + self.last_buffer = self.buffer; + } + + }; +} diff --git a/src/z/parser/parser.zig b/src/z/parser/parser.zig deleted file mode 100644 index b73d895..0000000 --- a/src/z/parser/parser.zig +++ /dev/null @@ -1,117 +0,0 @@ -const std = @import("std"); - -const mod = @import("root.zig"); -const Result = mod.Result; - -pub fn Parser(T: type) type { - const R = Result(T); - - return struct { - parse: *const fn ([]const u8) anyerror!R, - - pub fn parseAll(self: *const @This(), buffer: []const u8) !T { - const value, const next = try self.parse(buffer); - - if (next.len > 0) { - return error.NotExhaustive; - } - - return value; - } - - pub fn fromType(P: type) @This() { - return .{ .parse = P.parse }; - } - - pub fn variants(V: type) @This() { - if (@typeInfo(T) != .@"union") { - @compileError("variants is only allowed with an union as result type"); - } - - return .{ - .parse = struct { - fn parse(buffer: []const u8) !R { - inline for (std.meta.fields(T)) |field| { - if (@field(V, field.name).parse(buffer)) |result| { - const value, const next = result; - return .{ - @unionInit(T, field.name, value), - next, - }; - } else |_| { } - } - - return error.Variants; - } - }.parse, - }; - } - - pub fn any(P: type, comptime parsers: []const struct { Parser(P), T }) @This() { - return .{ - .parse = struct { - fn parse(buffer: []const u8) !R { - inline for (parsers) |pair| { - const parser, const value = pair; - - if (parser.parse(buffer)) |result| { - _, const next = result; - return .{ value, next }; - } else |_| { } - } - return error.Any; - } - }.parse, - }; - } - }; -} - -test "variants" { - const Char = union(enum) { - upper: u8, - lower: u8, - space: void, - }; - - const parser: Parser(Char) = .variants(struct { - const upper: Parser(u8) = .any(void, &.{ - .{ mod.builtin.literal("A"), 'A' }, - .{ mod.builtin.literal("B"), 'B' }, - }); - - const lower: Parser(u8) = .any(void, &.{ - .{ mod.builtin.literal("a"), 'a' }, - .{ mod.builtin.literal("b"), 'b' }, - }); - - const space: Parser(void) = mod.builtin.literal(" "); - }); - - { - const result = try parser.parseAll("a"); - try std.testing.expect(result.lower == 'a'); - } - - { - const result = try parser.parseAll("b"); - try std.testing.expect(result.lower == 'b'); - } - - { - const result = try parser.parseAll("A"); - try std.testing.expect(result.upper == 'A'); - } - - { - const result = try parser.parseAll("B"); - try std.testing.expect(result.upper == 'B'); - } - - { - const result = try parser.parseAll(" "); - try std.testing.expect(result == .space); - } - - try std.testing.expect(parser.parseAll("c") == error.Variants); -} diff --git a/src/z/parser/result.zig b/src/z/parser/result.zig deleted file mode 100644 index 5cc5878..0000000 --- a/src/z/parser/result.zig +++ /dev/null @@ -1,4 +0,0 @@ - -pub fn Result(T: type) type { - return struct { T, []const u8 }; -} diff --git a/src/z/parser/root.zig b/src/z/parser/root.zig new file mode 100644 index 0000000..e0b67fb --- /dev/null +++ b/src/z/parser/root.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +const lexer = @import("lexer.zig"); +pub const Lexer = lexer.Lexer; +pub const Token = lexer.Token; + +pub const js = @import("js/root.zig"); + +pub fn Parser(TokenType: type) type { + return struct { + buffer: []const TokenType, + }; +} + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/root.zig b/src/z/root.zig index b23ebc5..1fa8d99 100644 --- a/src/z/root.zig +++ b/src/z/root.zig @@ -3,6 +3,7 @@ const std = @import("std"); pub const Minifier = @import("Minifier.zig"); pub const Component = @import("Component.zig"); pub const File = @import("File.zig"); +pub const parser = @import("parser/root.zig"); const Element = @import("html").Element; -- cgit v1.2.3-70-g09d2