From 0bc6d195195d1e126b535554a4a4105468cd06d9 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Mon, 3 Aug 2026 17:07:05 +0200 Subject: add tokenizer for whitespaces in js --- src/z/parser/js/grammar/identifier.zig | 4 ++ src/z/parser/js/grammar/keyword.zig | 48 +++++++++++++++ src/z/parser/js/grammar/literal.zig | 5 ++ src/z/parser/js/grammar/punctuator.zig | 61 +++++++++++++++++++ src/z/parser/js/grammar/root.zig | 36 +++++++++++ src/z/parser/js/grammar/whitespace.zig | 106 +++++++++++++++++++++++++++++++++ 6 files changed, 260 insertions(+) create mode 100644 src/z/parser/js/grammar/identifier.zig create mode 100644 src/z/parser/js/grammar/keyword.zig create mode 100644 src/z/parser/js/grammar/literal.zig create mode 100644 src/z/parser/js/grammar/punctuator.zig create mode 100644 src/z/parser/js/grammar/root.zig create mode 100644 src/z/parser/js/grammar/whitespace.zig (limited to 'src/z/parser/js/grammar') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig new file mode 100644 index 0000000..038dbc5 --- /dev/null +++ b/src/z/parser/js/grammar/identifier.zig @@ -0,0 +1,4 @@ +pub const Identifier = enum { + public, + private, +}; diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig new file mode 100644 index 0000000..14c3272 --- /dev/null +++ b/src/z/parser/js/grammar/keyword.zig @@ -0,0 +1,48 @@ +pub const Keyword = enum { + @"break", + case, + @"catch", + class, + @"const", + @"continue", + debugger, + default, + delete, + do, + @"else", + @"export", + extends, + finally, + @"for", + function, + @"if", + import, + in, + instanceof, + new, + @"return", + super, + @"switch", + this, + throw, + @"try", + typeof, + @"var", + void, + @"while", + with, + yield, + let, + static, + await, + async, + of, + from, + get, + set, + meta, + @"enum", + null, + true, + false, +}; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..a2ef97a --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,5 @@ +pub const Literal = enum { + numeric, + bigint, + string, +}; diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig new file mode 100644 index 0000000..81b2e6e --- /dev/null +++ b/src/z/parser/js/grammar/punctuator.zig @@ -0,0 +1,61 @@ +pub const Punctuator = enum { + @"+", + @"-", + @"*", + @"/", + @"%", + @"**", + @"++", + @"--", + @"=", + @"+=", + @"-=", + @"*=", + @"/=", + @"%=", + @"**=", + @"&=", + @"|=", + @"^=", + @"<<=", + @">>=", + @">>>=", + @"&&=", + @"||=", + @"??=", + @"==", + @"===", + @"!=", + @"!==", + @"<", + @"<=", + @">", + @">=", + @"&&", + @"||", + @"??", + @"&", + @"|", + @"^", + @"~", + @"<<", + @">>", + @">>>", + @"!", + @"?", + @":", + @";", + @",", + @".", + @"...", + @"=>", + @"(", + @")", + @"[", + @"]", + @"{", + @"}", + @"?.", + @"#", + @"@", +}; diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig new file mode 100644 index 0000000..e1a1e96 --- /dev/null +++ b/src/z/parser/js/grammar/root.zig @@ -0,0 +1,36 @@ +const std = @import("std"); + +const parser = @import("../../root.zig"); +pub const Lexer = parser.Lexer(Grammar); +pub const Token = parser.Token(Grammar); + +pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Literal = @import("literal.zig").Literal; +pub const Punctuator = @import("punctuator.zig").Punctuator; +pub const Identifier = @import("identifier.zig").Identifier; + +pub const Grammar = union(enum) { + whitespace: Whitespace, + literal: Literal, + punctuator: Punctuator, + identifier: Identifier, + end_of_file: void, + + pub fn tokenize(lexer: *Lexer) Lexer.Error!void { + while (lexer.peakChar()) |_| { + Whitespace.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; + }; + } + + lexer.push(lexer.token(0, .end_of_file)); + } +}; + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..c3435c4 --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,106 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(undefined); + + switch (lexer.peekChar() orelse 0) { + ' ', '\t', '\r', '\n' => { + token.kind = .{ .whitespace = .space }; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekCharAt(1) orelse 0) { + '*' => { + token.kind = .{ .whitespace = .block_comment }; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind = .{ .whitespace = .line_comment }; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); + + inline for (lexer.tokens) |token| { + std.debug.print("{f}\n", .{token}); + } +} -- cgit v1.2.3-70-g09d2 From a10b091d78208907397a6d49791b27e04e675161 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Tue, 4 Aug 2026 19:02:27 +0200 Subject: add tokenizer for all other variants --- src/z/parser/js/grammar/identifier.zig | 54 +++++++++++++++++++++++ src/z/parser/js/grammar/keyword.zig | 65 +++++++++++++++++++++++++++ src/z/parser/js/grammar/literal.zig | 81 ++++++++++++++++++++++++++++++++++ src/z/parser/js/grammar/punctuator.zig | 54 +++++++++++++++++++++++ src/z/parser/js/grammar/root.zig | 34 ++++++++++---- src/z/parser/js/grammar/whitespace.zig | 16 +++---- src/z/parser/lexer.zig | 14 ++++++ 7 files changed, 299 insertions(+), 19 deletions(-) (limited to 'src/z/parser/js/grammar') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig index 038dbc5..905f937 100644 --- a/src/z/parser/js/grammar/identifier.zig +++ b/src/z/parser/js/grammar/identifier.zig @@ -1,4 +1,58 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Identifier = enum { public, private, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + defer lexer.revert(); + + const token = tok: { + if (lexer.peekChar() == '#') { + const t = lexer.start(.{ .identifier = .private }); + lexer.skip() catch unreachable; + break :tok t; + } else { + break :tok lexer.start(.{ .identifier = .public }); + } + }; + + switch (try lexer.consume()) { + 'a'...'z', 'A'...'Z', '_' => {}, + else => return Lexer.Error.UnexpectedToken, + } + + while (lexer.peekChar()) |ch| { + switch (ch) { + 'a'...'z', 'A'...'Z', '0'...'9', '_' => {}, + else => break, + } + + try lexer.skip(); + } + + lexer.commit(token); + } + } }; + +test "simple" { + { + comptime var lexer: Lexer = .init("some_literal9"); + try Identifier.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(13, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init("#some_literal9"); + try Identifier.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(14, lexer.tokens[0].slice.len); + } +} diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig index 14c3272..c3b32ac 100644 --- a/src/z/parser/js/grammar/keyword.zig +++ b/src/z/parser/js/grammar/keyword.zig @@ -1,3 +1,9 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Keyword = enum { @"break", case, @@ -45,4 +51,63 @@ pub const Keyword = enum { null, true, false, + + const Self = @This(); + + const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: { + var keywords: []const struct { []const u8, Self } = &.{}; + + for (@typeInfo(Self).@"enum".fields) |field| { + const name: []const u8 = field.name; + const value: Self = @field(Self, field.name); + + keywords = keywords ++ .{ .{ name, value } }; + } + + break :kws keywords; + }); + + pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void { + comptime { + @setEvalBranchQuota(10000); + errdefer lexer.revert(); + + const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; + + if (index == 0) return Lexer.Error.UnexpectedToken; + + if (keywordmap.get(lexer.buffer[0..index])) |tag| { + const token = lexer.start(.{ .keyword = tag }); + try lexer.skipTo(index); + lexer.commit(token); + } else { + return Lexer.Error.UnexpectedToken; + } + } + } }; + +test "simple" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key); + try Keyword.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "ending" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "."); + try Keyword.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "subword" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "a"); + try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer)); + } +} diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index a2ef97a..7f92240 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -1,5 +1,86 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Literal = enum { + // NOTE: currently only decimal literals are tokenized numeric, bigint, string, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(.{ .literal = undefined }); + + switch (try lexer.consume()) { + '0'...'9', '.' => |c| { + token.kind.literal = .numeric; + + var had_period = c == '.'; + + while (lexer.peekChar()) |ch| { + switch (ch) { + '0'...'9' => {}, + '.' => if (had_period) { + return Lexer.Error.UnexpectedToken; + } else { + had_period = true; + }, + else => return Lexer.Error.UnexpectedToken, + } + + try lexer.skip(); + } + }, + '\'', '"' => |quote| while (try lexer.consume() != quote) { + if (quote == '\\') { + try lexer.consume(); + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } }; + +test "numeric" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("1234", 4); + try Simple.run("1.234", 5); + try std.testing.expectError( + Lexer.Error.UnexpectedToken, + Simple.run("12.3.4", 0), + ); +} + +test "string" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("'hello'", 7); + try Simple.run("\"hello\"", 7); + try std.testing.expectError( + Lexer.Error.EndOfBuffer, + Simple.run("'hello", 0), + ); +} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index 81b2e6e..a72ab8a 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -1,3 +1,9 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Punctuator = enum { @"+", @"-", @@ -58,4 +64,52 @@ pub const Punctuator = enum { @"?.", @"#", @"@", + + const Self = @This(); + + const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: { + var punctuators: []const struct { []const u8, Self } = &.{}; + + for (@typeInfo(Self).@"enum".fields) |field| { + const name: []const u8 = field.name; + const value: Self = @field(Self, field.name); + + punctuators = punctuators ++ .{.{ name, value }}; + } + + break :kws punctuators; + }); + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + @setEvalBranchQuota(10_000); + defer lexer.revert(); + + if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { + const token = lexer.start(.{ .punctuator = pair.value }); + try lexer.skipTo(pair.key.len); + lexer.commit(token); + } else { + return Lexer.Error.UnexpectedToken; + } + } + } }; + +test "simple" { + inline for (comptime Punctuator.punctuatormap.keys()) |key| { + comptime var lexer: Lexer = .init(key); + try Punctuator.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "ending" { + inline for (comptime Punctuator.punctuatormap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "a"); + try Punctuator.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index e1a1e96..03b386e 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -1,3 +1,13 @@ +//! This is a Javascript parser and tokenizer. +//! It is and probably will never be complete, +//! since js is a very strange language and I want +//! to move forward and code on more interesting things +//! I started something called mujs (micro-js). It's a +//! subset of the javascript language. It does not support +//! the regex literal and currently also not template strings. +//! It is chosen like that to make parsing easier and also less +//! context-dependent. + const std = @import("std"); const parser = @import("../../root.zig"); @@ -5,29 +15,35 @@ pub const Lexer = parser.Lexer(Grammar); pub const Token = parser.Token(Grammar); pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Keyword = @import("keyword.zig").Keyword; pub const Literal = @import("literal.zig").Literal; pub const Punctuator = @import("punctuator.zig").Punctuator; pub const Identifier = @import("identifier.zig").Identifier; pub const Grammar = union(enum) { whitespace: Whitespace, + keyword: Keyword, literal: Literal, punctuator: Punctuator, identifier: Identifier, end_of_file: void, - pub fn tokenize(lexer: *Lexer) Lexer.Error!void { - while (lexer.peakChar()) |_| { - Whitespace.tokenize(lexer) catch { - Literal.tokenize(lexer) catch { - Punctuator.tokenize(lexer) catch { - try Identifier.tokenize(lexer); + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + while (lexer.peekChar()) |_| { + Whitespace.tokenize(lexer) catch { + Keyword.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; }; }; - }; - } + } - lexer.push(lexer.token(0, .end_of_file)); + lexer.commit(lexer.start(.end_of_file)); + } } }; diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig index c3435c4..08d031f 100644 --- a/src/z/parser/js/grammar/whitespace.zig +++ b/src/z/parser/js/grammar/whitespace.zig @@ -11,11 +11,11 @@ pub const Whitespace = enum { comptime { errdefer lexer.revert(); - var token = lexer.start(undefined); + var token = lexer.start(.{ .whitespace = undefined }); - switch (lexer.peekChar() orelse 0) { + switch (try lexer.consume()) { ' ', '\t', '\r', '\n' => { - token.kind = .{ .whitespace = .space }; + token.kind.whitespace = .space; while (lexer.peekChar()) |ch| { switch (ch) { @@ -25,13 +25,13 @@ pub const Whitespace = enum { } }, '/' => { - switch (lexer.peekCharAt(1) orelse 0) { + switch (lexer.peekChar() orelse 0) { '*' => { - token.kind = .{ .whitespace = .block_comment }; + token.kind.whitespace = .block_comment; try lexer.skipUntil("*/"); }, '/' => { - token.kind = .{ .whitespace = .line_comment }; + token.kind.whitespace = .line_comment; lexer.skipUntil("\n") catch lexer.skipToEnd(); }, else => return Lexer.Error.UnexpectedToken, @@ -99,8 +99,4 @@ test "mixed" { } else |_| {} try std.testing.expectEqual(7, lexer.tokens.len); - - inline for (lexer.tokens) |token| { - std.debug.print("{f}\n", .{token}); - } } diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index d9a2868..40e1209 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -69,6 +69,20 @@ pub fn Lexer(TokenKind: type) type { return null; } + pub fn consume(self: *Self) !u8 { + return (try self.consumeTo(1))[0]; + } + + pub fn consumeTo(self: *Self, n: usize) ![]const u8 { + if (n <= self.buffer.len) { + const slice = self.buffer[0..n]; + self.buffer = self.buffer[n..]; + return slice; + } else { + return Error.EndOfBuffer; + } + } + pub fn skip(self: *Self) !void { return self.skipTo(1); } -- cgit v1.2.3-70-g09d2 From 06d1607aabad0a967c0500efe305de3fa732f8d3 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Tue, 4 Aug 2026 20:42:35 +0200 Subject: add numeric tokenizer --- src/z/parser/js/grammar/literal.zig | 68 +++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 21 deletions(-) (limited to 'src/z/parser/js/grammar') diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 7f92240..6a91291 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -10,6 +10,8 @@ pub const Literal = enum { bigint, string, + const digits = "_0123456789abcdef"; + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { comptime { errdefer lexer.revert(); @@ -18,27 +20,50 @@ pub const Literal = enum { switch (try lexer.consume()) { '0'...'9', '.' => |c| { - token.kind.literal = .numeric; - - var had_period = c == '.'; - - while (lexer.peekChar()) |ch| { - switch (ch) { - '0'...'9' => {}, - '.' => if (had_period) { - return Lexer.Error.UnexpectedToken; - } else { - had_period = true; - }, - else => return Lexer.Error.UnexpectedToken, - } + const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { + 'x' => 16, + 'o' => 8, + 'b' => 2, + else => 10, + }; + + if (base != 10) lexer.skip() catch unreachable; + + { + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + + + if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken; + } + if (base == 10 and lexer.peekChar() == '.') { try lexer.skip(); + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + } + + if (lexer.peekChar() == 'n') { + lexer.skip() catch unreachable; + token.kind.literal = .bigint; + } else { + token.kind.literal = .numeric; } }, - '\'', '"' => |quote| while (try lexer.consume() != quote) { - if (quote == '\\') { - try lexer.consume(); + '\'', '"' => |quote| { + token.kind.literal = .string; + while (try lexer.consume() != quote) { + if (quote == '\\') { + try lexer.consume(); + } } }, else => return Lexer.Error.UnexpectedToken, @@ -61,10 +86,11 @@ test "numeric" { try Simple.run("1234", 4); try Simple.run("1.234", 5); - try std.testing.expectError( - Lexer.Error.UnexpectedToken, - Simple.run("12.3.4", 0), - ); + try Simple.run("1_2_3_4", 7); + try Simple.run("0b101011", 8); + try Simple.run("0xabcf01", 8); + try Simple.run("0o731734", 8); + try Simple.run("0o731734n", 9); } test "string" { -- cgit v1.2.3-70-g09d2 From 356c2bc0ccb1794b5ec219079d8c5b85ddcc4c87 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Tue, 4 Aug 2026 22:28:25 +0200 Subject: add templates to tokenizer --- src/z/parser/js/grammar/literal.zig | 63 ++++++++++++++++++++++++++++++---- src/z/parser/js/grammar/punctuator.zig | 5 +++ src/z/parser/js/grammar/root.zig | 13 +++++++ src/z/parser/lexer.zig | 2 +- 4 files changed, 75 insertions(+), 8 deletions(-) (limited to 'src/z/parser/js/grammar') diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 6a91291..039ebf4 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -4,11 +4,17 @@ const root = @import("root.zig"); const Lexer = root.Lexer; const Token = root.Token; -pub const Literal = enum { +pub const Literal = union(enum) { // NOTE: currently only decimal literals are tokenized - numeric, - bigint, - string, + numeric: void, + bigint: void, + string: void, + template: enum { + whole, + start, + middle, + end, + }, const digits = "_0123456789abcdef"; @@ -60,9 +66,52 @@ pub const Literal = enum { }, '\'', '"' => |quote| { token.kind.literal = .string; - while (try lexer.consume() != quote) { - if (quote == '\\') { - try lexer.consume(); + while (true) { + switch (try lexer.consume()) { + quote => break, + '\\' => try lexer.consume(), + else => {}, + } + } + }, + '`' => { + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .whole; + break; + }, + '\\' => try lexer.consume(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .start; + break; + }, + else => {}, + } + } + }, + '}' => if (lexer.context.template_nesting > 0) { + lexer.context.template_nesting -= 1; + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .end; + break; + }, + '\\' => try lexer.consume(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .middle; + break; + }, + else => {}, } } }, diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index a72ab8a..4fd5d16 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -86,6 +86,11 @@ pub const Punctuator = enum { defer lexer.revert(); if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { + + if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { + return Lexer.Error.UnexpectedToken; + } + const token = lexer.start(.{ .punctuator = pair.value }); try lexer.skipTo(pair.key.len); lexer.commit(token); diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index 03b386e..56f3990 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -11,6 +11,7 @@ const std = @import("std"); const parser = @import("../../root.zig"); + pub const Lexer = parser.Lexer(Grammar); pub const Token = parser.Token(Grammar); @@ -28,7 +29,12 @@ pub const Grammar = union(enum) { identifier: Identifier, end_of_file: void, + pub const Context = struct { + template_nesting: usize = 0, + }; + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + @setEvalBranchQuota(lexer.buffer.len * 1000); comptime { while (lexer.peekChar()) |_| { Whitespace.tokenize(lexer) catch { @@ -50,3 +56,10 @@ pub const Grammar = union(enum) { test { _ = std.testing.refAllDecls(@This()); } + +test "z.js" { + comptime var lexer: Lexer = .init(@embedFile("../../../z.js")); + Grammar.tokenize(&lexer) catch |err| { + std.debug.print("error: {} at {s}", .{ err, lexer.buffer }); + }; +} diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index 40e1209..66d4211 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -42,6 +42,7 @@ pub fn Lexer(TokenKind: type) type { buffer: []const u8, last_buffer: []const u8, tokens: []const Token(TokenKind) = &.{}, + context: TokenKind.Context = .{}, pub fn init(buffer: []const u8) Self { return .{ @@ -119,6 +120,5 @@ pub fn Lexer(TokenKind: type) type { self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; self.last_buffer = self.buffer; } - }; } -- cgit v1.2.3-70-g09d2 From 0eaa2a420cb699cdb23ce9b1573560a5d9e59a31 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Wed, 5 Aug 2026 09:08:24 +0200 Subject: add regex literals --- src/z/parser/js/grammar/identifier.zig | 2 ++ src/z/parser/js/grammar/keyword.zig | 2 ++ src/z/parser/js/grammar/literal.zig | 37 ++++++++++++++++++++++++++++++---- src/z/parser/js/grammar/punctuator.zig | 2 ++ src/z/parser/js/grammar/root.zig | 1 + src/z/parser/lexer.zig | 3 +++ 6 files changed, 43 insertions(+), 4 deletions(-) (limited to 'src/z/parser/js/grammar') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig index 905f937..ccc5268 100644 --- a/src/z/parser/js/grammar/identifier.zig +++ b/src/z/parser/js/grammar/identifier.zig @@ -12,6 +12,8 @@ pub const Identifier = enum { comptime { defer lexer.revert(); + lexer.context.regex_possible = false; + const token = tok: { if (lexer.peekChar() == '#') { const t = lexer.start(.{ .identifier = .private }); diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig index c3b32ac..8e99b4a 100644 --- a/src/z/parser/js/grammar/keyword.zig +++ b/src/z/parser/js/grammar/keyword.zig @@ -72,6 +72,8 @@ pub const Keyword = enum { @setEvalBranchQuota(10000); errdefer lexer.revert(); + lexer.context.regex_possible = true; + const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; if (index == 0) return Lexer.Error.UnexpectedToken; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 039ebf4..71b990d 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -24,6 +24,9 @@ pub const Literal = union(enum) { var token = lexer.start(.{ .literal = undefined }); + const regex_possible = lexer.context.regex_possible; + lexer.context.regex_possible = false; + switch (try lexer.consume()) { '0'...'9', '.' => |c| { const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { @@ -69,7 +72,7 @@ pub const Literal = union(enum) { while (true) { switch (try lexer.consume()) { quote => break, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), else => {}, } } @@ -83,11 +86,12 @@ pub const Literal = union(enum) { token.kind.literal.template = .whole; break; }, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .start; + lexer.context.regex_possible = true; break; }, else => {}, @@ -104,17 +108,28 @@ pub const Literal = union(enum) { token.kind.literal.template = .end; break; }, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .middle; + lexer.context.regex_possible = true; break; }, else => {}, } } - }, + } else return Lexer.Error.UnexpectedToken, + '/' => if (regex_possible) { + while (true) { + switch (try lexer.consume()) { + '/' => break, + '\\' => try lexer.skip(), + '\n' => return Lexer.Error.UnexpectedToken, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, else => return Lexer.Error.UnexpectedToken, } @@ -159,3 +174,17 @@ test "string" { Simple.run("'hello", 0), ); } + +test "regex" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("/hello/", 7); + try Simple.run("/some string\\/ here/", 20); +} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index 4fd5d16..b085ce1 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -85,6 +85,8 @@ pub const Punctuator = enum { @setEvalBranchQuota(10_000); defer lexer.revert(); + lexer.context.regex_possible = true; + if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index 56f3990..f732b7f 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -31,6 +31,7 @@ pub const Grammar = union(enum) { pub const Context = struct { template_nesting: usize = 0, + regex_possible: bool = true, }; pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index 66d4211..5beefde 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -43,6 +43,7 @@ pub fn Lexer(TokenKind: type) type { last_buffer: []const u8, tokens: []const Token(TokenKind) = &.{}, context: TokenKind.Context = .{}, + last_context: TokenKind.Context = .{}, pub fn init(buffer: []const u8) Self { return .{ @@ -114,11 +115,13 @@ pub fn Lexer(TokenKind: type) type { pub fn revert(self: *Self) void { self.buffer = self.last_buffer; + self.context = self.last_context; } pub fn commit(self: *Self, tok: Token(TokenKind)) void { self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; self.last_buffer = self.buffer; + self.last_context = self.context; } }; } -- cgit v1.2.3-70-g09d2 From 9b63c0f568832085b75ea6e3e983dc6cbeccfea4 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Wed, 5 Aug 2026 09:10:06 +0200 Subject: regex literal set token kind --- src/z/parser/js/grammar/literal.zig | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/z/parser/js/grammar') diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 71b990d..4316dee 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -15,6 +15,7 @@ pub const Literal = union(enum) { middle, end, }, + regex: void, const digits = "_0123456789abcdef"; @@ -121,6 +122,7 @@ pub const Literal = union(enum) { } } else return Lexer.Error.UnexpectedToken, '/' => if (regex_possible) { + token.kind.literal = .regex; while (true) { switch (try lexer.consume()) { '/' => break, -- cgit v1.2.3-70-g09d2