diff options
Diffstat (limited to 'src/z/parser')
| -rw-r--r-- | src/z/parser/js/grammar/identifier.zig | 2 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/keyword.zig | 2 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/literal.zig | 37 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/punctuator.zig | 2 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/root.zig | 1 | ||||
| -rw-r--r-- | src/z/parser/lexer.zig | 3 |
6 files changed, 43 insertions, 4 deletions
diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig index 905f937..ccc5268 100644 --- a/src/z/parser/js/grammar/identifier.zig +++ b/src/z/parser/js/grammar/identifier.zig @@ -12,6 +12,8 @@ pub const Identifier = enum { comptime { defer lexer.revert(); + lexer.context.regex_possible = false; + const token = tok: { if (lexer.peekChar() == '#') { const t = lexer.start(.{ .identifier = .private }); diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig index c3b32ac..8e99b4a 100644 --- a/src/z/parser/js/grammar/keyword.zig +++ b/src/z/parser/js/grammar/keyword.zig @@ -72,6 +72,8 @@ pub const Keyword = enum { @setEvalBranchQuota(10000); errdefer lexer.revert(); + lexer.context.regex_possible = true; + const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; if (index == 0) return Lexer.Error.UnexpectedToken; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 039ebf4..71b990d 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -24,6 +24,9 @@ pub const Literal = union(enum) { var token = lexer.start(.{ .literal = undefined }); + const regex_possible = lexer.context.regex_possible; + lexer.context.regex_possible = false; + switch (try lexer.consume()) { '0'...'9', '.' => |c| { const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { @@ -69,7 +72,7 @@ pub const Literal = union(enum) { while (true) { switch (try lexer.consume()) { quote => break, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), else => {}, } } @@ -83,11 +86,12 @@ pub const Literal = union(enum) { token.kind.literal.template = .whole; break; }, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .start; + lexer.context.regex_possible = true; break; }, else => {}, @@ -104,17 +108,28 @@ pub const Literal = union(enum) { token.kind.literal.template = .end; break; }, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .middle; + lexer.context.regex_possible = true; break; }, else => {}, } } - }, + } else return Lexer.Error.UnexpectedToken, + '/' => if (regex_possible) { + while (true) { + switch (try lexer.consume()) { + '/' => break, + '\\' => try lexer.skip(), + '\n' => return Lexer.Error.UnexpectedToken, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, else => return Lexer.Error.UnexpectedToken, } @@ -159,3 +174,17 @@ test "string" { Simple.run("'hello", 0), ); } + +test "regex" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("/hello/", 7); + try Simple.run("/some string\\/ here/", 20); +} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index 4fd5d16..b085ce1 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -85,6 +85,8 @@ pub const Punctuator = enum { @setEvalBranchQuota(10_000); defer lexer.revert(); + lexer.context.regex_possible = true; + if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index 56f3990..f732b7f 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -31,6 +31,7 @@ pub const Grammar = union(enum) { pub const Context = struct { template_nesting: usize = 0, + regex_possible: bool = true, }; pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index 66d4211..5beefde 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -43,6 +43,7 @@ pub fn Lexer(TokenKind: type) type { last_buffer: []const u8, tokens: []const Token(TokenKind) = &.{}, context: TokenKind.Context = .{}, + last_context: TokenKind.Context = .{}, pub fn init(buffer: []const u8) Self { return .{ @@ -114,11 +115,13 @@ pub fn Lexer(TokenKind: type) type { pub fn revert(self: *Self) void { self.buffer = self.last_buffer; + self.context = self.last_context; } pub fn commit(self: *Self, tok: Token(TokenKind)) void { self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; self.last_buffer = self.buffer; + self.last_context = self.context; } }; } |