diff options
Diffstat (limited to 'src/z/parser/js/grammar/literal.zig')
| -rw-r--r-- | src/z/parser/js/grammar/literal.zig | 192 |
1 files changed, 192 insertions, 0 deletions
diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..4316dee --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,192 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + +pub const Literal = union(enum) { + // NOTE: currently only decimal literals are tokenized + numeric: void, + bigint: void, + string: void, + template: enum { + whole, + start, + middle, + end, + }, + regex: void, + + const digits = "_0123456789abcdef"; + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(.{ .literal = undefined }); + + const regex_possible = lexer.context.regex_possible; + lexer.context.regex_possible = false; + + switch (try lexer.consume()) { + '0'...'9', '.' => |c| { + const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { + 'x' => 16, + 'o' => 8, + 'b' => 2, + else => 10, + }; + + if (base != 10) lexer.skip() catch unreachable; + + { + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + + + if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken; + } + + if (base == 10 and lexer.peekChar() == '.') { + try lexer.skip(); + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + } + + if (lexer.peekChar() == 'n') { + lexer.skip() catch unreachable; + token.kind.literal = .bigint; + } else { + token.kind.literal = .numeric; + } + }, + '\'', '"' => |quote| { + token.kind.literal = .string; + while (true) { + switch (try lexer.consume()) { + quote => break, + '\\' => try lexer.skip(), + else => {}, + } + } + }, + '`' => { + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .whole; + break; + }, + '\\' => try lexer.skip(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .start; + lexer.context.regex_possible = true; + break; + }, + else => {}, + } + } + }, + '}' => if (lexer.context.template_nesting > 0) { + lexer.context.template_nesting -= 1; + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .end; + break; + }, + '\\' => try lexer.skip(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .middle; + lexer.context.regex_possible = true; + break; + }, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, + '/' => if (regex_possible) { + token.kind.literal = .regex; + while (true) { + switch (try lexer.consume()) { + '/' => break, + '\\' => try lexer.skip(), + '\n' => return Lexer.Error.UnexpectedToken, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "numeric" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("1234", 4); + try Simple.run("1.234", 5); + try Simple.run("1_2_3_4", 7); + try Simple.run("0b101011", 8); + try Simple.run("0xabcf01", 8); + try Simple.run("0o731734", 8); + try Simple.run("0o731734n", 9); +} + +test "string" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("'hello'", 7); + try Simple.run("\"hello\"", 7); + try std.testing.expectError( + Lexer.Error.EndOfBuffer, + Simple.run("'hello", 0), + ); +} + +test "regex" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("/hello/", 7); + try Simple.run("/some string\\/ here/", 20); +} |