aboutsummaryrefslogtreecommitdiff
path: root/src/z/parser/js/grammar/literal.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/z/parser/js/grammar/literal.zig')
-rw-r--r--src/z/parser/js/grammar/literal.zig192
1 files changed, 192 insertions, 0 deletions
diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
new file mode 100644
index 0000000..4316dee
--- /dev/null
+++ b/src/z/parser/js/grammar/literal.zig
@@ -0,0 +1,192 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
+pub const Literal = union(enum) {
+ // NOTE: currently only decimal literals are tokenized
+ numeric: void,
+ bigint: void,
+ string: void,
+ template: enum {
+ whole,
+ start,
+ middle,
+ end,
+ },
+ regex: void,
+
+ const digits = "_0123456789abcdef";
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ errdefer lexer.revert();
+
+ var token = lexer.start(.{ .literal = undefined });
+
+ const regex_possible = lexer.context.regex_possible;
+ lexer.context.regex_possible = false;
+
+ switch (try lexer.consume()) {
+ '0'...'9', '.' => |c| {
+ const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) {
+ 'x' => 16,
+ 'o' => 8,
+ 'b' => 2,
+ else => 10,
+ };
+
+ if (base != 10) lexer.skip() catch unreachable;
+
+ {
+ const index = std.mem.findNone(
+ u8,
+ lexer.buffer,
+ digits[0 .. base + 1],
+ ) orelse lexer.buffer.len;
+ try lexer.skipTo(index);
+
+
+ if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken;
+ }
+
+ if (base == 10 and lexer.peekChar() == '.') {
+ try lexer.skip();
+ const index = std.mem.findNone(
+ u8,
+ lexer.buffer,
+ digits[0 .. base + 1],
+ ) orelse lexer.buffer.len;
+ try lexer.skipTo(index);
+ }
+
+ if (lexer.peekChar() == 'n') {
+ lexer.skip() catch unreachable;
+ token.kind.literal = .bigint;
+ } else {
+ token.kind.literal = .numeric;
+ }
+ },
+ '\'', '"' => |quote| {
+ token.kind.literal = .string;
+ while (true) {
+ switch (try lexer.consume()) {
+ quote => break,
+ '\\' => try lexer.skip(),
+ else => {},
+ }
+ }
+ },
+ '`' => {
+ token.kind.literal = .{ .template = undefined };
+
+ while (true) {
+ switch (try lexer.consume()) {
+ '`' => {
+ token.kind.literal.template = .whole;
+ break;
+ },
+ '\\' => try lexer.skip(),
+ '$' => if (lexer.peekChar() == '{') {
+ lexer.skip() catch unreachable;
+ lexer.context.template_nesting += 1;
+ token.kind.literal.template = .start;
+ lexer.context.regex_possible = true;
+ break;
+ },
+ else => {},
+ }
+ }
+ },
+ '}' => if (lexer.context.template_nesting > 0) {
+ lexer.context.template_nesting -= 1;
+ token.kind.literal = .{ .template = undefined };
+
+ while (true) {
+ switch (try lexer.consume()) {
+ '`' => {
+ token.kind.literal.template = .end;
+ break;
+ },
+ '\\' => try lexer.skip(),
+ '$' => if (lexer.peekChar() == '{') {
+ lexer.skip() catch unreachable;
+ lexer.context.template_nesting += 1;
+ token.kind.literal.template = .middle;
+ lexer.context.regex_possible = true;
+ break;
+ },
+ else => {},
+ }
+ }
+ } else return Lexer.Error.UnexpectedToken,
+ '/' => if (regex_possible) {
+ token.kind.literal = .regex;
+ while (true) {
+ switch (try lexer.consume()) {
+ '/' => break,
+ '\\' => try lexer.skip(),
+ '\n' => return Lexer.Error.UnexpectedToken,
+ else => {},
+ }
+ }
+ } else return Lexer.Error.UnexpectedToken,
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ lexer.commit(token);
+ }
+ }
+};
+
+test "numeric" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("1234", 4);
+ try Simple.run("1.234", 5);
+ try Simple.run("1_2_3_4", 7);
+ try Simple.run("0b101011", 8);
+ try Simple.run("0xabcf01", 8);
+ try Simple.run("0o731734", 8);
+ try Simple.run("0o731734n", 9);
+}
+
+test "string" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("'hello'", 7);
+ try Simple.run("\"hello\"", 7);
+ try std.testing.expectError(
+ Lexer.Error.EndOfBuffer,
+ Simple.run("'hello", 0),
+ );
+}
+
+test "regex" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("/hello/", 7);
+ try Simple.run("/some string\\/ here/", 20);
+}