const std = @import("std"); const root = @import("root.zig"); const Lexer = root.Lexer; const Token = root.Token; pub const Literal = union(enum) { // NOTE: currently only decimal literals are tokenized numeric: void, bigint: void, string: void, template: enum { whole, start, middle, end, }, regex: void, const digits = "_0123456789abcdef"; pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { comptime { errdefer lexer.revert(); var token = lexer.start(.{ .literal = undefined }); const regex_possible = lexer.context.regex_possible; lexer.context.regex_possible = false; switch (try lexer.consume()) { '0'...'9', '.' => |c| { const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { 'x' => 16, 'o' => 8, 'b' => 2, else => 10, }; if (base != 10) lexer.skip() catch unreachable; { const index = std.mem.findNone( u8, lexer.buffer, digits[0 .. base + 1], ) orelse lexer.buffer.len; try lexer.skipTo(index); if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken; } if (base == 10 and lexer.peekChar() == '.') { try lexer.skip(); const index = std.mem.findNone( u8, lexer.buffer, digits[0 .. base + 1], ) orelse lexer.buffer.len; try lexer.skipTo(index); } if (lexer.peekChar() == 'n') { lexer.skip() catch unreachable; token.kind.literal = .bigint; } else { token.kind.literal = .numeric; } }, '\'', '"' => |quote| { token.kind.literal = .string; while (true) { switch (try lexer.consume()) { quote => break, '\\' => try lexer.skip(), else => {}, } } }, '`' => { token.kind.literal = .{ .template = undefined }; while (true) { switch (try lexer.consume()) { '`' => { token.kind.literal.template = .whole; break; }, '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .start; lexer.context.regex_possible = true; break; }, else => {}, } } }, '}' => if (lexer.context.template_nesting > 0) { lexer.context.template_nesting -= 1; token.kind.literal = .{ .template = undefined }; while (true) { switch (try lexer.consume()) { '`' => { token.kind.literal.template = .end; break; }, '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .middle; lexer.context.regex_possible = true; break; }, else => {}, } } } else return Lexer.Error.UnexpectedToken, '/' => if (regex_possible) { token.kind.literal = .regex; while (true) { switch (try lexer.consume()) { '/' => break, '\\' => try lexer.skip(), '\n' => return Lexer.Error.UnexpectedToken, else => {}, } } } else return Lexer.Error.UnexpectedToken, else => return Lexer.Error.UnexpectedToken, } lexer.commit(token); } } }; test "numeric" { const Simple = struct { pub fn run(comptime buffer: []const u8, n: usize) !void { comptime var lexer: Lexer = .init(buffer); try Literal.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(n, lexer.tokens[0].slice.len); } }; try Simple.run("1234", 4); try Simple.run("1.234", 5); try Simple.run("1_2_3_4", 7); try Simple.run("0b101011", 8); try Simple.run("0xabcf01", 8); try Simple.run("0o731734", 8); try Simple.run("0o731734n", 9); } test "string" { const Simple = struct { pub fn run(comptime buffer: []const u8, n: usize) !void { comptime var lexer: Lexer = .init(buffer); try Literal.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(n, lexer.tokens[0].slice.len); } }; try Simple.run("'hello'", 7); try Simple.run("\"hello\"", 7); try std.testing.expectError( Lexer.Error.EndOfBuffer, Simple.run("'hello", 0), ); } test "regex" { const Simple = struct { pub fn run(comptime buffer: []const u8, n: usize) !void { comptime var lexer: Lexer = .init(buffer); try Literal.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(n, lexer.tokens[0].slice.len); } }; try Simple.run("/hello/", 7); try Simple.run("/some string\\/ here/", 20); }