const std = @import("std"); const root = @import("root.zig"); const Lexer = root.Lexer; const Token = root.Token; pub const Literal = enum { // NOTE: currently only decimal literals are tokenized numeric, bigint, string, pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { comptime { errdefer lexer.revert(); var token = lexer.start(.{ .literal = undefined }); switch (try lexer.consume()) { '0'...'9', '.' => |c| { token.kind.literal = .numeric; var had_period = c == '.'; while (lexer.peekChar()) |ch| { switch (ch) { '0'...'9' => {}, '.' => if (had_period) { return Lexer.Error.UnexpectedToken; } else { had_period = true; }, else => return Lexer.Error.UnexpectedToken, } try lexer.skip(); } }, '\'', '"' => |quote| while (try lexer.consume() != quote) { if (quote == '\\') { try lexer.consume(); } }, else => return Lexer.Error.UnexpectedToken, } lexer.commit(token); } } }; test "numeric" { const Simple = struct { pub fn run(comptime buffer: []const u8, n: usize) !void { comptime var lexer: Lexer = .init(buffer); try Literal.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(n, lexer.tokens[0].slice.len); } }; try Simple.run("1234", 4); try Simple.run("1.234", 5); try std.testing.expectError( Lexer.Error.UnexpectedToken, Simple.run("12.3.4", 0), ); } test "string" { const Simple = struct { pub fn run(comptime buffer: []const u8, n: usize) !void { comptime var lexer: Lexer = .init(buffer); try Literal.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(n, lexer.tokens[0].slice.len); } }; try Simple.run("'hello'", 7); try Simple.run("\"hello\"", 7); try std.testing.expectError( Lexer.Error.EndOfBuffer, Simple.run("'hello", 0), ); }