1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
|
const std = @import("std");
const root = @import("root.zig");
const Lexer = root.Lexer;
const Token = root.Token;
pub const Literal = enum {
// NOTE: currently only decimal literals are tokenized
numeric,
bigint,
string,
pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
comptime {
errdefer lexer.revert();
var token = lexer.start(.{ .literal = undefined });
switch (try lexer.consume()) {
'0'...'9', '.' => |c| {
token.kind.literal = .numeric;
var had_period = c == '.';
while (lexer.peekChar()) |ch| {
switch (ch) {
'0'...'9' => {},
'.' => if (had_period) {
return Lexer.Error.UnexpectedToken;
} else {
had_period = true;
},
else => return Lexer.Error.UnexpectedToken,
}
try lexer.skip();
}
},
'\'', '"' => |quote| while (try lexer.consume() != quote) {
if (quote == '\\') {
try lexer.consume();
}
},
else => return Lexer.Error.UnexpectedToken,
}
lexer.commit(token);
}
}
};
test "numeric" {
const Simple = struct {
pub fn run(comptime buffer: []const u8, n: usize) !void {
comptime var lexer: Lexer = .init(buffer);
try Literal.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
}
};
try Simple.run("1234", 4);
try Simple.run("1.234", 5);
try std.testing.expectError(
Lexer.Error.UnexpectedToken,
Simple.run("12.3.4", 0),
);
}
test "string" {
const Simple = struct {
pub fn run(comptime buffer: []const u8, n: usize) !void {
comptime var lexer: Lexer = .init(buffer);
try Literal.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
}
};
try Simple.run("'hello'", 7);
try Simple.run("\"hello\"", 7);
try std.testing.expectError(
Lexer.Error.EndOfBuffer,
Simple.run("'hello", 0),
);
}
|
