1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
const std = @import("std");
const root = @import("root.zig");
const Lexer = root.Lexer;
const Token = root.Token;
pub const Punctuator = enum {
@"+",
@"-",
@"*",
@"/",
@"%",
@"**",
@"++",
@"--",
@"=",
@"+=",
@"-=",
@"*=",
@"/=",
@"%=",
@"**=",
@"&=",
@"|=",
@"^=",
@"<<=",
@">>=",
@">>>=",
@"&&=",
@"||=",
@"??=",
@"==",
@"===",
@"!=",
@"!==",
@"<",
@"<=",
@">",
@">=",
@"&&",
@"||",
@"??",
@"&",
@"|",
@"^",
@"~",
@"<<",
@">>",
@">>>",
@"!",
@"?",
@":",
@";",
@",",
@".",
@"...",
@"=>",
@"(",
@")",
@"[",
@"]",
@"{",
@"}",
@"?.",
@"#",
@"@",
const Self = @This();
const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: {
var punctuators: []const struct { []const u8, Self } = &.{};
for (@typeInfo(Self).@"enum".fields) |field| {
const name: []const u8 = field.name;
const value: Self = @field(Self, field.name);
punctuators = punctuators ++ .{.{ name, value }};
}
break :kws punctuators;
});
pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
comptime {
@setEvalBranchQuota(10_000);
defer lexer.revert();
lexer.context.regex_possible = true;
if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| {
if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) {
return Lexer.Error.UnexpectedToken;
}
const token = lexer.start(.{ .punctuator = pair.value });
try lexer.skipTo(pair.key.len);
lexer.commit(token);
} else {
return Lexer.Error.UnexpectedToken;
}
}
}
};
test "simple" {
inline for (comptime Punctuator.punctuatormap.keys()) |key| {
comptime var lexer: Lexer = .init(key);
try Punctuator.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
}
}
test "ending" {
inline for (comptime Punctuator.punctuatormap.keys()) |key| {
comptime var lexer: Lexer = .init(key ++ "a");
try Punctuator.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
}
}
|
