1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
const std = @import("std");
const root = @import("root.zig");
const Lexer = root.Lexer;
const Token = root.Token;
pub const Keyword = enum {
@"break",
case,
@"catch",
class,
@"const",
@"continue",
debugger,
default,
delete,
do,
@"else",
@"export",
extends,
finally,
@"for",
function,
@"if",
import,
in,
instanceof,
new,
@"return",
super,
@"switch",
this,
throw,
@"try",
typeof,
@"var",
void,
@"while",
with,
yield,
let,
static,
await,
async,
of,
from,
get,
set,
meta,
@"enum",
null,
true,
false,
const Self = @This();
const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: {
var keywords: []const struct { []const u8, Self } = &.{};
for (@typeInfo(Self).@"enum".fields) |field| {
const name: []const u8 = field.name;
const value: Self = @field(Self, field.name);
keywords = keywords ++ .{ .{ name, value } };
}
break :kws keywords;
});
pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void {
comptime {
@setEvalBranchQuota(10000);
errdefer lexer.revert();
lexer.context.regex_possible = true;
const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len;
if (index == 0) return Lexer.Error.UnexpectedToken;
if (keywordmap.get(lexer.buffer[0..index])) |tag| {
const token = lexer.start(.{ .keyword = tag });
try lexer.skipTo(index);
lexer.commit(token);
} else {
return Lexer.Error.UnexpectedToken;
}
}
}
};
test "simple" {
inline for (comptime Keyword.keywordmap.keys()) |key| {
comptime var lexer: Lexer = .init(key);
try Keyword.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
}
}
test "ending" {
inline for (comptime Keyword.keywordmap.keys()) |key| {
comptime var lexer: Lexer = .init(key ++ ".");
try Keyword.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
}
}
test "subword" {
inline for (comptime Keyword.keywordmap.keys()) |key| {
comptime var lexer: Lexer = .init(key ++ "a");
try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer));
}
}
|
