1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
const std = @import("std");
const root = @import("root.zig");
const Lexer = root.Lexer;
const Token = root.Token;
pub const Identifier = enum {
public,
private,
pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
comptime {
defer lexer.revert();
lexer.context.regex_possible = false;
const token = tok: {
if (lexer.peekChar() == '#') {
const t = lexer.start(.{ .identifier = .private });
lexer.skip() catch unreachable;
break :tok t;
} else {
break :tok lexer.start(.{ .identifier = .public });
}
};
switch (try lexer.consume()) {
'a'...'z', 'A'...'Z', '_' => {},
else => return Lexer.Error.UnexpectedToken,
}
while (lexer.peekChar()) |ch| {
switch (ch) {
'a'...'z', 'A'...'Z', '0'...'9', '_' => {},
else => break,
}
try lexer.skip();
}
lexer.commit(token);
}
}
};
test "simple" {
{
comptime var lexer: Lexer = .init("some_literal9");
try Identifier.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(13, lexer.tokens[0].slice.len);
}
{
comptime var lexer: Lexer = .init("#some_literal9");
try Identifier.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(14, lexer.tokens[0].slice.len);
}
}
|
