1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
const std = @import("std");
const Lexer = @import("root.zig").Lexer;
const Token = @import("root.zig").Token;
pub const Whitespace = enum {
space,
line_comment,
block_comment,
pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
comptime {
errdefer lexer.revert();
var token = lexer.start(.{ .whitespace = undefined });
switch (try lexer.consume()) {
' ', '\t', '\r', '\n' => {
token.kind.whitespace = .space;
while (lexer.peekChar()) |ch| {
switch (ch) {
' ', '\t', '\r', '\n' => try lexer.skip(),
else => break,
}
}
},
'/' => {
switch (lexer.peekChar() orelse 0) {
'*' => {
token.kind.whitespace = .block_comment;
try lexer.skipUntil("*/");
},
'/' => {
token.kind.whitespace = .line_comment;
lexer.skipUntil("\n") catch lexer.skipToEnd();
},
else => return Lexer.Error.UnexpectedToken,
}
},
else => return Lexer.Error.UnexpectedToken,
}
lexer.commit(token);
}
}
};
test "space" {
{
comptime var lexer: Lexer = .init(" ");
try Whitespace.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(1, lexer.tokens[0].slice.len);
}
{
comptime var lexer: Lexer = .init(" \t\t \n\r ");
try Whitespace.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(10, lexer.tokens[0].slice.len);
}
}
test "line-comment" {
{
comptime var lexer: Lexer = .init("// some comment");
try Whitespace.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice);
}
{
comptime var lexer: Lexer = .init("// some comment\n");
try Whitespace.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice);
}
}
test "block-comment" {
{
comptime var lexer: Lexer = .init("/* some comment */");
try Whitespace.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice);
}
}
test "mixed" {
comptime var lexer: Lexer = .init(
\\ // some comment
\\/* other comment here */ // comment or so
\\ /* multi
\\ * line
\\ * comment
\\ */
);
inline while (Whitespace.tokenize(&lexer)) {
} else |_| {}
try std.testing.expectEqual(7, lexer.tokens.len);
}
|
