1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
const std = @import("std");
const root = @import("root.zig");
const Lexer = root.Lexer;
const Token = root.Token;
pub const Literal = enum {
// NOTE: currently only decimal literals are tokenized
numeric,
bigint,
string,
const digits = "_0123456789abcdef";
pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
comptime {
errdefer lexer.revert();
var token = lexer.start(.{ .literal = undefined });
switch (try lexer.consume()) {
'0'...'9', '.' => |c| {
const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) {
'x' => 16,
'o' => 8,
'b' => 2,
else => 10,
};
if (base != 10) lexer.skip() catch unreachable;
{
const index = std.mem.findNone(
u8,
lexer.buffer,
digits[0 .. base + 1],
) orelse lexer.buffer.len;
try lexer.skipTo(index);
if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken;
}
if (base == 10 and lexer.peekChar() == '.') {
try lexer.skip();
const index = std.mem.findNone(
u8,
lexer.buffer,
digits[0 .. base + 1],
) orelse lexer.buffer.len;
try lexer.skipTo(index);
}
if (lexer.peekChar() == 'n') {
lexer.skip() catch unreachable;
token.kind.literal = .bigint;
} else {
token.kind.literal = .numeric;
}
},
'\'', '"' => |quote| {
token.kind.literal = .string;
while (try lexer.consume() != quote) {
if (quote == '\\') {
try lexer.consume();
}
}
},
else => return Lexer.Error.UnexpectedToken,
}
lexer.commit(token);
}
}
};
test "numeric" {
const Simple = struct {
pub fn run(comptime buffer: []const u8, n: usize) !void {
comptime var lexer: Lexer = .init(buffer);
try Literal.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
}
};
try Simple.run("1234", 4);
try Simple.run("1.234", 5);
try Simple.run("1_2_3_4", 7);
try Simple.run("0b101011", 8);
try Simple.run("0xabcf01", 8);
try Simple.run("0o731734", 8);
try Simple.run("0o731734n", 9);
}
test "string" {
const Simple = struct {
pub fn run(comptime buffer: []const u8, n: usize) !void {
comptime var lexer: Lexer = .init(buffer);
try Literal.tokenize(&lexer);
try std.testing.expectEqual(1, lexer.tokens.len);
try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
}
};
try Simple.run("'hello'", 7);
try Simple.run("\"hello\"", 7);
try std.testing.expectError(
Lexer.Error.EndOfBuffer,
Simple.run("'hello", 0),
);
}
|
