blob: 56f3990521aba7b1a85bb68a39cfecc894029712 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
//! This is a Javascript parser and tokenizer.
//! It is and probably will never be complete,
//! since js is a very strange language and I want
//! to move forward and code on more interesting things
//! I started something called mujs (micro-js). It's a
//! subset of the javascript language. It does not support
//! the regex literal and currently also not template strings.
//! It is chosen like that to make parsing easier and also less
//! context-dependent.
const std = @import("std");
const parser = @import("../../root.zig");
pub const Lexer = parser.Lexer(Grammar);
pub const Token = parser.Token(Grammar);
pub const Whitespace = @import("whitespace.zig").Whitespace;
pub const Keyword = @import("keyword.zig").Keyword;
pub const Literal = @import("literal.zig").Literal;
pub const Punctuator = @import("punctuator.zig").Punctuator;
pub const Identifier = @import("identifier.zig").Identifier;
pub const Grammar = union(enum) {
whitespace: Whitespace,
keyword: Keyword,
literal: Literal,
punctuator: Punctuator,
identifier: Identifier,
end_of_file: void,
pub const Context = struct {
template_nesting: usize = 0,
};
pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
@setEvalBranchQuota(lexer.buffer.len * 1000);
comptime {
while (lexer.peekChar()) |_| {
Whitespace.tokenize(lexer) catch {
Keyword.tokenize(lexer) catch {
Literal.tokenize(lexer) catch {
Punctuator.tokenize(lexer) catch {
try Identifier.tokenize(lexer);
};
};
};
};
}
lexer.commit(lexer.start(.end_of_file));
}
}
};
test {
_ = std.testing.refAllDecls(@This());
}
test "z.js" {
comptime var lexer: Lexer = .init(@embedFile("../../../z.js"));
Grammar.tokenize(&lexer) catch |err| {
std.debug.print("error: {} at {s}", .{ err, lexer.buffer });
};
}
|
