aboutsummaryrefslogtreecommitdiff
path: root/src/z/js/parser/lexical_grammar.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/z/js/parser/lexical_grammar.zig')
-rw-r--r--src/z/js/parser/lexical_grammar.zig214
1 files changed, 126 insertions, 88 deletions
diff --git a/src/z/js/parser/lexical_grammar.zig b/src/z/js/parser/lexical_grammar.zig
index bd54e8f..2211fbe 100644
--- a/src/z/js/parser/lexical_grammar.zig
+++ b/src/z/js/parser/lexical_grammar.zig
@@ -1,9 +1,7 @@
const std = @import("std");
-pub const Loc = struct {
- start: usize,
- end: usize,
-};
+const ast = @import("ast.zig");
+pub const Loc = ast.Loc;
pub const TokenType = enum {
// Keywords
@@ -47,6 +45,11 @@ pub const TokenType = enum {
with,
yield,
using,
+ async,
+ static,
+ get,
+ set,
+ of,
// Identifiers and literals
identifier,
@@ -122,6 +125,7 @@ pub const TokenType = enum {
@"#",
// Special
+ from,
eof,
unknown,
@@ -167,6 +171,12 @@ pub const TokenType = enum {
.with,
.yield,
.using,
+ .async,
+ .static,
+ .get,
+ .set,
+ .of,
+ .from,
=> true,
else => false,
};
@@ -175,6 +185,10 @@ pub const TokenType = enum {
pub fn isIdentifier(self: TokenType) bool {
return self == .identifier or self.isKeyword();
}
+
+ pub fn isKeywordButNotLet(self: TokenType) bool {
+ return self.isKeyword() and self != .let;
+ }
};
pub const Token = struct {
@@ -194,8 +208,13 @@ pub const Lexer = struct {
}
pub fn tokenize(source: []const u8) []const Token {
+ var buf: [4096]Token = undefined;
+ return tokenizeBuf(source, &buf);
+ }
+
+ pub fn tokenizeBuf(source: []const u8, tokens: *[4096]Token) []const Token {
+ @setEvalBranchQuota(100000);
var lexer = Self.init(source);
- var tokens: [4096]Token = undefined;
var count: usize = 0;
while (true) {
@@ -237,7 +256,6 @@ pub const Lexer = struct {
fn isLineTerminator(c: u8) bool {
return switch (c) {
'\n', '\r' => true,
- 0x2028, 0x2029 => true,
else => false,
};
}
@@ -291,11 +309,13 @@ pub const Lexer = struct {
self.advance();
self.advance();
while (self.ch()) |c| {
- if (c == '*' and self.chAt(1)) |n| {
- if (n == '/') {
- self.advance();
- self.advance();
- return;
+ if (c == '*') {
+ if (self.chAt(1)) |n| {
+ if (n == '/') {
+ self.advance();
+ self.advance();
+ return;
+ }
}
}
self.advance();
@@ -309,8 +329,8 @@ pub const Lexer = struct {
if (self.ch()) |n| {
if (n == '!') {
self.advance();
- while (self.ch()) |ch| {
- if (isLineTerminator(ch)) break;
+ while (self.ch()) |c2| {
+ if (isLineTerminator(c2)) break;
self.advance();
}
}
@@ -367,15 +387,17 @@ pub const Lexer = struct {
.slice = self.source[start .. self.pos - 1],
};
}
- if (c == '$' and self.chAt(1)) |n| {
- if (n == '{') {
- self.advance();
- self.advance();
- return .{
- .kind = .template_head,
- .loc = .{ .start = start, .end = self.pos },
- .slice = self.source[start .. self.pos - 2],
- };
+ if (c == '$') {
+ if (self.chAt(1)) |n| {
+ if (n == '{') {
+ self.advance();
+ self.advance();
+ return .{
+ .kind = .template_head,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 2],
+ };
+ }
}
}
if (c == '\\') {
@@ -406,15 +428,17 @@ pub const Lexer = struct {
.slice = self.source[start .. self.pos - 1],
};
}
- if (c == '$' and self.chAt(1)) |n| {
- if (n == '{') {
- self.advance();
- self.advance();
- return .{
- .kind = .template_middle,
- .loc = .{ .start = start, .end = self.pos },
- .slice = self.source[start .. self.pos - 2],
- };
+ if (c == '$') {
+ if (self.chAt(1)) |n| {
+ if (n == '{') {
+ self.advance();
+ self.advance();
+ return .{
+ .kind = .template_middle,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 2],
+ };
+ }
}
}
if (c == '\\') {
@@ -764,11 +788,13 @@ pub const Lexer = struct {
'.' => {
self.advance();
if (self.ch()) |n| {
- if (n == '.' and self.chAt(1)) |n2| {
- if (n2 == '.') {
- self.advance();
- self.advance();
- return simple(.@"...", start);
+ if (n == '.') {
+ if (self.chAt(1)) |n2| {
+ if (n2 == '.') {
+ self.advance();
+ self.advance();
+ return simple(.@"...", start);
+ }
}
}
}
@@ -1019,23 +1045,23 @@ pub const Lexer = struct {
}
fn simple(kind: TokenType, start: usize) Token {
- _ = start;
- return .{ .kind = kind, .loc = undefined, .slice = "" };
+ return .{ .kind = kind, .loc = .{ .start = start, .end = start }, .slice = "" };
}
fn simpleWithSlice(comptime kind: TokenType, start: usize, slice: []const u8) Token {
- return .{ .kind = kind, .loc = undefined, .slice = slice };
+ return .{ .kind = kind, .loc = .{ .start = start, .end = start }, .slice = slice };
}
pub fn next(self: *Self) Token {
- const start = self.pos;
if (self.ch()) |c| {
_ = c;
// Handle hashbang at beginning
- if (self.pos == 0 and self.ch()) |c2| {
- if (c2 == '#') {
- self.skipHashbang();
- return self.next();
+ if (self.pos == 0) {
+ if (self.ch()) |c2| {
+ if (c2 == '#') {
+ self.skipHashbang();
+ return self.next();
+ }
}
}
return self.scanPunctuator();
@@ -1044,50 +1070,62 @@ pub const Lexer = struct {
}
fn keywordFromString(s: []const u8) TokenType {
- const keywords = std.ComptimeStringMap(TokenType, .{
- .{ "await", .await },
- .{ "break", .@"break" },
- .{ "case", .case },
- .{ "catch", .@"catch" },
- .{ "class", .class },
- .{ "const", .@"const" },
- .{ "continue", .@"continue" },
- .{ "debugger", .debugger },
- .{ "default", .default },
- .{ "delete", .delete },
- .{ "do", .do },
- .{ "else", .@"else" },
- .{ "enum", .@"enum" },
- .{ "export", .@"export" },
- .{ "extends", .extends },
- .{ "false", .false },
- .{ "finally", .finally },
- .{ "for", .@"for" },
- .{ "function", .function },
- .{ "if", .@"if" },
- .{ "import", .import },
- .{ "in", .in },
- .{ "instanceof", .instanceof },
- .{ "let", .let },
- .{ "new", .new },
- .{ "null", .null },
- .{ "return", .@"return" },
- .{ "super", .super },
- .{ "switch", .@"switch" },
- .{ "this", .this },
- .{ "throw", .throw },
- .{ "true", .true },
- .{ "try", .@"try" },
- .{ "typeof", .typeof },
- .{ "var", .@"var" },
- .{ "void", .void },
- .{ "while", .@"while" },
- .{ "with", .with },
- .{ "yield", .yield },
- .{ "using", .using },
- });
-
- if (keywords.get(s)) |k| return k;
+ const keywords = comptime keywords: {
+ const arr = struct {
+ const data = [_]struct { key: []const u8, val: TokenType }{
+ .{ .key = "await", .val = .await },
+ .{ .key = "break", .val = .@"break" },
+ .{ .key = "case", .val = .case },
+ .{ .key = "catch", .val = .@"catch" },
+ .{ .key = "class", .val = .class },
+ .{ .key = "const", .val = .@"const" },
+ .{ .key = "continue", .val = .@"continue" },
+ .{ .key = "debugger", .val = .debugger },
+ .{ .key = "default", .val = .default },
+ .{ .key = "delete", .val = .delete },
+ .{ .key = "do", .val = .do },
+ .{ .key = "else", .val = .@"else" },
+ .{ .key = "enum", .val = .@"enum" },
+ .{ .key = "export", .val = .@"export" },
+ .{ .key = "extends", .val = .extends },
+ .{ .key = "false", .val = .false },
+ .{ .key = "finally", .val = .finally },
+ .{ .key = "for", .val = .@"for" },
+ .{ .key = "function", .val = .function },
+ .{ .key = "if", .val = .@"if" },
+ .{ .key = "import", .val = .import },
+ .{ .key = "in", .val = .in },
+ .{ .key = "instanceof", .val = .instanceof },
+ .{ .key = "let", .val = .let },
+ .{ .key = "new", .val = .new },
+ .{ .key = "null", .val = .null },
+ .{ .key = "return", .val = .@"return" },
+ .{ .key = "super", .val = .super },
+ .{ .key = "switch", .val = .@"switch" },
+ .{ .key = "this", .val = .this },
+ .{ .key = "throw", .val = .throw },
+ .{ .key = "true", .val = .true },
+ .{ .key = "try", .val = .@"try" },
+ .{ .key = "typeof", .val = .typeof },
+ .{ .key = "var", .val = .@"var" },
+ .{ .key = "void", .val = .void },
+ .{ .key = "while", .val = .@"while" },
+ .{ .key = "with", .val = .with },
+ .{ .key = "yield", .val = .yield },
+ .{ .key = "using", .val = .using },
+ .{ .key = "async", .val = .async },
+ .{ .key = "static", .val = .static },
+ .{ .key = "get", .val = .get },
+ .{ .key = "set", .val = .set },
+ .{ .key = "of", .val = .of },
+ .{ .key = "from", .val = .from },
+ };
+ };
+ break :keywords arr.data;
+ };
+ inline for (keywords) |entry| {
+ if (std.mem.eql(u8, s, entry.key)) return entry.val;
+ }
return .identifier;
}
generated by cgit v1.2.3-70-g09d2 (git 2.53.0) at 2026-09-16 12:54:43 +0000