From 006674d7e08146a9bd980e2138ce9444d2e40e6f Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Thu, 30 Jul 2026 09:19:48 +0200 Subject: add formatting --- src/z/js/parser/lexical_grammar.zig | 214 +++++++++++++++++++++--------------- 1 file changed, 126 insertions(+), 88 deletions(-) (limited to 'src/z/js/parser/lexical_grammar.zig') diff --git a/src/z/js/parser/lexical_grammar.zig b/src/z/js/parser/lexical_grammar.zig index bd54e8f..2211fbe 100644 --- a/src/z/js/parser/lexical_grammar.zig +++ b/src/z/js/parser/lexical_grammar.zig @@ -1,9 +1,7 @@ const std = @import("std"); -pub const Loc = struct { - start: usize, - end: usize, -}; +const ast = @import("ast.zig"); +pub const Loc = ast.Loc; pub const TokenType = enum { // Keywords @@ -47,6 +45,11 @@ pub const TokenType = enum { with, yield, using, + async, + static, + get, + set, + of, // Identifiers and literals identifier, @@ -122,6 +125,7 @@ pub const TokenType = enum { @"#", // Special + from, eof, unknown, @@ -167,6 +171,12 @@ pub const TokenType = enum { .with, .yield, .using, + .async, + .static, + .get, + .set, + .of, + .from, => true, else => false, }; @@ -175,6 +185,10 @@ pub const TokenType = enum { pub fn isIdentifier(self: TokenType) bool { return self == .identifier or self.isKeyword(); } + + pub fn isKeywordButNotLet(self: TokenType) bool { + return self.isKeyword() and self != .let; + } }; pub const Token = struct { @@ -194,8 +208,13 @@ pub const Lexer = struct { } pub fn tokenize(source: []const u8) []const Token { + var buf: [4096]Token = undefined; + return tokenizeBuf(source, &buf); + } + + pub fn tokenizeBuf(source: []const u8, tokens: *[4096]Token) []const Token { + @setEvalBranchQuota(100000); var lexer = Self.init(source); - var tokens: [4096]Token = undefined; var count: usize = 0; while (true) { @@ -237,7 +256,6 @@ pub const Lexer = struct { fn isLineTerminator(c: u8) bool { return switch (c) { '\n', '\r' => true, - 0x2028, 0x2029 => true, else => false, }; } @@ -291,11 +309,13 @@ pub const Lexer = struct { self.advance(); self.advance(); while (self.ch()) |c| { - if (c == '*' and self.chAt(1)) |n| { - if (n == '/') { - self.advance(); - self.advance(); - return; + if (c == '*') { + if (self.chAt(1)) |n| { + if (n == '/') { + self.advance(); + self.advance(); + return; + } } } self.advance(); @@ -309,8 +329,8 @@ pub const Lexer = struct { if (self.ch()) |n| { if (n == '!') { self.advance(); - while (self.ch()) |ch| { - if (isLineTerminator(ch)) break; + while (self.ch()) |c2| { + if (isLineTerminator(c2)) break; self.advance(); } } @@ -367,15 +387,17 @@ pub const Lexer = struct { .slice = self.source[start .. self.pos - 1], }; } - if (c == '$' and self.chAt(1)) |n| { - if (n == '{') { - self.advance(); - self.advance(); - return .{ - .kind = .template_head, - .loc = .{ .start = start, .end = self.pos }, - .slice = self.source[start .. self.pos - 2], - }; + if (c == '$') { + if (self.chAt(1)) |n| { + if (n == '{') { + self.advance(); + self.advance(); + return .{ + .kind = .template_head, + .loc = .{ .start = start, .end = self.pos }, + .slice = self.source[start .. self.pos - 2], + }; + } } } if (c == '\\') { @@ -406,15 +428,17 @@ pub const Lexer = struct { .slice = self.source[start .. self.pos - 1], }; } - if (c == '$' and self.chAt(1)) |n| { - if (n == '{') { - self.advance(); - self.advance(); - return .{ - .kind = .template_middle, - .loc = .{ .start = start, .end = self.pos }, - .slice = self.source[start .. self.pos - 2], - }; + if (c == '$') { + if (self.chAt(1)) |n| { + if (n == '{') { + self.advance(); + self.advance(); + return .{ + .kind = .template_middle, + .loc = .{ .start = start, .end = self.pos }, + .slice = self.source[start .. self.pos - 2], + }; + } } } if (c == '\\') { @@ -764,11 +788,13 @@ pub const Lexer = struct { '.' => { self.advance(); if (self.ch()) |n| { - if (n == '.' and self.chAt(1)) |n2| { - if (n2 == '.') { - self.advance(); - self.advance(); - return simple(.@"...", start); + if (n == '.') { + if (self.chAt(1)) |n2| { + if (n2 == '.') { + self.advance(); + self.advance(); + return simple(.@"...", start); + } } } } @@ -1019,23 +1045,23 @@ pub const Lexer = struct { } fn simple(kind: TokenType, start: usize) Token { - _ = start; - return .{ .kind = kind, .loc = undefined, .slice = "" }; + return .{ .kind = kind, .loc = .{ .start = start, .end = start }, .slice = "" }; } fn simpleWithSlice(comptime kind: TokenType, start: usize, slice: []const u8) Token { - return .{ .kind = kind, .loc = undefined, .slice = slice }; + return .{ .kind = kind, .loc = .{ .start = start, .end = start }, .slice = slice }; } pub fn next(self: *Self) Token { - const start = self.pos; if (self.ch()) |c| { _ = c; // Handle hashbang at beginning - if (self.pos == 0 and self.ch()) |c2| { - if (c2 == '#') { - self.skipHashbang(); - return self.next(); + if (self.pos == 0) { + if (self.ch()) |c2| { + if (c2 == '#') { + self.skipHashbang(); + return self.next(); + } } } return self.scanPunctuator(); @@ -1044,50 +1070,62 @@ pub const Lexer = struct { } fn keywordFromString(s: []const u8) TokenType { - const keywords = std.ComptimeStringMap(TokenType, .{ - .{ "await", .await }, - .{ "break", .@"break" }, - .{ "case", .case }, - .{ "catch", .@"catch" }, - .{ "class", .class }, - .{ "const", .@"const" }, - .{ "continue", .@"continue" }, - .{ "debugger", .debugger }, - .{ "default", .default }, - .{ "delete", .delete }, - .{ "do", .do }, - .{ "else", .@"else" }, - .{ "enum", .@"enum" }, - .{ "export", .@"export" }, - .{ "extends", .extends }, - .{ "false", .false }, - .{ "finally", .finally }, - .{ "for", .@"for" }, - .{ "function", .function }, - .{ "if", .@"if" }, - .{ "import", .import }, - .{ "in", .in }, - .{ "instanceof", .instanceof }, - .{ "let", .let }, - .{ "new", .new }, - .{ "null", .null }, - .{ "return", .@"return" }, - .{ "super", .super }, - .{ "switch", .@"switch" }, - .{ "this", .this }, - .{ "throw", .throw }, - .{ "true", .true }, - .{ "try", .@"try" }, - .{ "typeof", .typeof }, - .{ "var", .@"var" }, - .{ "void", .void }, - .{ "while", .@"while" }, - .{ "with", .with }, - .{ "yield", .yield }, - .{ "using", .using }, - }); - - if (keywords.get(s)) |k| return k; + const keywords = comptime keywords: { + const arr = struct { + const data = [_]struct { key: []const u8, val: TokenType }{ + .{ .key = "await", .val = .await }, + .{ .key = "break", .val = .@"break" }, + .{ .key = "case", .val = .case }, + .{ .key = "catch", .val = .@"catch" }, + .{ .key = "class", .val = .class }, + .{ .key = "const", .val = .@"const" }, + .{ .key = "continue", .val = .@"continue" }, + .{ .key = "debugger", .val = .debugger }, + .{ .key = "default", .val = .default }, + .{ .key = "delete", .val = .delete }, + .{ .key = "do", .val = .do }, + .{ .key = "else", .val = .@"else" }, + .{ .key = "enum", .val = .@"enum" }, + .{ .key = "export", .val = .@"export" }, + .{ .key = "extends", .val = .extends }, + .{ .key = "false", .val = .false }, + .{ .key = "finally", .val = .finally }, + .{ .key = "for", .val = .@"for" }, + .{ .key = "function", .val = .function }, + .{ .key = "if", .val = .@"if" }, + .{ .key = "import", .val = .import }, + .{ .key = "in", .val = .in }, + .{ .key = "instanceof", .val = .instanceof }, + .{ .key = "let", .val = .let }, + .{ .key = "new", .val = .new }, + .{ .key = "null", .val = .null }, + .{ .key = "return", .val = .@"return" }, + .{ .key = "super", .val = .super }, + .{ .key = "switch", .val = .@"switch" }, + .{ .key = "this", .val = .this }, + .{ .key = "throw", .val = .throw }, + .{ .key = "true", .val = .true }, + .{ .key = "try", .val = .@"try" }, + .{ .key = "typeof", .val = .typeof }, + .{ .key = "var", .val = .@"var" }, + .{ .key = "void", .val = .void }, + .{ .key = "while", .val = .@"while" }, + .{ .key = "with", .val = .with }, + .{ .key = "yield", .val = .yield }, + .{ .key = "using", .val = .using }, + .{ .key = "async", .val = .async }, + .{ .key = "static", .val = .static }, + .{ .key = "get", .val = .get }, + .{ .key = "set", .val = .set }, + .{ .key = "of", .val = .of }, + .{ .key = "from", .val = .from }, + }; + }; + break :keywords arr.data; + }; + inline for (keywords) |entry| { + if (std.mem.eql(u8, s, entry.key)) return entry.val; + } return .identifier; } -- cgit v1.2.3-70-g09d2