aboutsummaryrefslogtreecommitdiff
path: root/src/z/js/parser/lexical_grammar.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/z/js/parser/lexical_grammar.zig')
-rw-r--r--src/z/js/parser/lexical_grammar.zig1099
1 files changed, 1099 insertions, 0 deletions
diff --git a/src/z/js/parser/lexical_grammar.zig b/src/z/js/parser/lexical_grammar.zig
new file mode 100644
index 0000000..bd54e8f
--- /dev/null
+++ b/src/z/js/parser/lexical_grammar.zig
@@ -0,0 +1,1099 @@
+const std = @import("std");
+
+pub const Loc = struct {
+ start: usize,
+ end: usize,
+};
+
+pub const TokenType = enum {
+ // Keywords
+ await,
+ @"break",
+ case,
+ @"catch",
+ class,
+ @"const",
+ @"continue",
+ debugger,
+ default,
+ delete,
+ do,
+ @"else",
+ @"enum",
+ @"export",
+ extends,
+ false,
+ finally,
+ @"for",
+ function,
+ @"if",
+ import,
+ in,
+ instanceof,
+ let,
+ new,
+ null,
+ @"return",
+ super,
+ @"switch",
+ this,
+ throw,
+ true,
+ @"try",
+ typeof,
+ @"var",
+ void,
+ @"while",
+ with,
+ yield,
+ using,
+
+ // Identifiers and literals
+ identifier,
+ private_identifier,
+ number,
+ string,
+ bigint,
+ regex,
+ template_head,
+ template_middle,
+ template_tail,
+ no_sub_template,
+
+ // Punctuators
+ lbrace,
+ rbrace,
+ lparen,
+ rparen,
+ lbracket,
+ rbracket,
+ dot,
+ semicolon,
+ comma,
+ @"...",
+ @"<",
+ @">",
+ @"<=",
+ @">=",
+ @"==",
+ @"!=",
+ @"===",
+ @"!==",
+ @"+",
+ @"-",
+ @"*",
+ @"%",
+ @"**",
+ @"++",
+ @"--",
+ @"<<",
+ @">>",
+ @">>>",
+ @"&",
+ @"|",
+ @"^",
+ @"!",
+ @"~",
+ @"&&",
+ @"||",
+ @"??",
+ @"?",
+ @":",
+ @"=",
+ @"+=",
+ @"-=",
+ @"*=",
+ @"%=",
+ @"**=",
+ @"<<=",
+ @">>=",
+ @">>>=",
+ @"&=",
+ @"|=",
+ @"^=",
+ @"&&=",
+ @"||=",
+ @"??=",
+ @"=>",
+ @"/",
+ @"/=",
+ @"?.",
+ @".",
+ @"#",
+
+ // Special
+ eof,
+ unknown,
+
+ pub fn isKeyword(self: TokenType) bool {
+ return switch (self) {
+ .await,
+ .@"break",
+ .case,
+ .@"catch",
+ .class,
+ .@"const",
+ .@"continue",
+ .debugger,
+ .default,
+ .delete,
+ .do,
+ .@"else",
+ .@"enum",
+ .@"export",
+ .extends,
+ .false,
+ .finally,
+ .@"for",
+ .function,
+ .@"if",
+ .import,
+ .in,
+ .instanceof,
+ .let,
+ .new,
+ .null,
+ .@"return",
+ .super,
+ .@"switch",
+ .this,
+ .throw,
+ .true,
+ .@"try",
+ .typeof,
+ .@"var",
+ .void,
+ .@"while",
+ .with,
+ .yield,
+ .using,
+ => true,
+ else => false,
+ };
+ }
+
+ pub fn isIdentifier(self: TokenType) bool {
+ return self == .identifier or self.isKeyword();
+ }
+};
+
+pub const Token = struct {
+ kind: TokenType,
+ loc: Loc,
+ slice: []const u8,
+};
+
+pub const Lexer = struct {
+ source: []const u8,
+ pos: usize,
+
+ const Self = @This();
+
+ pub fn init(source: []const u8) Self {
+ return .{ .source = source, .pos = 0 };
+ }
+
+ pub fn tokenize(source: []const u8) []const Token {
+ var lexer = Self.init(source);
+ var tokens: [4096]Token = undefined;
+ var count: usize = 0;
+
+ while (true) {
+ const tok = lexer.next();
+ tokens[count] = tok;
+ count += 1;
+ if (tok.kind == .eof) break;
+ if (count >= tokens.len - 1) {
+ tokens[count] = .{ .kind = .eof, .loc = .{ .start = lexer.pos, .end = lexer.pos }, .slice = "" };
+ break;
+ }
+ }
+
+ return tokens[0..count];
+ }
+
+ fn ch(self: Self) ?u8 {
+ if (self.pos >= self.source.len) return null;
+ return self.source[self.pos];
+ }
+
+ fn chAt(self: Self, offset: usize) ?u8 {
+ const idx = self.pos + offset;
+ if (idx >= self.source.len) return null;
+ return self.source[idx];
+ }
+
+ fn advance(self: *Self) void {
+ if (self.pos < self.source.len) self.pos += 1;
+ }
+
+ fn isWhiteSpace(c: u8) bool {
+ return switch (c) {
+ ' ', '\t', 0x0B, 0x0C => true,
+ else => false,
+ };
+ }
+
+ fn isLineTerminator(c: u8) bool {
+ return switch (c) {
+ '\n', '\r' => true,
+ 0x2028, 0x2029 => true,
+ else => false,
+ };
+ }
+
+ fn isDigit(c: u8) bool {
+ return c >= '0' and c <= '9';
+ }
+
+ fn isHexDigit(c: u8) bool {
+ return switch (c) {
+ '0'...'9', 'a'...'f', 'A'...'F' => true,
+ else => false,
+ };
+ }
+
+ fn isOctalDigit(c: u8) bool {
+ return c >= '0' and c <= '7';
+ }
+
+ fn isIdStart(c: u8) bool {
+ return switch (c) {
+ 'a'...'z', 'A'...'Z', '_', '$' => true,
+ else => c > 0x7F,
+ };
+ }
+
+ fn isIdContinue(c: u8) bool {
+ return isIdStart(c) or isDigit(c) or c == 0x200C or c == 0x200D;
+ }
+
+ fn skipWhiteSpace(self: *Self) void {
+ while (self.ch()) |c| {
+ if (isWhiteSpace(c) or isLineTerminator(c)) {
+ self.advance();
+ } else {
+ break;
+ }
+ }
+ }
+
+ fn skipSingleLineComment(self: *Self) void {
+ self.advance();
+ self.advance();
+ while (self.ch()) |c| {
+ if (isLineTerminator(c)) break;
+ self.advance();
+ }
+ }
+
+ fn skipMultiLineComment(self: *Self) void {
+ self.advance();
+ self.advance();
+ while (self.ch()) |c| {
+ if (c == '*' and self.chAt(1)) |n| {
+ if (n == '/') {
+ self.advance();
+ self.advance();
+ return;
+ }
+ }
+ self.advance();
+ }
+ }
+
+ fn skipHashbang(self: *Self) void {
+ if (self.ch()) |c| {
+ if (c == '#') {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '!') {
+ self.advance();
+ while (self.ch()) |ch| {
+ if (isLineTerminator(ch)) break;
+ self.advance();
+ }
+ }
+ }
+ }
+ }
+ }
+
+ fn scanString(self: *Self, quote: u8) Token {
+ const start = self.pos;
+ self.advance();
+
+ while (self.ch()) |c| {
+ if (c == quote) {
+ self.advance();
+ return .{
+ .kind = .string,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start + 1 .. self.pos - 1],
+ };
+ }
+ if (c == '\\') {
+ self.advance();
+ if (self.ch()) |_| self.advance();
+ } else if (isLineTerminator(c) and c != 0x2028 and c != 0x2029) {
+ break;
+ } else {
+ self.advance();
+ }
+ }
+
+ return .{
+ .kind = .string,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start + 1 .. self.pos],
+ };
+ }
+
+ fn scanTemplate(self: *Self, start_kind: TokenType) Token {
+ const start = self.pos;
+ if (start_kind == .template_middle or start_kind == .template_head) {
+ self.advance();
+ }
+ return self.scanTemplateRest(start);
+ }
+
+ fn scanTemplateRest(self: *Self, start: usize) Token {
+ while (self.ch()) |c| {
+ if (c == '`') {
+ self.advance();
+ return .{
+ .kind = .no_sub_template,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ if (c == '$' and self.chAt(1)) |n| {
+ if (n == '{') {
+ self.advance();
+ self.advance();
+ return .{
+ .kind = .template_head,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 2],
+ };
+ }
+ }
+ if (c == '\\') {
+ self.advance();
+ if (self.ch()) |_| self.advance();
+ } else {
+ self.advance();
+ }
+ }
+
+ return .{
+ .kind = .no_sub_template,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanTemplateMiddleOrTail(self: *Self) Token {
+ const start = self.pos;
+ self.advance();
+
+ while (self.ch()) |c| {
+ if (c == '`') {
+ self.advance();
+ return .{
+ .kind = .template_tail,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ if (c == '$' and self.chAt(1)) |n| {
+ if (n == '{') {
+ self.advance();
+ self.advance();
+ return .{
+ .kind = .template_middle,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 2],
+ };
+ }
+ }
+ if (c == '\\') {
+ self.advance();
+ if (self.ch()) |_| self.advance();
+ } else {
+ self.advance();
+ }
+ }
+
+ return .{
+ .kind = .template_tail,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanNumber(self: *Self) Token {
+ const start = self.pos;
+
+ if (self.ch()) |c| {
+ if (c == '0') {
+ self.advance();
+ if (self.ch()) |n| {
+ switch (n) {
+ 'x', 'X' => return self.scanHex(start),
+ 'b', 'B' => return self.scanBinary(start),
+ 'o', 'O' => return self.scanOctal(start),
+ 'n' => {
+ self.advance();
+ return .{
+ .kind = .bigint,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ },
+ 'e', 'E' => return self.scanDecimalAfterInt(start),
+ '.', '0'...'9' => return self.scanDecimalAfterInt(start),
+ else => {},
+ }
+ }
+ }
+ }
+
+ return self.scanDecimal(start);
+ }
+
+ fn scanDecimal(self: *Self, start: usize) Token {
+ while (self.ch()) |c| {
+ if (!isDigit(c) and c != '_') break;
+ self.advance();
+ }
+
+ if (self.ch()) |c| {
+ if (c == '.') {
+ self.advance();
+ while (self.ch()) |c2| {
+ if (!isDigit(c2) and c2 != '_') break;
+ self.advance();
+ }
+ }
+ if (self.ch()) |c2| {
+ if (c2 == 'e' or c2 == 'E') {
+ return self.scanExponent(start);
+ }
+ }
+ if (self.ch()) |c2| {
+ if (c2 == 'n') {
+ self.advance();
+ return .{
+ .kind = .bigint,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ }
+ }
+
+ return .{
+ .kind = .number,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanDecimalAfterInt(self: *Self, start: usize) Token {
+ while (self.ch()) |c| {
+ if (!isDigit(c) and c != '_') break;
+ self.advance();
+ }
+ if (self.ch()) |c| {
+ if (c == '.') {
+ self.advance();
+ while (self.ch()) |c2| {
+ if (!isDigit(c2) and c2 != '_') break;
+ self.advance();
+ }
+ }
+ if (self.ch()) |c2| {
+ if (c2 == 'e' or c2 == 'E') {
+ return self.scanExponent(start);
+ }
+ }
+ if (self.ch()) |c2| {
+ if (c2 == 'n') {
+ self.advance();
+ return .{
+ .kind = .bigint,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ }
+ }
+ return .{
+ .kind = .number,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanExponent(self: *Self, start: usize) Token {
+ self.advance();
+ if (self.ch()) |c| {
+ if (c == '+' or c == '-') self.advance();
+ }
+ while (self.ch()) |c| {
+ if (!isDigit(c) and c != '_') break;
+ self.advance();
+ }
+ return .{
+ .kind = .number,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanHex(self: *Self, start: usize) Token {
+ self.advance();
+ while (self.ch()) |c| {
+ if (!isHexDigit(c) and c != '_') break;
+ self.advance();
+ }
+ if (self.ch()) |c| {
+ if (c == 'n') {
+ self.advance();
+ return .{
+ .kind = .bigint,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ }
+ return .{
+ .kind = .number,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanBinary(self: *Self, start: usize) Token {
+ self.advance();
+ while (self.ch()) |c| {
+ if (c != '0' and c != '1' and c != '_') break;
+ self.advance();
+ }
+ if (self.ch()) |c| {
+ if (c == 'n') {
+ self.advance();
+ return .{
+ .kind = .bigint,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ }
+ return .{
+ .kind = .number,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanOctal(self: *Self, start: usize) Token {
+ self.advance();
+ while (self.ch()) |c| {
+ if (!isOctalDigit(c) and c != '_') break;
+ self.advance();
+ }
+ if (self.ch()) |c| {
+ if (c == 'n') {
+ self.advance();
+ return .{
+ .kind = .bigint,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start .. self.pos - 1],
+ };
+ }
+ }
+ return .{
+ .kind = .number,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanIdentifierOrKeyword(self: *Self) Token {
+ const start = self.pos;
+ while (self.ch()) |c| {
+ if (!isIdContinue(c)) break;
+ self.advance();
+ }
+
+ const slice = self.source[start..self.pos];
+ const kind = keywordFromString(slice);
+
+ return .{
+ .kind = kind,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = slice,
+ };
+ }
+
+ fn scanPrivateIdentifier(self: *Self) Token {
+ const start = self.pos;
+ self.advance();
+ while (self.ch()) |c| {
+ if (!isIdContinue(c)) break;
+ self.advance();
+ }
+ return .{
+ .kind = .private_identifier,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanRegex(self: *Self) Token {
+ const start = self.pos;
+ self.advance();
+
+ var in_class = false;
+ while (self.ch()) |c| {
+ if (c == '\\') {
+ self.advance();
+ if (self.ch()) |_| self.advance();
+ } else if (c == '[') {
+ in_class = true;
+ self.advance();
+ } else if (c == ']') {
+ in_class = false;
+ self.advance();
+ } else if (c == '/' and !in_class) {
+ self.advance();
+ break;
+ } else if (isLineTerminator(c)) {
+ break;
+ } else {
+ self.advance();
+ }
+ }
+
+ while (self.ch()) |c| {
+ if (!isIdContinue(c)) break;
+ self.advance();
+ }
+
+ return .{
+ .kind = .regex,
+ .loc = .{ .start = start, .end = self.pos },
+ .slice = self.source[start..self.pos],
+ };
+ }
+
+ fn scanPunctuator(self: *Self) Token {
+ const start = self.pos;
+ const c = self.ch().?;
+
+ switch (c) {
+ '{' => {
+ self.advance();
+ return simple(.lbrace, start);
+ },
+ '}' => {
+ self.advance();
+ return simple(.rbrace, start);
+ },
+ '(' => {
+ self.advance();
+ return simple(.lparen, start);
+ },
+ ')' => {
+ self.advance();
+ return simple(.rparen, start);
+ },
+ '[' => {
+ self.advance();
+ return simple(.lbracket, start);
+ },
+ ']' => {
+ self.advance();
+ return simple(.rbracket, start);
+ },
+ ';' => {
+ self.advance();
+ return simple(.semicolon, start);
+ },
+ ',' => {
+ self.advance();
+ return simple(.comma, start);
+ },
+ ':' => {
+ self.advance();
+ return simple(.@":", start);
+ },
+ '?' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '?') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"??=", start);
+ }
+ }
+ return simple(.@"??", start);
+ }
+ if (n == '.') {
+ if (self.chAt(1)) |n2| {
+ if (!isDigit(n2)) {
+ self.advance();
+ return simple(.@"?.", start);
+ }
+ } else {
+ self.advance();
+ return simple(.@"?.", start);
+ }
+ }
+ }
+ return simple(.@"?", start);
+ },
+ '~' => {
+ self.advance();
+ return simple(.@"~", start);
+ },
+ '.' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '.' and self.chAt(1)) |n2| {
+ if (n2 == '.') {
+ self.advance();
+ self.advance();
+ return simple(.@"...", start);
+ }
+ }
+ }
+ return simple(.@".", start);
+ },
+ '#' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (isIdStart(n)) {
+ return self.scanPrivateIdentifier();
+ }
+ }
+ return simple(.@"#", start);
+ },
+ '+' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '+') {
+ self.advance();
+ return simple(.@"++", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"+=", start);
+ }
+ }
+ return simple(.@"+", start);
+ },
+ '-' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '-') {
+ self.advance();
+ return simple(.@"--", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"-=", start);
+ }
+ }
+ return simple(.@"-", start);
+ },
+ '*' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '*') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"**=", start);
+ }
+ }
+ return simple(.@"**", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"*=", start);
+ }
+ }
+ return simple(.@"*", start);
+ },
+ '%' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '=') {
+ self.advance();
+ return simple(.@"%=", start);
+ }
+ }
+ return simple(.@"%", start);
+ },
+ '=' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '=') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"===", start);
+ }
+ }
+ return simple(.@"==", start);
+ }
+ if (n == '>') {
+ self.advance();
+ return simple(.@"=>", start);
+ }
+ }
+ return simple(.@"=", start);
+ },
+ '!' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '=') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"!==", start);
+ }
+ }
+ return simple(.@"!=", start);
+ }
+ }
+ return simple(.@"!", start);
+ },
+ '<' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '<') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"<<=", start);
+ }
+ }
+ return simple(.@"<<", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"<=", start);
+ }
+ }
+ return simple(.@"<", start);
+ },
+ '>' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '>') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '>') {
+ self.advance();
+ if (self.ch()) |n3| {
+ if (n3 == '=') {
+ self.advance();
+ return simple(.@">>>=", start);
+ }
+ }
+ return simple(.@">>>", start);
+ }
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@">>=", start);
+ }
+ }
+ return simple(.@">>", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@">=", start);
+ }
+ }
+ return simple(.@">", start);
+ },
+ '&' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '&') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"&&=", start);
+ }
+ }
+ return simple(.@"&&", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"&=", start);
+ }
+ }
+ return simple(.@"&", start);
+ },
+ '|' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '|') {
+ self.advance();
+ if (self.ch()) |n2| {
+ if (n2 == '=') {
+ self.advance();
+ return simple(.@"||=", start);
+ }
+ }
+ return simple(.@"||", start);
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"|=", start);
+ }
+ }
+ return simple(.@"|", start);
+ },
+ '^' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '=') {
+ self.advance();
+ return simple(.@"^=", start);
+ }
+ }
+ return simple(.@"^", start);
+ },
+ '`' => {
+ self.advance();
+ return self.scanTemplateRest(start);
+ },
+ '\'' => return self.scanString('\''),
+ '"' => return self.scanString('"'),
+ '/' => {
+ self.advance();
+ if (self.ch()) |n| {
+ if (n == '/') {
+ self.skipSingleLineComment();
+ return self.next();
+ }
+ if (n == '*') {
+ self.skipMultiLineComment();
+ return self.next();
+ }
+ if (n == '=') {
+ self.advance();
+ return simple(.@"/=", start);
+ }
+ }
+ return simple(.@"/", start);
+ },
+ else => {
+ if (isLineTerminator(c) or isWhiteSpace(c)) {
+ self.skipWhiteSpace();
+ return self.next();
+ }
+ if (isDigit(c)) return self.scanNumber();
+ if (isIdStart(c)) return self.scanIdentifierOrKeyword();
+ if (c == '\\') {
+ self.advance();
+ return self.scanIdentifierOrKeyword();
+ }
+ self.advance();
+ return simple(.unknown, start);
+ },
+ }
+ }
+
+ fn simple(kind: TokenType, start: usize) Token {
+ _ = start;
+ return .{ .kind = kind, .loc = undefined, .slice = "" };
+ }
+
+ fn simpleWithSlice(comptime kind: TokenType, start: usize, slice: []const u8) Token {
+ return .{ .kind = kind, .loc = undefined, .slice = slice };
+ }
+
+ pub fn next(self: *Self) Token {
+ const start = self.pos;
+ if (self.ch()) |c| {
+ _ = c;
+ // Handle hashbang at beginning
+ if (self.pos == 0 and self.ch()) |c2| {
+ if (c2 == '#') {
+ self.skipHashbang();
+ return self.next();
+ }
+ }
+ return self.scanPunctuator();
+ }
+ return simple(.eof, self.pos);
+ }
+
+ fn keywordFromString(s: []const u8) TokenType {
+ const keywords = std.ComptimeStringMap(TokenType, .{
+ .{ "await", .await },
+ .{ "break", .@"break" },
+ .{ "case", .case },
+ .{ "catch", .@"catch" },
+ .{ "class", .class },
+ .{ "const", .@"const" },
+ .{ "continue", .@"continue" },
+ .{ "debugger", .debugger },
+ .{ "default", .default },
+ .{ "delete", .delete },
+ .{ "do", .do },
+ .{ "else", .@"else" },
+ .{ "enum", .@"enum" },
+ .{ "export", .@"export" },
+ .{ "extends", .extends },
+ .{ "false", .false },
+ .{ "finally", .finally },
+ .{ "for", .@"for" },
+ .{ "function", .function },
+ .{ "if", .@"if" },
+ .{ "import", .import },
+ .{ "in", .in },
+ .{ "instanceof", .instanceof },
+ .{ "let", .let },
+ .{ "new", .new },
+ .{ "null", .null },
+ .{ "return", .@"return" },
+ .{ "super", .super },
+ .{ "switch", .@"switch" },
+ .{ "this", .this },
+ .{ "throw", .throw },
+ .{ "true", .true },
+ .{ "try", .@"try" },
+ .{ "typeof", .typeof },
+ .{ "var", .@"var" },
+ .{ "void", .void },
+ .{ "while", .@"while" },
+ .{ "with", .with },
+ .{ "yield", .yield },
+ .{ "using", .using },
+ });
+
+ if (keywords.get(s)) |k| return k;
+ return .identifier;
+ }
+
+ pub fn tokenizeComptime(comptime source: []const u8) []const Token {
+ comptime {
+ return tokenize(source);
+ }
+ }
+};
generated by cgit v1.2.3-70-g09d2 (git 2.53.0) at 2026-09-16 11:57:33 +0000