add templates to tokenizer

4 files changed, 75 insertions, 8 deletions

diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
index 6a91291..039ebf4 100644
--- a/src/z/parser/js/grammar/literal.zig
+++ b/src/z/parser/js/grammar/literal.zig
@@ -4,11 +4,17 @@ const root = @import("root.zig");
 const Lexer = root.Lexer;
 const Token = root.Token;
 
-pub const Literal = enum {
+pub const Literal = union(enum) {
     // NOTE: currently only decimal literals are tokenized
-    numeric,
-    bigint,
-    string,
+    numeric: void,
+    bigint: void,
+    string: void,
+    template: enum {
+        whole,
+        start,
+        middle,
+        end,
+    },
 
     const digits = "_0123456789abcdef";
 
@@ -60,9 +66,52 @@ pub const Literal = enum {
                 },
                 '\'', '"' => |quote| {
                     token.kind.literal = .string;
-                    while (try lexer.consume() != quote) {
-                        if (quote == '\\') {
-                            try lexer.consume();
+                    while (true) {
+                        switch (try lexer.consume()) {
+                            quote => break,
+                            '\\' => try lexer.consume(),
+                            else => {},
+                        }
+                    }
+                },
+                '`' => {
+                    token.kind.literal = .{ .template = undefined };
+
+                    while (true) {
+                        switch (try lexer.consume()) {
+                            '`' => {
+                                token.kind.literal.template = .whole;
+                                break;
+                            },
+                            '\\' => try lexer.consume(),
+                            '$' => if (lexer.peekChar() == '{') {
+                                lexer.skip() catch unreachable;
+                                lexer.context.template_nesting += 1;
+                                token.kind.literal.template = .start;
+                                break;
+                            },
+                            else => {},
+                        }
+                    }
+                },
+                '}' => if (lexer.context.template_nesting > 0) {
+                    lexer.context.template_nesting -= 1;
+                    token.kind.literal = .{ .template = undefined };
+
+                    while (true) {
+                        switch (try lexer.consume()) {
+                            '`' => {
+                                token.kind.literal.template = .end;
+                                break;
+                            },
+                            '\\' => try lexer.consume(),
+                            '$' => if (lexer.peekChar() == '{') {
+                                lexer.skip() catch unreachable;
+                                lexer.context.template_nesting += 1;
+                                token.kind.literal.template = .middle;
+                                break;
+                            },
+                            else => {},
                         }
                     }
                 },
diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig
index a72ab8a..4fd5d16 100644
--- a/src/z/parser/js/grammar/punctuator.zig
+++ b/src/z/parser/js/grammar/punctuator.zig
@@ -86,6 +86,11 @@ pub const Punctuator = enum {
             defer lexer.revert();
 
             if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| {
+
+                if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) {
+                    return Lexer.Error.UnexpectedToken;
+                }
+
                 const token = lexer.start(.{ .punctuator = pair.value });
                 try lexer.skipTo(pair.key.len);
                 lexer.commit(token);
diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
index 03b386e..56f3990 100644
--- a/src/z/parser/js/grammar/root.zig
+++ b/src/z/parser/js/grammar/root.zig
@@ -11,6 +11,7 @@
 const std = @import("std");
 
 const parser = @import("../../root.zig");
+
 pub const Lexer = parser.Lexer(Grammar);
 pub const Token = parser.Token(Grammar);
 
@@ -28,7 +29,12 @@ pub const Grammar = union(enum) {
     identifier: Identifier,
     end_of_file: void,
 
+    pub const Context = struct {
+        template_nesting: usize = 0,
+    };
+
     pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        @setEvalBranchQuota(lexer.buffer.len * 1000);
         comptime {
             while (lexer.peekChar()) |_| {
                 Whitespace.tokenize(lexer) catch {
@@ -50,3 +56,10 @@ pub const Grammar = union(enum) {
 test {
     _ = std.testing.refAllDecls(@This());
 }
+
+test "z.js" {
+    comptime var lexer: Lexer = .init(@embedFile("../../../z.js"));
+    Grammar.tokenize(&lexer) catch |err| {
+        std.debug.print("error: {} at {s}", .{ err, lexer.buffer });
+    };
+}
diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig
index 40e1209..66d4211 100644
--- a/src/z/parser/lexer.zig
+++ b/src/z/parser/lexer.zig
@@ -42,6 +42,7 @@ pub fn Lexer(TokenKind: type) type {
         buffer: []const u8,
         last_buffer: []const u8,
         tokens: []const Token(TokenKind) = &.{},
+        context: TokenKind.Context = .{},
 
         pub fn init(buffer: []const u8) Self {
             return .{
@@ -119,6 +120,5 @@ pub fn Lexer(TokenKind: type) type {
             self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)};
             self.last_buffer = self.buffer;
         }
-
     };
 }