aboutsummaryrefslogtreecommitdiff
path: root/src/z/parser/js
diff options
context:
space:
mode:
authorAlexander Rolley <arolley35@gmail.com>2026-08-05 17:25:27 +0200
committerAlexander Rolley <arolley35@gmail.com>2026-08-05 17:25:27 +0200
commitdc90bc811648da695e4cb47826ea6479cfd22a06 (patch)
tree3621725aadb61d267f1e88afb570a4ab3c824ab8 /src/z/parser/js
parent23ac91196552068d44bab5677a09a106b8172e27 (diff)
parent9b63c0f568832085b75ea6e3e983dc6cbeccfea4 (diff)
Merge branch 'dev' of https://git.nathanreiner.xyz/memora into develop
Diffstat (limited to 'src/z/parser/js')
-rw-r--r--src/z/parser/js/grammar/identifier.zig60
-rw-r--r--src/z/parser/js/grammar/keyword.zig115
-rw-r--r--src/z/parser/js/grammar/literal.zig192
-rw-r--r--src/z/parser/js/grammar/punctuator.zig122
-rw-r--r--src/z/parser/js/grammar/root.zig66
-rw-r--r--src/z/parser/js/grammar/whitespace.zig102
-rw-r--r--src/z/parser/js/root.zig7
7 files changed, 664 insertions, 0 deletions
diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig
new file mode 100644
index 0000000..ccc5268
--- /dev/null
+++ b/src/z/parser/js/grammar/identifier.zig
@@ -0,0 +1,60 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
+pub const Identifier = enum {
+ public,
+ private,
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ defer lexer.revert();
+
+ lexer.context.regex_possible = false;
+
+ const token = tok: {
+ if (lexer.peekChar() == '#') {
+ const t = lexer.start(.{ .identifier = .private });
+ lexer.skip() catch unreachable;
+ break :tok t;
+ } else {
+ break :tok lexer.start(.{ .identifier = .public });
+ }
+ };
+
+ switch (try lexer.consume()) {
+ 'a'...'z', 'A'...'Z', '_' => {},
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ while (lexer.peekChar()) |ch| {
+ switch (ch) {
+ 'a'...'z', 'A'...'Z', '0'...'9', '_' => {},
+ else => break,
+ }
+
+ try lexer.skip();
+ }
+
+ lexer.commit(token);
+ }
+ }
+};
+
+test "simple" {
+ {
+ comptime var lexer: Lexer = .init("some_literal9");
+ try Identifier.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(13, lexer.tokens[0].slice.len);
+ }
+
+ {
+ comptime var lexer: Lexer = .init("#some_literal9");
+ try Identifier.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(14, lexer.tokens[0].slice.len);
+ }
+}
diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig
new file mode 100644
index 0000000..8e99b4a
--- /dev/null
+++ b/src/z/parser/js/grammar/keyword.zig
@@ -0,0 +1,115 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
+pub const Keyword = enum {
+ @"break",
+ case,
+ @"catch",
+ class,
+ @"const",
+ @"continue",
+ debugger,
+ default,
+ delete,
+ do,
+ @"else",
+ @"export",
+ extends,
+ finally,
+ @"for",
+ function,
+ @"if",
+ import,
+ in,
+ instanceof,
+ new,
+ @"return",
+ super,
+ @"switch",
+ this,
+ throw,
+ @"try",
+ typeof,
+ @"var",
+ void,
+ @"while",
+ with,
+ yield,
+ let,
+ static,
+ await,
+ async,
+ of,
+ from,
+ get,
+ set,
+ meta,
+ @"enum",
+ null,
+ true,
+ false,
+
+ const Self = @This();
+
+ const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: {
+ var keywords: []const struct { []const u8, Self } = &.{};
+
+ for (@typeInfo(Self).@"enum".fields) |field| {
+ const name: []const u8 = field.name;
+ const value: Self = @field(Self, field.name);
+
+ keywords = keywords ++ .{ .{ name, value } };
+ }
+
+ break :kws keywords;
+ });
+
+ pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ @setEvalBranchQuota(10000);
+ errdefer lexer.revert();
+
+ lexer.context.regex_possible = true;
+
+ const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len;
+
+ if (index == 0) return Lexer.Error.UnexpectedToken;
+
+ if (keywordmap.get(lexer.buffer[0..index])) |tag| {
+ const token = lexer.start(.{ .keyword = tag });
+ try lexer.skipTo(index);
+ lexer.commit(token);
+ } else {
+ return Lexer.Error.UnexpectedToken;
+ }
+ }
+ }
+};
+
+test "simple" {
+ inline for (comptime Keyword.keywordmap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key);
+ try Keyword.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
+
+test "ending" {
+ inline for (comptime Keyword.keywordmap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key ++ ".");
+ try Keyword.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
+
+test "subword" {
+ inline for (comptime Keyword.keywordmap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key ++ "a");
+ try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer));
+ }
+}
diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
new file mode 100644
index 0000000..4316dee
--- /dev/null
+++ b/src/z/parser/js/grammar/literal.zig
@@ -0,0 +1,192 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
+pub const Literal = union(enum) {
+ // NOTE: currently only decimal literals are tokenized
+ numeric: void,
+ bigint: void,
+ string: void,
+ template: enum {
+ whole,
+ start,
+ middle,
+ end,
+ },
+ regex: void,
+
+ const digits = "_0123456789abcdef";
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ errdefer lexer.revert();
+
+ var token = lexer.start(.{ .literal = undefined });
+
+ const regex_possible = lexer.context.regex_possible;
+ lexer.context.regex_possible = false;
+
+ switch (try lexer.consume()) {
+ '0'...'9', '.' => |c| {
+ const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) {
+ 'x' => 16,
+ 'o' => 8,
+ 'b' => 2,
+ else => 10,
+ };
+
+ if (base != 10) lexer.skip() catch unreachable;
+
+ {
+ const index = std.mem.findNone(
+ u8,
+ lexer.buffer,
+ digits[0 .. base + 1],
+ ) orelse lexer.buffer.len;
+ try lexer.skipTo(index);
+
+
+ if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken;
+ }
+
+ if (base == 10 and lexer.peekChar() == '.') {
+ try lexer.skip();
+ const index = std.mem.findNone(
+ u8,
+ lexer.buffer,
+ digits[0 .. base + 1],
+ ) orelse lexer.buffer.len;
+ try lexer.skipTo(index);
+ }
+
+ if (lexer.peekChar() == 'n') {
+ lexer.skip() catch unreachable;
+ token.kind.literal = .bigint;
+ } else {
+ token.kind.literal = .numeric;
+ }
+ },
+ '\'', '"' => |quote| {
+ token.kind.literal = .string;
+ while (true) {
+ switch (try lexer.consume()) {
+ quote => break,
+ '\\' => try lexer.skip(),
+ else => {},
+ }
+ }
+ },
+ '`' => {
+ token.kind.literal = .{ .template = undefined };
+
+ while (true) {
+ switch (try lexer.consume()) {
+ '`' => {
+ token.kind.literal.template = .whole;
+ break;
+ },
+ '\\' => try lexer.skip(),
+ '$' => if (lexer.peekChar() == '{') {
+ lexer.skip() catch unreachable;
+ lexer.context.template_nesting += 1;
+ token.kind.literal.template = .start;
+ lexer.context.regex_possible = true;
+ break;
+ },
+ else => {},
+ }
+ }
+ },
+ '}' => if (lexer.context.template_nesting > 0) {
+ lexer.context.template_nesting -= 1;
+ token.kind.literal = .{ .template = undefined };
+
+ while (true) {
+ switch (try lexer.consume()) {
+ '`' => {
+ token.kind.literal.template = .end;
+ break;
+ },
+ '\\' => try lexer.skip(),
+ '$' => if (lexer.peekChar() == '{') {
+ lexer.skip() catch unreachable;
+ lexer.context.template_nesting += 1;
+ token.kind.literal.template = .middle;
+ lexer.context.regex_possible = true;
+ break;
+ },
+ else => {},
+ }
+ }
+ } else return Lexer.Error.UnexpectedToken,
+ '/' => if (regex_possible) {
+ token.kind.literal = .regex;
+ while (true) {
+ switch (try lexer.consume()) {
+ '/' => break,
+ '\\' => try lexer.skip(),
+ '\n' => return Lexer.Error.UnexpectedToken,
+ else => {},
+ }
+ }
+ } else return Lexer.Error.UnexpectedToken,
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ lexer.commit(token);
+ }
+ }
+};
+
+test "numeric" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("1234", 4);
+ try Simple.run("1.234", 5);
+ try Simple.run("1_2_3_4", 7);
+ try Simple.run("0b101011", 8);
+ try Simple.run("0xabcf01", 8);
+ try Simple.run("0o731734", 8);
+ try Simple.run("0o731734n", 9);
+}
+
+test "string" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("'hello'", 7);
+ try Simple.run("\"hello\"", 7);
+ try std.testing.expectError(
+ Lexer.Error.EndOfBuffer,
+ Simple.run("'hello", 0),
+ );
+}
+
+test "regex" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("/hello/", 7);
+ try Simple.run("/some string\\/ here/", 20);
+}
diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig
new file mode 100644
index 0000000..b085ce1
--- /dev/null
+++ b/src/z/parser/js/grammar/punctuator.zig
@@ -0,0 +1,122 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
+pub const Punctuator = enum {
+ @"+",
+ @"-",
+ @"*",
+ @"/",
+ @"%",
+ @"**",
+ @"++",
+ @"--",
+ @"=",
+ @"+=",
+ @"-=",
+ @"*=",
+ @"/=",
+ @"%=",
+ @"**=",
+ @"&=",
+ @"|=",
+ @"^=",
+ @"<<=",
+ @">>=",
+ @">>>=",
+ @"&&=",
+ @"||=",
+ @"??=",
+ @"==",
+ @"===",
+ @"!=",
+ @"!==",
+ @"<",
+ @"<=",
+ @">",
+ @">=",
+ @"&&",
+ @"||",
+ @"??",
+ @"&",
+ @"|",
+ @"^",
+ @"~",
+ @"<<",
+ @">>",
+ @">>>",
+ @"!",
+ @"?",
+ @":",
+ @";",
+ @",",
+ @".",
+ @"...",
+ @"=>",
+ @"(",
+ @")",
+ @"[",
+ @"]",
+ @"{",
+ @"}",
+ @"?.",
+ @"#",
+ @"@",
+
+ const Self = @This();
+
+ const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: {
+ var punctuators: []const struct { []const u8, Self } = &.{};
+
+ for (@typeInfo(Self).@"enum".fields) |field| {
+ const name: []const u8 = field.name;
+ const value: Self = @field(Self, field.name);
+
+ punctuators = punctuators ++ .{.{ name, value }};
+ }
+
+ break :kws punctuators;
+ });
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ @setEvalBranchQuota(10_000);
+ defer lexer.revert();
+
+ lexer.context.regex_possible = true;
+
+ if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| {
+
+ if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) {
+ return Lexer.Error.UnexpectedToken;
+ }
+
+ const token = lexer.start(.{ .punctuator = pair.value });
+ try lexer.skipTo(pair.key.len);
+ lexer.commit(token);
+ } else {
+ return Lexer.Error.UnexpectedToken;
+ }
+ }
+ }
+};
+
+test "simple" {
+ inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key);
+ try Punctuator.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
+
+test "ending" {
+ inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key ++ "a");
+ try Punctuator.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
new file mode 100644
index 0000000..f732b7f
--- /dev/null
+++ b/src/z/parser/js/grammar/root.zig
@@ -0,0 +1,66 @@
+//! This is a Javascript parser and tokenizer.
+//! It is and probably will never be complete,
+//! since js is a very strange language and I want
+//! to move forward and code on more interesting things
+//! I started something called mujs (micro-js). It's a
+//! subset of the javascript language. It does not support
+//! the regex literal and currently also not template strings.
+//! It is chosen like that to make parsing easier and also less
+//! context-dependent.
+
+const std = @import("std");
+
+const parser = @import("../../root.zig");
+
+pub const Lexer = parser.Lexer(Grammar);
+pub const Token = parser.Token(Grammar);
+
+pub const Whitespace = @import("whitespace.zig").Whitespace;
+pub const Keyword = @import("keyword.zig").Keyword;
+pub const Literal = @import("literal.zig").Literal;
+pub const Punctuator = @import("punctuator.zig").Punctuator;
+pub const Identifier = @import("identifier.zig").Identifier;
+
+pub const Grammar = union(enum) {
+ whitespace: Whitespace,
+ keyword: Keyword,
+ literal: Literal,
+ punctuator: Punctuator,
+ identifier: Identifier,
+ end_of_file: void,
+
+ pub const Context = struct {
+ template_nesting: usize = 0,
+ regex_possible: bool = true,
+ };
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ @setEvalBranchQuota(lexer.buffer.len * 1000);
+ comptime {
+ while (lexer.peekChar()) |_| {
+ Whitespace.tokenize(lexer) catch {
+ Keyword.tokenize(lexer) catch {
+ Literal.tokenize(lexer) catch {
+ Punctuator.tokenize(lexer) catch {
+ try Identifier.tokenize(lexer);
+ };
+ };
+ };
+ };
+ }
+
+ lexer.commit(lexer.start(.end_of_file));
+ }
+ }
+};
+
+test {
+ _ = std.testing.refAllDecls(@This());
+}
+
+test "z.js" {
+ comptime var lexer: Lexer = .init(@embedFile("../../../z.js"));
+ Grammar.tokenize(&lexer) catch |err| {
+ std.debug.print("error: {} at {s}", .{ err, lexer.buffer });
+ };
+}
diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig
new file mode 100644
index 0000000..08d031f
--- /dev/null
+++ b/src/z/parser/js/grammar/whitespace.zig
@@ -0,0 +1,102 @@
+const std = @import("std");
+const Lexer = @import("root.zig").Lexer;
+const Token = @import("root.zig").Token;
+
+pub const Whitespace = enum {
+ space,
+ line_comment,
+ block_comment,
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ errdefer lexer.revert();
+
+ var token = lexer.start(.{ .whitespace = undefined });
+
+ switch (try lexer.consume()) {
+ ' ', '\t', '\r', '\n' => {
+ token.kind.whitespace = .space;
+
+ while (lexer.peekChar()) |ch| {
+ switch (ch) {
+ ' ', '\t', '\r', '\n' => try lexer.skip(),
+ else => break,
+ }
+ }
+ },
+ '/' => {
+ switch (lexer.peekChar() orelse 0) {
+ '*' => {
+ token.kind.whitespace = .block_comment;
+ try lexer.skipUntil("*/");
+ },
+ '/' => {
+ token.kind.whitespace = .line_comment;
+ lexer.skipUntil("\n") catch lexer.skipToEnd();
+ },
+ else => return Lexer.Error.UnexpectedToken,
+ }
+ },
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ lexer.commit(token);
+ }
+ }
+};
+
+test "space" {
+ {
+ comptime var lexer: Lexer = .init(" ");
+ try Whitespace.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(1, lexer.tokens[0].slice.len);
+ }
+
+ {
+ comptime var lexer: Lexer = .init(" \t\t \n\r ");
+ try Whitespace.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(10, lexer.tokens[0].slice.len);
+ }
+}
+
+test "line-comment" {
+ {
+ comptime var lexer: Lexer = .init("// some comment");
+ try Whitespace.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice);
+ }
+
+ {
+ comptime var lexer: Lexer = .init("// some comment\n");
+ try Whitespace.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice);
+ }
+}
+
+test "block-comment" {
+ {
+ comptime var lexer: Lexer = .init("/* some comment */");
+ try Whitespace.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice);
+ }
+}
+
+test "mixed" {
+ comptime var lexer: Lexer = .init(
+ \\ // some comment
+ \\/* other comment here */ // comment or so
+ \\ /* multi
+ \\ * line
+ \\ * comment
+ \\ */
+ );
+ inline while (Whitespace.tokenize(&lexer)) {
+ } else |_| {}
+
+ try std.testing.expectEqual(7, lexer.tokens.len);
+}
diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig
new file mode 100644
index 0000000..e604855
--- /dev/null
+++ b/src/z/parser/js/root.zig
@@ -0,0 +1,7 @@
+const std = @import("std");
+
+pub const grammar = @import("./grammar/root.zig");
+
+test {
+ _ = std.testing.refAllDecls(@This());
+}