From 02582cc490fc0d753dd153211cc172eb28fbee5d Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Mon, 5 Oct 2026 15:33:46 +0200 Subject: first sketch of wasm environment --- src/z/parser/js/grammar/identifier.zig | 60 ----------- src/z/parser/js/grammar/keyword.zig | 115 -------------------- src/z/parser/js/grammar/literal.zig | 192 --------------------------------- src/z/parser/js/grammar/punctuator.zig | 122 --------------------- src/z/parser/js/grammar/root.zig | 66 ------------ src/z/parser/js/grammar/whitespace.zig | 102 ------------------ src/z/parser/js/root.zig | 7 -- 7 files changed, 664 deletions(-) delete mode 100644 src/z/parser/js/grammar/identifier.zig delete mode 100644 src/z/parser/js/grammar/keyword.zig delete mode 100644 src/z/parser/js/grammar/literal.zig delete mode 100644 src/z/parser/js/grammar/punctuator.zig delete mode 100644 src/z/parser/js/grammar/root.zig delete mode 100644 src/z/parser/js/grammar/whitespace.zig delete mode 100644 src/z/parser/js/root.zig (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig deleted file mode 100644 index ccc5268..0000000 --- a/src/z/parser/js/grammar/identifier.zig +++ /dev/null @@ -1,60 +0,0 @@ -const std = @import("std"); - -const root = @import("root.zig"); -const Lexer = root.Lexer; -const Token = root.Token; - -pub const Identifier = enum { - public, - private, - - pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { - comptime { - defer lexer.revert(); - - lexer.context.regex_possible = false; - - const token = tok: { - if (lexer.peekChar() == '#') { - const t = lexer.start(.{ .identifier = .private }); - lexer.skip() catch unreachable; - break :tok t; - } else { - break :tok lexer.start(.{ .identifier = .public }); - } - }; - - switch (try lexer.consume()) { - 'a'...'z', 'A'...'Z', '_' => {}, - else => return Lexer.Error.UnexpectedToken, - } - - while (lexer.peekChar()) |ch| { - switch (ch) { - 'a'...'z', 'A'...'Z', '0'...'9', '_' => {}, - else => break, - } - - try lexer.skip(); - } - - lexer.commit(token); - } - } -}; - -test "simple" { - { - comptime var lexer: Lexer = .init("some_literal9"); - try Identifier.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(13, lexer.tokens[0].slice.len); - } - - { - comptime var lexer: Lexer = .init("#some_literal9"); - try Identifier.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(14, lexer.tokens[0].slice.len); - } -} diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig deleted file mode 100644 index 8e99b4a..0000000 --- a/src/z/parser/js/grammar/keyword.zig +++ /dev/null @@ -1,115 +0,0 @@ -const std = @import("std"); - -const root = @import("root.zig"); -const Lexer = root.Lexer; -const Token = root.Token; - -pub const Keyword = enum { - @"break", - case, - @"catch", - class, - @"const", - @"continue", - debugger, - default, - delete, - do, - @"else", - @"export", - extends, - finally, - @"for", - function, - @"if", - import, - in, - instanceof, - new, - @"return", - super, - @"switch", - this, - throw, - @"try", - typeof, - @"var", - void, - @"while", - with, - yield, - let, - static, - await, - async, - of, - from, - get, - set, - meta, - @"enum", - null, - true, - false, - - const Self = @This(); - - const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: { - var keywords: []const struct { []const u8, Self } = &.{}; - - for (@typeInfo(Self).@"enum".fields) |field| { - const name: []const u8 = field.name; - const value: Self = @field(Self, field.name); - - keywords = keywords ++ .{ .{ name, value } }; - } - - break :kws keywords; - }); - - pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void { - comptime { - @setEvalBranchQuota(10000); - errdefer lexer.revert(); - - lexer.context.regex_possible = true; - - const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; - - if (index == 0) return Lexer.Error.UnexpectedToken; - - if (keywordmap.get(lexer.buffer[0..index])) |tag| { - const token = lexer.start(.{ .keyword = tag }); - try lexer.skipTo(index); - lexer.commit(token); - } else { - return Lexer.Error.UnexpectedToken; - } - } - } -}; - -test "simple" { - inline for (comptime Keyword.keywordmap.keys()) |key| { - comptime var lexer: Lexer = .init(key); - try Keyword.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); - } -} - -test "ending" { - inline for (comptime Keyword.keywordmap.keys()) |key| { - comptime var lexer: Lexer = .init(key ++ "."); - try Keyword.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); - } -} - -test "subword" { - inline for (comptime Keyword.keywordmap.keys()) |key| { - comptime var lexer: Lexer = .init(key ++ "a"); - try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer)); - } -} diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig deleted file mode 100644 index 4316dee..0000000 --- a/src/z/parser/js/grammar/literal.zig +++ /dev/null @@ -1,192 +0,0 @@ -const std = @import("std"); - -const root = @import("root.zig"); -const Lexer = root.Lexer; -const Token = root.Token; - -pub const Literal = union(enum) { - // NOTE: currently only decimal literals are tokenized - numeric: void, - bigint: void, - string: void, - template: enum { - whole, - start, - middle, - end, - }, - regex: void, - - const digits = "_0123456789abcdef"; - - pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { - comptime { - errdefer lexer.revert(); - - var token = lexer.start(.{ .literal = undefined }); - - const regex_possible = lexer.context.regex_possible; - lexer.context.regex_possible = false; - - switch (try lexer.consume()) { - '0'...'9', '.' => |c| { - const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { - 'x' => 16, - 'o' => 8, - 'b' => 2, - else => 10, - }; - - if (base != 10) lexer.skip() catch unreachable; - - { - const index = std.mem.findNone( - u8, - lexer.buffer, - digits[0 .. base + 1], - ) orelse lexer.buffer.len; - try lexer.skipTo(index); - - - if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken; - } - - if (base == 10 and lexer.peekChar() == '.') { - try lexer.skip(); - const index = std.mem.findNone( - u8, - lexer.buffer, - digits[0 .. base + 1], - ) orelse lexer.buffer.len; - try lexer.skipTo(index); - } - - if (lexer.peekChar() == 'n') { - lexer.skip() catch unreachable; - token.kind.literal = .bigint; - } else { - token.kind.literal = .numeric; - } - }, - '\'', '"' => |quote| { - token.kind.literal = .string; - while (true) { - switch (try lexer.consume()) { - quote => break, - '\\' => try lexer.skip(), - else => {}, - } - } - }, - '`' => { - token.kind.literal = .{ .template = undefined }; - - while (true) { - switch (try lexer.consume()) { - '`' => { - token.kind.literal.template = .whole; - break; - }, - '\\' => try lexer.skip(), - '$' => if (lexer.peekChar() == '{') { - lexer.skip() catch unreachable; - lexer.context.template_nesting += 1; - token.kind.literal.template = .start; - lexer.context.regex_possible = true; - break; - }, - else => {}, - } - } - }, - '}' => if (lexer.context.template_nesting > 0) { - lexer.context.template_nesting -= 1; - token.kind.literal = .{ .template = undefined }; - - while (true) { - switch (try lexer.consume()) { - '`' => { - token.kind.literal.template = .end; - break; - }, - '\\' => try lexer.skip(), - '$' => if (lexer.peekChar() == '{') { - lexer.skip() catch unreachable; - lexer.context.template_nesting += 1; - token.kind.literal.template = .middle; - lexer.context.regex_possible = true; - break; - }, - else => {}, - } - } - } else return Lexer.Error.UnexpectedToken, - '/' => if (regex_possible) { - token.kind.literal = .regex; - while (true) { - switch (try lexer.consume()) { - '/' => break, - '\\' => try lexer.skip(), - '\n' => return Lexer.Error.UnexpectedToken, - else => {}, - } - } - } else return Lexer.Error.UnexpectedToken, - else => return Lexer.Error.UnexpectedToken, - } - - lexer.commit(token); - } - } -}; - -test "numeric" { - const Simple = struct { - pub fn run(comptime buffer: []const u8, n: usize) !void { - comptime var lexer: Lexer = .init(buffer); - try Literal.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(n, lexer.tokens[0].slice.len); - } - }; - - try Simple.run("1234", 4); - try Simple.run("1.234", 5); - try Simple.run("1_2_3_4", 7); - try Simple.run("0b101011", 8); - try Simple.run("0xabcf01", 8); - try Simple.run("0o731734", 8); - try Simple.run("0o731734n", 9); -} - -test "string" { - const Simple = struct { - pub fn run(comptime buffer: []const u8, n: usize) !void { - comptime var lexer: Lexer = .init(buffer); - try Literal.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(n, lexer.tokens[0].slice.len); - } - }; - - try Simple.run("'hello'", 7); - try Simple.run("\"hello\"", 7); - try std.testing.expectError( - Lexer.Error.EndOfBuffer, - Simple.run("'hello", 0), - ); -} - -test "regex" { - const Simple = struct { - pub fn run(comptime buffer: []const u8, n: usize) !void { - comptime var lexer: Lexer = .init(buffer); - try Literal.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(n, lexer.tokens[0].slice.len); - } - }; - - try Simple.run("/hello/", 7); - try Simple.run("/some string\\/ here/", 20); -} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig deleted file mode 100644 index b085ce1..0000000 --- a/src/z/parser/js/grammar/punctuator.zig +++ /dev/null @@ -1,122 +0,0 @@ -const std = @import("std"); - -const root = @import("root.zig"); -const Lexer = root.Lexer; -const Token = root.Token; - -pub const Punctuator = enum { - @"+", - @"-", - @"*", - @"/", - @"%", - @"**", - @"++", - @"--", - @"=", - @"+=", - @"-=", - @"*=", - @"/=", - @"%=", - @"**=", - @"&=", - @"|=", - @"^=", - @"<<=", - @">>=", - @">>>=", - @"&&=", - @"||=", - @"??=", - @"==", - @"===", - @"!=", - @"!==", - @"<", - @"<=", - @">", - @">=", - @"&&", - @"||", - @"??", - @"&", - @"|", - @"^", - @"~", - @"<<", - @">>", - @">>>", - @"!", - @"?", - @":", - @";", - @",", - @".", - @"...", - @"=>", - @"(", - @")", - @"[", - @"]", - @"{", - @"}", - @"?.", - @"#", - @"@", - - const Self = @This(); - - const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: { - var punctuators: []const struct { []const u8, Self } = &.{}; - - for (@typeInfo(Self).@"enum".fields) |field| { - const name: []const u8 = field.name; - const value: Self = @field(Self, field.name); - - punctuators = punctuators ++ .{.{ name, value }}; - } - - break :kws punctuators; - }); - - pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { - comptime { - @setEvalBranchQuota(10_000); - defer lexer.revert(); - - lexer.context.regex_possible = true; - - if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { - - if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { - return Lexer.Error.UnexpectedToken; - } - - const token = lexer.start(.{ .punctuator = pair.value }); - try lexer.skipTo(pair.key.len); - lexer.commit(token); - } else { - return Lexer.Error.UnexpectedToken; - } - } - } -}; - -test "simple" { - inline for (comptime Punctuator.punctuatormap.keys()) |key| { - comptime var lexer: Lexer = .init(key); - try Punctuator.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); - } -} - -test "ending" { - inline for (comptime Punctuator.punctuatormap.keys()) |key| { - comptime var lexer: Lexer = .init(key ++ "a"); - try Punctuator.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); - } -} diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig deleted file mode 100644 index f732b7f..0000000 --- a/src/z/parser/js/grammar/root.zig +++ /dev/null @@ -1,66 +0,0 @@ -//! This is a Javascript parser and tokenizer. -//! It is and probably will never be complete, -//! since js is a very strange language and I want -//! to move forward and code on more interesting things -//! I started something called mujs (micro-js). It's a -//! subset of the javascript language. It does not support -//! the regex literal and currently also not template strings. -//! It is chosen like that to make parsing easier and also less -//! context-dependent. - -const std = @import("std"); - -const parser = @import("../../root.zig"); - -pub const Lexer = parser.Lexer(Grammar); -pub const Token = parser.Token(Grammar); - -pub const Whitespace = @import("whitespace.zig").Whitespace; -pub const Keyword = @import("keyword.zig").Keyword; -pub const Literal = @import("literal.zig").Literal; -pub const Punctuator = @import("punctuator.zig").Punctuator; -pub const Identifier = @import("identifier.zig").Identifier; - -pub const Grammar = union(enum) { - whitespace: Whitespace, - keyword: Keyword, - literal: Literal, - punctuator: Punctuator, - identifier: Identifier, - end_of_file: void, - - pub const Context = struct { - template_nesting: usize = 0, - regex_possible: bool = true, - }; - - pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { - @setEvalBranchQuota(lexer.buffer.len * 1000); - comptime { - while (lexer.peekChar()) |_| { - Whitespace.tokenize(lexer) catch { - Keyword.tokenize(lexer) catch { - Literal.tokenize(lexer) catch { - Punctuator.tokenize(lexer) catch { - try Identifier.tokenize(lexer); - }; - }; - }; - }; - } - - lexer.commit(lexer.start(.end_of_file)); - } - } -}; - -test { - _ = std.testing.refAllDecls(@This()); -} - -test "z.js" { - comptime var lexer: Lexer = .init(@embedFile("../../../z.js")); - Grammar.tokenize(&lexer) catch |err| { - std.debug.print("error: {} at {s}", .{ err, lexer.buffer }); - }; -} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig deleted file mode 100644 index 08d031f..0000000 --- a/src/z/parser/js/grammar/whitespace.zig +++ /dev/null @@ -1,102 +0,0 @@ -const std = @import("std"); -const Lexer = @import("root.zig").Lexer; -const Token = @import("root.zig").Token; - -pub const Whitespace = enum { - space, - line_comment, - block_comment, - - pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { - comptime { - errdefer lexer.revert(); - - var token = lexer.start(.{ .whitespace = undefined }); - - switch (try lexer.consume()) { - ' ', '\t', '\r', '\n' => { - token.kind.whitespace = .space; - - while (lexer.peekChar()) |ch| { - switch (ch) { - ' ', '\t', '\r', '\n' => try lexer.skip(), - else => break, - } - } - }, - '/' => { - switch (lexer.peekChar() orelse 0) { - '*' => { - token.kind.whitespace = .block_comment; - try lexer.skipUntil("*/"); - }, - '/' => { - token.kind.whitespace = .line_comment; - lexer.skipUntil("\n") catch lexer.skipToEnd(); - }, - else => return Lexer.Error.UnexpectedToken, - } - }, - else => return Lexer.Error.UnexpectedToken, - } - - lexer.commit(token); - } - } -}; - -test "space" { - { - comptime var lexer: Lexer = .init(" "); - try Whitespace.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(1, lexer.tokens[0].slice.len); - } - - { - comptime var lexer: Lexer = .init(" \t\t \n\r "); - try Whitespace.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqual(10, lexer.tokens[0].slice.len); - } -} - -test "line-comment" { - { - comptime var lexer: Lexer = .init("// some comment"); - try Whitespace.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); - } - - { - comptime var lexer: Lexer = .init("// some comment\n"); - try Whitespace.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); - } -} - -test "block-comment" { - { - comptime var lexer: Lexer = .init("/* some comment */"); - try Whitespace.tokenize(&lexer); - try std.testing.expectEqual(1, lexer.tokens.len); - try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); - } -} - -test "mixed" { - comptime var lexer: Lexer = .init( - \\ // some comment - \\/* other comment here */ // comment or so - \\ /* multi - \\ * line - \\ * comment - \\ */ - ); - inline while (Whitespace.tokenize(&lexer)) { - } else |_| {} - - try std.testing.expectEqual(7, lexer.tokens.len); -} diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig deleted file mode 100644 index e604855..0000000 --- a/src/z/parser/js/root.zig +++ /dev/null @@ -1,7 +0,0 @@ -const std = @import("std"); - -pub const grammar = @import("./grammar/root.zig"); - -test { - _ = std.testing.refAllDecls(@This()); -} -- cgit v1.2.3-70-g09d2