From 77642e5081b4ed2516dc08b3ef611c3c444ec6c5 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Sat, 1 Aug 2026 20:10:10 +0200 Subject: restructure parser --- src/z/parser/js/lexical_grammar.zig | 3 +++ src/z/parser/js/root.zig | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 src/z/parser/js/lexical_grammar.zig create mode 100644 src/z/parser/js/root.zig (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/lexical_grammar.zig b/src/z/parser/js/lexical_grammar.zig new file mode 100644 index 0000000..7060766 --- /dev/null +++ b/src/z/parser/js/lexical_grammar.zig @@ -0,0 +1,3 @@ +const parser = @import("root.zig"); +const Result = parser.Result; + diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig new file mode 100644 index 0000000..a6be9ca --- /dev/null +++ b/src/z/parser/js/root.zig @@ -0,0 +1,11 @@ +const std = @import("std"); + +pub const lexical_grammar = @import("lexical_grammar.zig"); + +pub const Result = @import("result.zig").Result; +pub const Parser = @import("parser.zig").Parser; +pub const builtin = @import("builtin.zig"); + +test { + _ = std.testing.refAllDecls(@This()); +} -- cgit v1.2.3-70-g09d2 From 0bc6d195195d1e126b535554a4a4105468cd06d9 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Mon, 3 Aug 2026 17:07:05 +0200 Subject: add tokenizer for whitespaces in js --- src/api/hello-json.zig | 2 +- src/api/hello-param.zig | 4 +- src/api/root.zig | 2 +- src/db/sqlite.zig | 95 +++++++++++++------------- src/db/sqlite_new.zig | 32 ++++----- src/html/Element.zig | 5 +- src/main.zig | 10 +-- src/web/root.zig | 1 - src/z/File.zig | 1 - src/z/parser/builtin.zig | 17 ----- src/z/parser/js/grammar/identifier.zig | 4 ++ src/z/parser/js/grammar/keyword.zig | 48 ++++++++++++++ src/z/parser/js/grammar/literal.zig | 5 ++ src/z/parser/js/grammar/punctuator.zig | 61 +++++++++++++++++ src/z/parser/js/grammar/root.zig | 36 ++++++++++ src/z/parser/js/grammar/whitespace.zig | 106 +++++++++++++++++++++++++++++ src/z/parser/js/lexical_grammar.zig | 3 - src/z/parser/js/root.zig | 6 +- src/z/parser/lexer.zig | 110 +++++++++++++++++++++++++++++++ src/z/parser/parser.zig | 117 --------------------------------- src/z/parser/result.zig | 4 -- src/z/parser/root.zig | 17 +++++ src/z/root.zig | 1 + 23 files changed, 461 insertions(+), 226 deletions(-) delete mode 100644 src/z/parser/builtin.zig create mode 100644 src/z/parser/js/grammar/identifier.zig create mode 100644 src/z/parser/js/grammar/keyword.zig create mode 100644 src/z/parser/js/grammar/literal.zig create mode 100644 src/z/parser/js/grammar/punctuator.zig create mode 100644 src/z/parser/js/grammar/root.zig create mode 100644 src/z/parser/js/grammar/whitespace.zig delete mode 100644 src/z/parser/js/lexical_grammar.zig create mode 100644 src/z/parser/lexer.zig delete mode 100644 src/z/parser/parser.zig delete mode 100644 src/z/parser/result.zig create mode 100644 src/z/parser/root.zig (limited to 'src/z/parser/js') diff --git a/src/api/hello-json.zig b/src/api/hello-json.zig index 6c29ba7..12f1e72 100644 --- a/src/api/hello-json.zig +++ b/src/api/hello-json.zig @@ -2,7 +2,7 @@ const http = @import("http"); const Context = http.handler.Context; const Response = http.Response; -const Ctx = Context(.{ "hello-json" }); +const Ctx = Context(.{"hello-json"}); pub const ctx: Ctx = .{ .get = get, }; diff --git a/src/api/hello-param.zig b/src/api/hello-param.zig index f4de1dd..b67212b 100644 --- a/src/api/hello-param.zig +++ b/src/api/hello-param.zig @@ -10,5 +10,7 @@ pub const ctx: Ctx = .{ fn get(request: Ctx.Request) Response { const value = request.params.value.as(usize) catch return .badRequest(.string("parameter needs to be an int")); - return .okOrError(.json(request.arena, .{ .value = value, })); + return .okOrError(.json(request.arena, .{ + .value = value, + })); } diff --git a/src/api/root.zig b/src/api/root.zig index 7570a76..2a54b91 100644 --- a/src/api/root.zig +++ b/src/api/root.zig @@ -16,7 +16,7 @@ pub const interfaces: []const Interface = ifs: { var ifs: []const Interface = &.{}; for (modules) |module| { - const interface: []const Interface = &.{ module.ctx.interface() }; + const interface: []const Interface = &.{module.ctx.interface()}; ifs = ifs ++ interface; } diff --git a/src/db/sqlite.zig b/src/db/sqlite.zig index 28d6487..d1accd5 100644 --- a/src/db/sqlite.zig +++ b/src/db/sqlite.zig @@ -5,7 +5,7 @@ const assert = std.debug.assert; pub fn Database(comptime T: type) type { const tables = switch (@typeInfo(T)) { .@"struct" => |s| s.fields, - else => @compileError("Database schema must be of type `struct`"), + else => @compileError("Database schema must be of type `struct`"), }; // NOTE: `std.math.log2_int_ceil(@TypeOf(tables.len), tables.len)` panics @@ -13,14 +13,14 @@ pub fn Database(comptime T: type) type { // a (useless) schema that does not define any tables. const TagIntTables = @Int(.unsigned, @bitSizeOf(@TypeOf(tables.len)) - @clz(tables.len)); - var table_names : [tables.len][]const u8 = undefined; - var table_types : [tables.len]type = undefined; + var table_names: [tables.len][]const u8 = undefined; + var table_types: [tables.len]type = undefined; var table_values: [tables.len]TagIntTables = undefined; inline for (&table_names, &table_types, &table_values, tables, 0..) |*table_name, *table_type, *table_value, table, table_index| { const columns = switch (@typeInfo(table.type)) { .@"struct" => |s| s.fields, - else => @compileError("Table `" ++ table.name ++ "` must be of type `struct`"), + else => @compileError("Table `" ++ table.name ++ "` must be of type `struct`"), }; // NOTE: A table must have at least one column @@ -29,28 +29,28 @@ pub fn Database(comptime T: type) type { const TagIntColumns = @Int(.unsigned, std.math.log2_int_ceil(@TypeOf(columns.len), columns.len)); - var column_names : [columns.len][]const u8 = undefined; - var column_types : [columns.len]type = undefined; + var column_names: [columns.len][]const u8 = undefined; + var column_types: [columns.len]type = undefined; var column_values: [columns.len]TagIntColumns = undefined; inline for (&column_names, &column_types, &column_values, columns, 0..) |*column_name, *column_type, *column_value, column, column_index| { // TODO: check column.type (of type `Type`) - column_name.* = column.name; - column_type.* = column.type; + column_name.* = column.name; + column_type.* = column.type; column_value.* = column_index; } const TagColumns = @Enum(TagIntColumns, .exhaustive, &column_names, &column_values); - const Columns = @Union(.auto, TagColumns, &column_names, &column_types, &@splat(.{})); + const Columns = @Union(.auto, TagColumns, &column_names, &column_types, &@splat(.{})); - table_name.* = table.name; - table_type.* = Columns; + table_name.* = table.name; + table_type.* = Columns; table_value.* = table_index; } const TagTables = @Enum(TagIntTables, .exhaustive, &table_names, &table_values); - const Tables = @Union(.auto, TagTables, &table_names, &table_types, &@splat(.{})); + const Tables = @Union(.auto, TagTables, &table_names, &table_types, &@splat(.{})); return struct { inner: Inner, @@ -80,7 +80,7 @@ pub fn Database(comptime T: type) type { }; const Schema = Tables; - const Self = @This(); + const Self = @This(); const Inner = struct { handle: *sqlite.sqlite3, @@ -89,14 +89,14 @@ pub fn Database(comptime T: type) type { pub fn init(kind: Kind) Error!Self { const name = switch (kind) { .temp_in_memory => ":memory:", - .temp_on_disk => "", - .path => |path| path, + .temp_on_disk => "", + .path => |path| path, }; var handle: ?*sqlite.sqlite3 = null; const result: Result = .from(sqlite.sqlite3_open(name.ptr, &handle)); const primary = result.asPrimary(); - + return if (primary == .ok) .{ .handle = handle.? } else result.toError(Error, handle); } @@ -132,8 +132,6 @@ fn Select(comptime Db: type, comptime table: std.meta.Tag(Db.Schema), comptime c }; } - - // pub const Type = struct { // string: String.Static, // type: type, @@ -266,7 +264,6 @@ const String = struct { // } // }; - /// SQLite result codes. /// /// Primary result codes define a broad category of results, whereas extended @@ -445,14 +442,14 @@ test { const Contacts = struct { contact_id: usize, // CHANGEME: to `Type` - const all = &.{ .contact_id }; + const all = &.{.contact_id}; }; }; - var db: Database(Schema) = try .init(.temp_in_memory); + var db: Database(Schema) = try .init(.temp_in_memory); defer assert(db.deinit() == .ok); - _ = db.select(.contacts, &.{ .contact_id }, .all); + _ = db.select(.contacts, &.{.contact_id}, .all); var ty = db.select(.contacts, Schema.Contacts.all, .all); ty.table = .foo; @@ -462,7 +459,7 @@ test { // db.select(.distinct(...)).from(...)... // // // If the simple SELECT is a SELECT ALL, then the entire set of result rows are returned by the SELECT. If neither ALL or DISTINCT are present, then the behavior is as if ALL were specified. - // db.select(enum { all, distinct }, + // db.select(enum { all, distinct }, // * == .all(), // table-name.* == .table(.table_name), // @@ -470,30 +467,30 @@ test { } // test { - // NOTE: we don't use `union(enum)` here as if a column has a default value, - // an integer tag type is needed explicitely, which, from a usability - // standpoint, is not great; we can calulate this value in `Schema.from`. - // - // CREATE TABLE contacts ( - // contact_id INTEGER PRIMARY KEY, - // first_name TEXT NOT NULL DEFAULT "Hans", - // last_name TEXT NOT NULL DEFAULT "Mustermann", - // nickname TEXT, - // email TEXT NOT NULL UNIQUE, - // phone TEXT NOT NULL UNIQUE - // ); - // const S = struct { - // contacts: struct { - // contact_id: PrimaryKey(Integer, .{}), - // first_name: VarChar(255, .{}) = "Hans", - // last_name: VarChar(255, .{}) = "Mustermann", - // nickname: ?VarChar(255, .{}), - // email: VarChar(255, .{.unique}), - // phone: VarChar(255, .{.unique}), - // }, - // }; - // - // // NOTE: init connects to DB and creates tables if not already present - // var db: Database(Schema(S)) = try .init(.temp_in_memory); - // db.deinit(); +// NOTE: we don't use `union(enum)` here as if a column has a default value, +// an integer tag type is needed explicitely, which, from a usability +// standpoint, is not great; we can calulate this value in `Schema.from`. +// +// CREATE TABLE contacts ( +// contact_id INTEGER PRIMARY KEY, +// first_name TEXT NOT NULL DEFAULT "Hans", +// last_name TEXT NOT NULL DEFAULT "Mustermann", +// nickname TEXT, +// email TEXT NOT NULL UNIQUE, +// phone TEXT NOT NULL UNIQUE +// ); +// const S = struct { +// contacts: struct { +// contact_id: PrimaryKey(Integer, .{}), +// first_name: VarChar(255, .{}) = "Hans", +// last_name: VarChar(255, .{}) = "Mustermann", +// nickname: ?VarChar(255, .{}), +// email: VarChar(255, .{.unique}), +// phone: VarChar(255, .{.unique}), +// }, +// }; +// +// // NOTE: init connects to DB and creates tables if not already present +// var db: Database(Schema(S)) = try .init(.temp_in_memory); +// db.deinit(); // } diff --git a/src/db/sqlite_new.zig b/src/db/sqlite_new.zig index 46706b8..36eaf61 100644 --- a/src/db/sqlite_new.zig +++ b/src/db/sqlite_new.zig @@ -25,7 +25,7 @@ pub fn Database(comptime T: type) type { // TODO: make functions out of `table` and `columns`: Table(Schema), []const Column(Schema, table) for readability? pub fn select(self: *Self, comptime table: std.meta.Tag(Schema), comptime columns: []const std.meta.Tag(@FieldType(Schema, @tagName(table)))) Select(Schema, table, columns) { _ = self; - std.debug.print("{any}\n", .{ columns }); + std.debug.print("{any}\n", .{columns}); return .{ // .handle = self.handle, }; @@ -152,17 +152,17 @@ fn Unique(comptime T: type) type { const Type = T.Type; // TODO: `contains(expected, str)` function const string = if (std.mem.count(u8, T.string, "UNIQUE") > 0) - @compileError(std.fmt.comptimePrint("Type `{s}` is already unique", .{ @typeName(T) })) + @compileError(std.fmt.comptimePrint("Type `{s}` is already unique", .{@typeName(T)})) else - std.fmt.comptimePrint("UNIQUE {s}", .{ T.string }); + std.fmt.comptimePrint("UNIQUE {s}", .{T.string}); }; } fn Default(comptime T: type, comptime value: T.Type) type { return struct { const Type = T.type; - const string = if(std.mem.count(u8, T.string, "DEFAULT") > 0) - @compileError(std.fmt.comptimePrint("Type `{s}` already has a default value", .{ @typeName(T) })) + const string = if (std.mem.count(u8, T.string, "DEFAULT") > 0) + @compileError(std.fmt.comptimePrint("Type `{s}` already has a default value", .{@typeName(T)})) else std.fmt.comptimePrint("{s} DEFAULT {}", .{ T.string, value }); }; @@ -202,12 +202,12 @@ fn Test(comptime Schema: type, comptime table: std.meta.Tag(Schema), comptime co fn Eql(comptime column: std.meta.Tag(ColumnUnion), value: @FieldType(ColumnUnion, @tagName(column))) type { return struct { - const fmt: []const u8 = std.fmt.comptimePrint("{s} = ", .{ @tagName(column) }); - const args: @Tuple(&.{ @TypeOf(value) }) = .{ value }; + const fmt: []const u8 = std.fmt.comptimePrint("{s} = ", .{@tagName(column)}); + const args: @Tuple(&.{@TypeOf(value)}) = .{value}; }; } // fn And() type { - // todo: concat fmt slices and args tuples + // todo: concat fmt slices and args tuples // } string: Node, @@ -236,10 +236,10 @@ test "schema" { const Schema = struct { contacts: Contacts, // contacts: struct { - // contact_id: Integer = .constrain(.primary_key), - // age: Integer = .unconstrained, - // contact_id: Integer(usize) = .constrain(.{ .primary_key = true }), - // first_name: Default(VarChar(255), "Hans"), + // contact_id: Integer = .constrain(.primary_key), + // age: Integer = .unconstrained, + // contact_id: Integer(usize) = .constrain(.{ .primary_key = true }), + // first_name: Default(VarChar(255), "Hans"), // }, }; @@ -254,10 +254,10 @@ test "schema" { const t: Test(@TypeOf(db).Schema, .contacts, &.{ .id, .first_name }) = .eql(.id, 42); // const t: Test(@TypeOf(db).Schema, .contacts, &.{ .id, .first_name }) = .op_and(.eql(.id, 42), .eql(.first_name, "name")); - std.debug.print("test output: {s}{}\n", switch (t.string) { .eql => |eq| .{ eq.fmt, eq.args } }); + std.debug.print("test output: {s}{}\n", switch (t.string) { + .eql => |eq| .{ eq.fmt, eq.args }, + }); const Int = Default(Unique(Integer(usize)), 42); - std.debug.print("default unique integer with default 42: {s}\n", .{ Int.string }); - - + std.debug.print("default unique integer with default 42: {s}\n", .{Int.string}); } diff --git a/src/html/Element.zig b/src/html/Element.zig index 8ecf31e..a9e5897 100644 --- a/src/html/Element.zig +++ b/src/html/Element.zig @@ -39,7 +39,7 @@ fn attributesFrom(attributes: anytype) []const Attribute { const name = field.name; const value = @field(attributes, name); - const next_attr: []const Attribute = &.{ .{ .name = name, .value = value } }; + const next_attr: []const Attribute = &.{.{ .name = name, .value = value }}; attrs = attrs ++ next_attr; } @@ -79,7 +79,6 @@ pub fn toHtml(comptime self: Self) []const u8 { break :string string; }; - switch (self.content) { .void => {}, .string => |s| { @@ -91,7 +90,7 @@ pub fn toHtml(comptime self: Self) []const u8 { } string = string ++ ""; - } + }, } string = string ++ if (self.tag.len == 0) "" else ""; diff --git a/src/main.zig b/src/main.zig index a429d14..176d3b2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -12,11 +12,7 @@ const web = @import("web/root.zig"); const routes: http.RouteSet = .init(api.interfaces ++ web.interfaces); pub fn main(init: std.process.Init) !void { - try http.serve( - init.gpa, - routes, - .{ - .address = try .parseLiteral("0.0.0.0:8080"), - } - ); + try http.serve(init.gpa, routes, .{ + .address = try .parseLiteral("0.0.0.0:8080"), + }); } diff --git a/src/web/root.zig b/src/web/root.zig index c9c1b08..d027dc8 100644 --- a/src/web/root.zig +++ b/src/web/root.zig @@ -24,4 +24,3 @@ const document: html.Element = .native(.html, .{}, .children(&.{ pub const interfaces: []const http.handler.Interface = &.{ Static(.index, document.toDocument()).interface(), }; - diff --git a/src/z/File.zig b/src/z/File.zig index cd7094f..cd19dba 100644 --- a/src/z/File.zig +++ b/src/z/File.zig @@ -15,4 +15,3 @@ pub fn load(comptime self: @This(), comptime content: []const u8) Element { return .transparent(.content(comptime self.minifier.minify(content))); } } - diff --git a/src/z/parser/builtin.zig b/src/z/parser/builtin.zig deleted file mode 100644 index fc96fc3..0000000 --- a/src/z/parser/builtin.zig +++ /dev/null @@ -1,17 +0,0 @@ -const std = @import("std"); - -const parser = @import("root.zig"); -const Parser = parser.Parser; -const Result = parser.Result; - -pub fn literal(lit: []const u8) Parser(void) { - return .fromType(struct { - pub fn parse(buffer: []const u8) !Result(void) { - if (std.mem.startsWith(u8, buffer, lit)) { - return .{ void{}, buffer[lit.len..] }; - } - - return error.Literal; - } - }); -} diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig new file mode 100644 index 0000000..038dbc5 --- /dev/null +++ b/src/z/parser/js/grammar/identifier.zig @@ -0,0 +1,4 @@ +pub const Identifier = enum { + public, + private, +}; diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig new file mode 100644 index 0000000..14c3272 --- /dev/null +++ b/src/z/parser/js/grammar/keyword.zig @@ -0,0 +1,48 @@ +pub const Keyword = enum { + @"break", + case, + @"catch", + class, + @"const", + @"continue", + debugger, + default, + delete, + do, + @"else", + @"export", + extends, + finally, + @"for", + function, + @"if", + import, + in, + instanceof, + new, + @"return", + super, + @"switch", + this, + throw, + @"try", + typeof, + @"var", + void, + @"while", + with, + yield, + let, + static, + await, + async, + of, + from, + get, + set, + meta, + @"enum", + null, + true, + false, +}; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..a2ef97a --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,5 @@ +pub const Literal = enum { + numeric, + bigint, + string, +}; diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig new file mode 100644 index 0000000..81b2e6e --- /dev/null +++ b/src/z/parser/js/grammar/punctuator.zig @@ -0,0 +1,61 @@ +pub const Punctuator = enum { + @"+", + @"-", + @"*", + @"/", + @"%", + @"**", + @"++", + @"--", + @"=", + @"+=", + @"-=", + @"*=", + @"/=", + @"%=", + @"**=", + @"&=", + @"|=", + @"^=", + @"<<=", + @">>=", + @">>>=", + @"&&=", + @"||=", + @"??=", + @"==", + @"===", + @"!=", + @"!==", + @"<", + @"<=", + @">", + @">=", + @"&&", + @"||", + @"??", + @"&", + @"|", + @"^", + @"~", + @"<<", + @">>", + @">>>", + @"!", + @"?", + @":", + @";", + @",", + @".", + @"...", + @"=>", + @"(", + @")", + @"[", + @"]", + @"{", + @"}", + @"?.", + @"#", + @"@", +}; diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig new file mode 100644 index 0000000..e1a1e96 --- /dev/null +++ b/src/z/parser/js/grammar/root.zig @@ -0,0 +1,36 @@ +const std = @import("std"); + +const parser = @import("../../root.zig"); +pub const Lexer = parser.Lexer(Grammar); +pub const Token = parser.Token(Grammar); + +pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Literal = @import("literal.zig").Literal; +pub const Punctuator = @import("punctuator.zig").Punctuator; +pub const Identifier = @import("identifier.zig").Identifier; + +pub const Grammar = union(enum) { + whitespace: Whitespace, + literal: Literal, + punctuator: Punctuator, + identifier: Identifier, + end_of_file: void, + + pub fn tokenize(lexer: *Lexer) Lexer.Error!void { + while (lexer.peakChar()) |_| { + Whitespace.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; + }; + } + + lexer.push(lexer.token(0, .end_of_file)); + } +}; + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..c3435c4 --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,106 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(undefined); + + switch (lexer.peekChar() orelse 0) { + ' ', '\t', '\r', '\n' => { + token.kind = .{ .whitespace = .space }; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekCharAt(1) orelse 0) { + '*' => { + token.kind = .{ .whitespace = .block_comment }; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind = .{ .whitespace = .line_comment }; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); + + inline for (lexer.tokens) |token| { + std.debug.print("{f}\n", .{token}); + } +} diff --git a/src/z/parser/js/lexical_grammar.zig b/src/z/parser/js/lexical_grammar.zig deleted file mode 100644 index 7060766..0000000 --- a/src/z/parser/js/lexical_grammar.zig +++ /dev/null @@ -1,3 +0,0 @@ -const parser = @import("root.zig"); -const Result = parser.Result; - diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig index a6be9ca..e604855 100644 --- a/src/z/parser/js/root.zig +++ b/src/z/parser/js/root.zig @@ -1,10 +1,6 @@ const std = @import("std"); -pub const lexical_grammar = @import("lexical_grammar.zig"); - -pub const Result = @import("result.zig").Result; -pub const Parser = @import("parser.zig").Parser; -pub const builtin = @import("builtin.zig"); +pub const grammar = @import("./grammar/root.zig"); test { _ = std.testing.refAllDecls(@This()); diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig new file mode 100644 index 0000000..d9a2868 --- /dev/null +++ b/src/z/parser/lexer.zig @@ -0,0 +1,110 @@ +const std = @import("std"); + +pub fn Token(TokenKind: type) type { + return struct { + const Self = @This(); + + kind: TokenKind, + slice: []const u8, + + pub fn start(comptime lexer: *const Lexer(TokenKind), kind: TokenKind) Self { + return .{ + .kind = kind, + .slice = lexer.buffer, + }; + } + + pub inline fn stop(comptime self: Self, comptime lexer: *const Lexer(TokenKind)) Self { + comptime { + const length = self.slice.len - lexer.buffer.len; + return .{ + .kind = self.kind, + .slice = self.slice[0..length], + }; + } + } + + pub fn format(self: *const Self, writer: *std.Io.Writer) !void { + try writer.print("Token{}(\"{s}\")", .{self.kind, self.slice}); + } + }; +} + +pub fn Lexer(TokenKind: type) type { + return struct { + const Self = @This(); + + pub const Error = error{ + UnexpectedToken, + EndOfBuffer, + }; + + buffer: []const u8, + last_buffer: []const u8, + tokens: []const Token(TokenKind) = &.{}, + + pub fn init(buffer: []const u8) Self { + return .{ + .buffer = buffer, + .last_buffer = buffer, + }; + } + + pub inline fn peekChar(self: *Self) ?u8 { + return self.peekCharAt(0); + } + + pub inline fn peekCharAt(self: *Self, n: usize) ?u8 { + return if (n < self.buffer.len) self.buffer[n] else null; + } + + pub fn peekSlice(self: *Self, length: usize) ?[]const u8 { + return self.peekSliceAt(length, 0); + } + + pub fn peekSliceAt(self: *Self, length: usize, offset: usize) ?[]const u8 { + if (length + offset <= self.buffer.len) { + return self.buffer[offset .. offset + length]; + } + return null; + } + + pub fn skip(self: *Self) !void { + return self.skipTo(1); + } + + pub fn skipTo(self: *Self, n: usize) Error!void { + if (n <= self.buffer.len) { + self.buffer = self.buffer[n..]; + } else { + return Error.EndOfBuffer; + } + } + + pub fn skipUntil(self: *Self, literal: []const u8) Error!void { + if (std.mem.find(u8, self.buffer, literal)) |index| { + self.buffer = self.buffer[index + literal.len..]; + } else { + return Error.EndOfBuffer; + } + } + + pub fn skipToEnd(self: *Self) void { + self.buffer = self.buffer[self.buffer.len..]; + } + + pub fn start(self: *Self, kind: TokenKind) Token(TokenKind) { + return .start(self, kind); + } + + pub fn revert(self: *Self) void { + self.buffer = self.last_buffer; + } + + pub fn commit(self: *Self, tok: Token(TokenKind)) void { + self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; + self.last_buffer = self.buffer; + } + + }; +} diff --git a/src/z/parser/parser.zig b/src/z/parser/parser.zig deleted file mode 100644 index b73d895..0000000 --- a/src/z/parser/parser.zig +++ /dev/null @@ -1,117 +0,0 @@ -const std = @import("std"); - -const mod = @import("root.zig"); -const Result = mod.Result; - -pub fn Parser(T: type) type { - const R = Result(T); - - return struct { - parse: *const fn ([]const u8) anyerror!R, - - pub fn parseAll(self: *const @This(), buffer: []const u8) !T { - const value, const next = try self.parse(buffer); - - if (next.len > 0) { - return error.NotExhaustive; - } - - return value; - } - - pub fn fromType(P: type) @This() { - return .{ .parse = P.parse }; - } - - pub fn variants(V: type) @This() { - if (@typeInfo(T) != .@"union") { - @compileError("variants is only allowed with an union as result type"); - } - - return .{ - .parse = struct { - fn parse(buffer: []const u8) !R { - inline for (std.meta.fields(T)) |field| { - if (@field(V, field.name).parse(buffer)) |result| { - const value, const next = result; - return .{ - @unionInit(T, field.name, value), - next, - }; - } else |_| { } - } - - return error.Variants; - } - }.parse, - }; - } - - pub fn any(P: type, comptime parsers: []const struct { Parser(P), T }) @This() { - return .{ - .parse = struct { - fn parse(buffer: []const u8) !R { - inline for (parsers) |pair| { - const parser, const value = pair; - - if (parser.parse(buffer)) |result| { - _, const next = result; - return .{ value, next }; - } else |_| { } - } - return error.Any; - } - }.parse, - }; - } - }; -} - -test "variants" { - const Char = union(enum) { - upper: u8, - lower: u8, - space: void, - }; - - const parser: Parser(Char) = .variants(struct { - const upper: Parser(u8) = .any(void, &.{ - .{ mod.builtin.literal("A"), 'A' }, - .{ mod.builtin.literal("B"), 'B' }, - }); - - const lower: Parser(u8) = .any(void, &.{ - .{ mod.builtin.literal("a"), 'a' }, - .{ mod.builtin.literal("b"), 'b' }, - }); - - const space: Parser(void) = mod.builtin.literal(" "); - }); - - { - const result = try parser.parseAll("a"); - try std.testing.expect(result.lower == 'a'); - } - - { - const result = try parser.parseAll("b"); - try std.testing.expect(result.lower == 'b'); - } - - { - const result = try parser.parseAll("A"); - try std.testing.expect(result.upper == 'A'); - } - - { - const result = try parser.parseAll("B"); - try std.testing.expect(result.upper == 'B'); - } - - { - const result = try parser.parseAll(" "); - try std.testing.expect(result == .space); - } - - try std.testing.expect(parser.parseAll("c") == error.Variants); -} diff --git a/src/z/parser/result.zig b/src/z/parser/result.zig deleted file mode 100644 index 5cc5878..0000000 --- a/src/z/parser/result.zig +++ /dev/null @@ -1,4 +0,0 @@ - -pub fn Result(T: type) type { - return struct { T, []const u8 }; -} diff --git a/src/z/parser/root.zig b/src/z/parser/root.zig new file mode 100644 index 0000000..e0b67fb --- /dev/null +++ b/src/z/parser/root.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +const lexer = @import("lexer.zig"); +pub const Lexer = lexer.Lexer; +pub const Token = lexer.Token; + +pub const js = @import("js/root.zig"); + +pub fn Parser(TokenType: type) type { + return struct { + buffer: []const TokenType, + }; +} + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/root.zig b/src/z/root.zig index b23ebc5..1fa8d99 100644 --- a/src/z/root.zig +++ b/src/z/root.zig @@ -3,6 +3,7 @@ const std = @import("std"); pub const Minifier = @import("Minifier.zig"); pub const Component = @import("Component.zig"); pub const File = @import("File.zig"); +pub const parser = @import("parser/root.zig"); const Element = @import("html").Element; -- cgit v1.2.3-70-g09d2 From a10b091d78208907397a6d49791b27e04e675161 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Tue, 4 Aug 2026 19:02:27 +0200 Subject: add tokenizer for all other variants --- src/z/parser/js/grammar/identifier.zig | 54 +++++++++++++++++++++++ src/z/parser/js/grammar/keyword.zig | 65 +++++++++++++++++++++++++++ src/z/parser/js/grammar/literal.zig | 81 ++++++++++++++++++++++++++++++++++ src/z/parser/js/grammar/punctuator.zig | 54 +++++++++++++++++++++++ src/z/parser/js/grammar/root.zig | 34 ++++++++++---- src/z/parser/js/grammar/whitespace.zig | 16 +++---- src/z/parser/lexer.zig | 14 ++++++ 7 files changed, 299 insertions(+), 19 deletions(-) (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig index 038dbc5..905f937 100644 --- a/src/z/parser/js/grammar/identifier.zig +++ b/src/z/parser/js/grammar/identifier.zig @@ -1,4 +1,58 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Identifier = enum { public, private, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + defer lexer.revert(); + + const token = tok: { + if (lexer.peekChar() == '#') { + const t = lexer.start(.{ .identifier = .private }); + lexer.skip() catch unreachable; + break :tok t; + } else { + break :tok lexer.start(.{ .identifier = .public }); + } + }; + + switch (try lexer.consume()) { + 'a'...'z', 'A'...'Z', '_' => {}, + else => return Lexer.Error.UnexpectedToken, + } + + while (lexer.peekChar()) |ch| { + switch (ch) { + 'a'...'z', 'A'...'Z', '0'...'9', '_' => {}, + else => break, + } + + try lexer.skip(); + } + + lexer.commit(token); + } + } }; + +test "simple" { + { + comptime var lexer: Lexer = .init("some_literal9"); + try Identifier.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(13, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init("#some_literal9"); + try Identifier.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(14, lexer.tokens[0].slice.len); + } +} diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig index 14c3272..c3b32ac 100644 --- a/src/z/parser/js/grammar/keyword.zig +++ b/src/z/parser/js/grammar/keyword.zig @@ -1,3 +1,9 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Keyword = enum { @"break", case, @@ -45,4 +51,63 @@ pub const Keyword = enum { null, true, false, + + const Self = @This(); + + const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: { + var keywords: []const struct { []const u8, Self } = &.{}; + + for (@typeInfo(Self).@"enum".fields) |field| { + const name: []const u8 = field.name; + const value: Self = @field(Self, field.name); + + keywords = keywords ++ .{ .{ name, value } }; + } + + break :kws keywords; + }); + + pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void { + comptime { + @setEvalBranchQuota(10000); + errdefer lexer.revert(); + + const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; + + if (index == 0) return Lexer.Error.UnexpectedToken; + + if (keywordmap.get(lexer.buffer[0..index])) |tag| { + const token = lexer.start(.{ .keyword = tag }); + try lexer.skipTo(index); + lexer.commit(token); + } else { + return Lexer.Error.UnexpectedToken; + } + } + } }; + +test "simple" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key); + try Keyword.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "ending" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "."); + try Keyword.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "subword" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "a"); + try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer)); + } +} diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index a2ef97a..7f92240 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -1,5 +1,86 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Literal = enum { + // NOTE: currently only decimal literals are tokenized numeric, bigint, string, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(.{ .literal = undefined }); + + switch (try lexer.consume()) { + '0'...'9', '.' => |c| { + token.kind.literal = .numeric; + + var had_period = c == '.'; + + while (lexer.peekChar()) |ch| { + switch (ch) { + '0'...'9' => {}, + '.' => if (had_period) { + return Lexer.Error.UnexpectedToken; + } else { + had_period = true; + }, + else => return Lexer.Error.UnexpectedToken, + } + + try lexer.skip(); + } + }, + '\'', '"' => |quote| while (try lexer.consume() != quote) { + if (quote == '\\') { + try lexer.consume(); + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } }; + +test "numeric" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("1234", 4); + try Simple.run("1.234", 5); + try std.testing.expectError( + Lexer.Error.UnexpectedToken, + Simple.run("12.3.4", 0), + ); +} + +test "string" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("'hello'", 7); + try Simple.run("\"hello\"", 7); + try std.testing.expectError( + Lexer.Error.EndOfBuffer, + Simple.run("'hello", 0), + ); +} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index 81b2e6e..a72ab8a 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -1,3 +1,9 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + pub const Punctuator = enum { @"+", @"-", @@ -58,4 +64,52 @@ pub const Punctuator = enum { @"?.", @"#", @"@", + + const Self = @This(); + + const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: { + var punctuators: []const struct { []const u8, Self } = &.{}; + + for (@typeInfo(Self).@"enum".fields) |field| { + const name: []const u8 = field.name; + const value: Self = @field(Self, field.name); + + punctuators = punctuators ++ .{.{ name, value }}; + } + + break :kws punctuators; + }); + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + @setEvalBranchQuota(10_000); + defer lexer.revert(); + + if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { + const token = lexer.start(.{ .punctuator = pair.value }); + try lexer.skipTo(pair.key.len); + lexer.commit(token); + } else { + return Lexer.Error.UnexpectedToken; + } + } + } }; + +test "simple" { + inline for (comptime Punctuator.punctuatormap.keys()) |key| { + comptime var lexer: Lexer = .init(key); + try Punctuator.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "ending" { + inline for (comptime Punctuator.punctuatormap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "a"); + try Punctuator.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index e1a1e96..03b386e 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -1,3 +1,13 @@ +//! This is a Javascript parser and tokenizer. +//! It is and probably will never be complete, +//! since js is a very strange language and I want +//! to move forward and code on more interesting things +//! I started something called mujs (micro-js). It's a +//! subset of the javascript language. It does not support +//! the regex literal and currently also not template strings. +//! It is chosen like that to make parsing easier and also less +//! context-dependent. + const std = @import("std"); const parser = @import("../../root.zig"); @@ -5,29 +15,35 @@ pub const Lexer = parser.Lexer(Grammar); pub const Token = parser.Token(Grammar); pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Keyword = @import("keyword.zig").Keyword; pub const Literal = @import("literal.zig").Literal; pub const Punctuator = @import("punctuator.zig").Punctuator; pub const Identifier = @import("identifier.zig").Identifier; pub const Grammar = union(enum) { whitespace: Whitespace, + keyword: Keyword, literal: Literal, punctuator: Punctuator, identifier: Identifier, end_of_file: void, - pub fn tokenize(lexer: *Lexer) Lexer.Error!void { - while (lexer.peakChar()) |_| { - Whitespace.tokenize(lexer) catch { - Literal.tokenize(lexer) catch { - Punctuator.tokenize(lexer) catch { - try Identifier.tokenize(lexer); + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + while (lexer.peekChar()) |_| { + Whitespace.tokenize(lexer) catch { + Keyword.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; }; }; - }; - } + } - lexer.push(lexer.token(0, .end_of_file)); + lexer.commit(lexer.start(.end_of_file)); + } } }; diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig index c3435c4..08d031f 100644 --- a/src/z/parser/js/grammar/whitespace.zig +++ b/src/z/parser/js/grammar/whitespace.zig @@ -11,11 +11,11 @@ pub const Whitespace = enum { comptime { errdefer lexer.revert(); - var token = lexer.start(undefined); + var token = lexer.start(.{ .whitespace = undefined }); - switch (lexer.peekChar() orelse 0) { + switch (try lexer.consume()) { ' ', '\t', '\r', '\n' => { - token.kind = .{ .whitespace = .space }; + token.kind.whitespace = .space; while (lexer.peekChar()) |ch| { switch (ch) { @@ -25,13 +25,13 @@ pub const Whitespace = enum { } }, '/' => { - switch (lexer.peekCharAt(1) orelse 0) { + switch (lexer.peekChar() orelse 0) { '*' => { - token.kind = .{ .whitespace = .block_comment }; + token.kind.whitespace = .block_comment; try lexer.skipUntil("*/"); }, '/' => { - token.kind = .{ .whitespace = .line_comment }; + token.kind.whitespace = .line_comment; lexer.skipUntil("\n") catch lexer.skipToEnd(); }, else => return Lexer.Error.UnexpectedToken, @@ -99,8 +99,4 @@ test "mixed" { } else |_| {} try std.testing.expectEqual(7, lexer.tokens.len); - - inline for (lexer.tokens) |token| { - std.debug.print("{f}\n", .{token}); - } } diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index d9a2868..40e1209 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -69,6 +69,20 @@ pub fn Lexer(TokenKind: type) type { return null; } + pub fn consume(self: *Self) !u8 { + return (try self.consumeTo(1))[0]; + } + + pub fn consumeTo(self: *Self, n: usize) ![]const u8 { + if (n <= self.buffer.len) { + const slice = self.buffer[0..n]; + self.buffer = self.buffer[n..]; + return slice; + } else { + return Error.EndOfBuffer; + } + } + pub fn skip(self: *Self) !void { return self.skipTo(1); } -- cgit v1.2.3-70-g09d2 From 06d1607aabad0a967c0500efe305de3fa732f8d3 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Tue, 4 Aug 2026 20:42:35 +0200 Subject: add numeric tokenizer --- src/z/parser/js/grammar/literal.zig | 68 +++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 21 deletions(-) (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 7f92240..6a91291 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -10,6 +10,8 @@ pub const Literal = enum { bigint, string, + const digits = "_0123456789abcdef"; + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { comptime { errdefer lexer.revert(); @@ -18,27 +20,50 @@ pub const Literal = enum { switch (try lexer.consume()) { '0'...'9', '.' => |c| { - token.kind.literal = .numeric; - - var had_period = c == '.'; - - while (lexer.peekChar()) |ch| { - switch (ch) { - '0'...'9' => {}, - '.' => if (had_period) { - return Lexer.Error.UnexpectedToken; - } else { - had_period = true; - }, - else => return Lexer.Error.UnexpectedToken, - } + const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { + 'x' => 16, + 'o' => 8, + 'b' => 2, + else => 10, + }; + + if (base != 10) lexer.skip() catch unreachable; + + { + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + + + if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken; + } + if (base == 10 and lexer.peekChar() == '.') { try lexer.skip(); + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + } + + if (lexer.peekChar() == 'n') { + lexer.skip() catch unreachable; + token.kind.literal = .bigint; + } else { + token.kind.literal = .numeric; } }, - '\'', '"' => |quote| while (try lexer.consume() != quote) { - if (quote == '\\') { - try lexer.consume(); + '\'', '"' => |quote| { + token.kind.literal = .string; + while (try lexer.consume() != quote) { + if (quote == '\\') { + try lexer.consume(); + } } }, else => return Lexer.Error.UnexpectedToken, @@ -61,10 +86,11 @@ test "numeric" { try Simple.run("1234", 4); try Simple.run("1.234", 5); - try std.testing.expectError( - Lexer.Error.UnexpectedToken, - Simple.run("12.3.4", 0), - ); + try Simple.run("1_2_3_4", 7); + try Simple.run("0b101011", 8); + try Simple.run("0xabcf01", 8); + try Simple.run("0o731734", 8); + try Simple.run("0o731734n", 9); } test "string" { -- cgit v1.2.3-70-g09d2 From 356c2bc0ccb1794b5ec219079d8c5b85ddcc4c87 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Tue, 4 Aug 2026 22:28:25 +0200 Subject: add templates to tokenizer --- src/z/parser/js/grammar/literal.zig | 63 ++++++++++++++++++++++++++++++---- src/z/parser/js/grammar/punctuator.zig | 5 +++ src/z/parser/js/grammar/root.zig | 13 +++++++ src/z/parser/lexer.zig | 2 +- 4 files changed, 75 insertions(+), 8 deletions(-) (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 6a91291..039ebf4 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -4,11 +4,17 @@ const root = @import("root.zig"); const Lexer = root.Lexer; const Token = root.Token; -pub const Literal = enum { +pub const Literal = union(enum) { // NOTE: currently only decimal literals are tokenized - numeric, - bigint, - string, + numeric: void, + bigint: void, + string: void, + template: enum { + whole, + start, + middle, + end, + }, const digits = "_0123456789abcdef"; @@ -60,9 +66,52 @@ pub const Literal = enum { }, '\'', '"' => |quote| { token.kind.literal = .string; - while (try lexer.consume() != quote) { - if (quote == '\\') { - try lexer.consume(); + while (true) { + switch (try lexer.consume()) { + quote => break, + '\\' => try lexer.consume(), + else => {}, + } + } + }, + '`' => { + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .whole; + break; + }, + '\\' => try lexer.consume(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .start; + break; + }, + else => {}, + } + } + }, + '}' => if (lexer.context.template_nesting > 0) { + lexer.context.template_nesting -= 1; + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .end; + break; + }, + '\\' => try lexer.consume(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .middle; + break; + }, + else => {}, } } }, diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index a72ab8a..4fd5d16 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -86,6 +86,11 @@ pub const Punctuator = enum { defer lexer.revert(); if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { + + if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { + return Lexer.Error.UnexpectedToken; + } + const token = lexer.start(.{ .punctuator = pair.value }); try lexer.skipTo(pair.key.len); lexer.commit(token); diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index 03b386e..56f3990 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -11,6 +11,7 @@ const std = @import("std"); const parser = @import("../../root.zig"); + pub const Lexer = parser.Lexer(Grammar); pub const Token = parser.Token(Grammar); @@ -28,7 +29,12 @@ pub const Grammar = union(enum) { identifier: Identifier, end_of_file: void, + pub const Context = struct { + template_nesting: usize = 0, + }; + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + @setEvalBranchQuota(lexer.buffer.len * 1000); comptime { while (lexer.peekChar()) |_| { Whitespace.tokenize(lexer) catch { @@ -50,3 +56,10 @@ pub const Grammar = union(enum) { test { _ = std.testing.refAllDecls(@This()); } + +test "z.js" { + comptime var lexer: Lexer = .init(@embedFile("../../../z.js")); + Grammar.tokenize(&lexer) catch |err| { + std.debug.print("error: {} at {s}", .{ err, lexer.buffer }); + }; +} diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index 40e1209..66d4211 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -42,6 +42,7 @@ pub fn Lexer(TokenKind: type) type { buffer: []const u8, last_buffer: []const u8, tokens: []const Token(TokenKind) = &.{}, + context: TokenKind.Context = .{}, pub fn init(buffer: []const u8) Self { return .{ @@ -119,6 +120,5 @@ pub fn Lexer(TokenKind: type) type { self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; self.last_buffer = self.buffer; } - }; } -- cgit v1.2.3-70-g09d2 From 0eaa2a420cb699cdb23ce9b1573560a5d9e59a31 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Wed, 5 Aug 2026 09:08:24 +0200 Subject: add regex literals --- src/z/parser/js/grammar/identifier.zig | 2 ++ src/z/parser/js/grammar/keyword.zig | 2 ++ src/z/parser/js/grammar/literal.zig | 37 ++++++++++++++++++++++++++++++---- src/z/parser/js/grammar/punctuator.zig | 2 ++ src/z/parser/js/grammar/root.zig | 1 + src/z/parser/lexer.zig | 3 +++ 6 files changed, 43 insertions(+), 4 deletions(-) (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig index 905f937..ccc5268 100644 --- a/src/z/parser/js/grammar/identifier.zig +++ b/src/z/parser/js/grammar/identifier.zig @@ -12,6 +12,8 @@ pub const Identifier = enum { comptime { defer lexer.revert(); + lexer.context.regex_possible = false; + const token = tok: { if (lexer.peekChar() == '#') { const t = lexer.start(.{ .identifier = .private }); diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig index c3b32ac..8e99b4a 100644 --- a/src/z/parser/js/grammar/keyword.zig +++ b/src/z/parser/js/grammar/keyword.zig @@ -72,6 +72,8 @@ pub const Keyword = enum { @setEvalBranchQuota(10000); errdefer lexer.revert(); + lexer.context.regex_possible = true; + const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; if (index == 0) return Lexer.Error.UnexpectedToken; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 039ebf4..71b990d 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -24,6 +24,9 @@ pub const Literal = union(enum) { var token = lexer.start(.{ .literal = undefined }); + const regex_possible = lexer.context.regex_possible; + lexer.context.regex_possible = false; + switch (try lexer.consume()) { '0'...'9', '.' => |c| { const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { @@ -69,7 +72,7 @@ pub const Literal = union(enum) { while (true) { switch (try lexer.consume()) { quote => break, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), else => {}, } } @@ -83,11 +86,12 @@ pub const Literal = union(enum) { token.kind.literal.template = .whole; break; }, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .start; + lexer.context.regex_possible = true; break; }, else => {}, @@ -104,17 +108,28 @@ pub const Literal = union(enum) { token.kind.literal.template = .end; break; }, - '\\' => try lexer.consume(), + '\\' => try lexer.skip(), '$' => if (lexer.peekChar() == '{') { lexer.skip() catch unreachable; lexer.context.template_nesting += 1; token.kind.literal.template = .middle; + lexer.context.regex_possible = true; break; }, else => {}, } } - }, + } else return Lexer.Error.UnexpectedToken, + '/' => if (regex_possible) { + while (true) { + switch (try lexer.consume()) { + '/' => break, + '\\' => try lexer.skip(), + '\n' => return Lexer.Error.UnexpectedToken, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, else => return Lexer.Error.UnexpectedToken, } @@ -159,3 +174,17 @@ test "string" { Simple.run("'hello", 0), ); } + +test "regex" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("/hello/", 7); + try Simple.run("/some string\\/ here/", 20); +} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig index 4fd5d16..b085ce1 100644 --- a/src/z/parser/js/grammar/punctuator.zig +++ b/src/z/parser/js/grammar/punctuator.zig @@ -85,6 +85,8 @@ pub const Punctuator = enum { @setEvalBranchQuota(10_000); defer lexer.revert(); + lexer.context.regex_possible = true; + if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig index 56f3990..f732b7f 100644 --- a/src/z/parser/js/grammar/root.zig +++ b/src/z/parser/js/grammar/root.zig @@ -31,6 +31,7 @@ pub const Grammar = union(enum) { pub const Context = struct { template_nesting: usize = 0, + regex_possible: bool = true, }; pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig index 66d4211..5beefde 100644 --- a/src/z/parser/lexer.zig +++ b/src/z/parser/lexer.zig @@ -43,6 +43,7 @@ pub fn Lexer(TokenKind: type) type { last_buffer: []const u8, tokens: []const Token(TokenKind) = &.{}, context: TokenKind.Context = .{}, + last_context: TokenKind.Context = .{}, pub fn init(buffer: []const u8) Self { return .{ @@ -114,11 +115,13 @@ pub fn Lexer(TokenKind: type) type { pub fn revert(self: *Self) void { self.buffer = self.last_buffer; + self.context = self.last_context; } pub fn commit(self: *Self, tok: Token(TokenKind)) void { self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; self.last_buffer = self.buffer; + self.last_context = self.context; } }; } -- cgit v1.2.3-70-g09d2 From 9b63c0f568832085b75ea6e3e983dc6cbeccfea4 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Wed, 5 Aug 2026 09:10:06 +0200 Subject: regex literal set token kind --- src/z/parser/js/grammar/literal.zig | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/z/parser/js') diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig index 71b990d..4316dee 100644 --- a/src/z/parser/js/grammar/literal.zig +++ b/src/z/parser/js/grammar/literal.zig @@ -15,6 +15,7 @@ pub const Literal = union(enum) { middle, end, }, + regex: void, const digits = "_0123456789abcdef"; @@ -121,6 +122,7 @@ pub const Literal = union(enum) { } } else return Lexer.Error.UnexpectedToken, '/' => if (regex_possible) { + token.kind.literal = .regex; while (true) { switch (try lexer.consume()) { '/' => break, -- cgit v1.2.3-70-g09d2