From 0bc6d195195d1e126b535554a4a4105468cd06d9 Mon Sep 17 00:00:00 2001 From: Nathan Reiner Date: Mon, 3 Aug 2026 17:07:05 +0200 Subject: add tokenizer for whitespaces in js --- src/api/hello-json.zig | 2 +- src/api/hello-param.zig | 4 +- src/api/root.zig | 2 +- src/db/sqlite.zig | 95 +++++++++++++------------- src/db/sqlite_new.zig | 32 ++++----- src/html/Element.zig | 5 +- src/main.zig | 10 +-- src/web/root.zig | 1 - src/z/File.zig | 1 - src/z/parser/builtin.zig | 17 ----- src/z/parser/js/grammar/identifier.zig | 4 ++ src/z/parser/js/grammar/keyword.zig | 48 ++++++++++++++ src/z/parser/js/grammar/literal.zig | 5 ++ src/z/parser/js/grammar/punctuator.zig | 61 +++++++++++++++++ src/z/parser/js/grammar/root.zig | 36 ++++++++++ src/z/parser/js/grammar/whitespace.zig | 106 +++++++++++++++++++++++++++++ src/z/parser/js/lexical_grammar.zig | 3 - src/z/parser/js/root.zig | 6 +- src/z/parser/lexer.zig | 110 +++++++++++++++++++++++++++++++ src/z/parser/parser.zig | 117 --------------------------------- src/z/parser/result.zig | 4 -- src/z/parser/root.zig | 17 +++++ src/z/root.zig | 1 + 23 files changed, 461 insertions(+), 226 deletions(-) delete mode 100644 src/z/parser/builtin.zig create mode 100644 src/z/parser/js/grammar/identifier.zig create mode 100644 src/z/parser/js/grammar/keyword.zig create mode 100644 src/z/parser/js/grammar/literal.zig create mode 100644 src/z/parser/js/grammar/punctuator.zig create mode 100644 src/z/parser/js/grammar/root.zig create mode 100644 src/z/parser/js/grammar/whitespace.zig delete mode 100644 src/z/parser/js/lexical_grammar.zig create mode 100644 src/z/parser/lexer.zig delete mode 100644 src/z/parser/parser.zig delete mode 100644 src/z/parser/result.zig create mode 100644 src/z/parser/root.zig diff --git a/src/api/hello-json.zig b/src/api/hello-json.zig index 6c29ba7..12f1e72 100644 --- a/src/api/hello-json.zig +++ b/src/api/hello-json.zig @@ -2,7 +2,7 @@ const http = @import("http"); const Context = http.handler.Context; const Response = http.Response; -const Ctx = Context(.{ "hello-json" }); +const Ctx = Context(.{"hello-json"}); pub const ctx: Ctx = .{ .get = get, }; diff --git a/src/api/hello-param.zig b/src/api/hello-param.zig index f4de1dd..b67212b 100644 --- a/src/api/hello-param.zig +++ b/src/api/hello-param.zig @@ -10,5 +10,7 @@ pub const ctx: Ctx = .{ fn get(request: Ctx.Request) Response { const value = request.params.value.as(usize) catch return .badRequest(.string("parameter needs to be an int")); - return .okOrError(.json(request.arena, .{ .value = value, })); + return .okOrError(.json(request.arena, .{ + .value = value, + })); } diff --git a/src/api/root.zig b/src/api/root.zig index 7570a76..2a54b91 100644 --- a/src/api/root.zig +++ b/src/api/root.zig @@ -16,7 +16,7 @@ pub const interfaces: []const Interface = ifs: { var ifs: []const Interface = &.{}; for (modules) |module| { - const interface: []const Interface = &.{ module.ctx.interface() }; + const interface: []const Interface = &.{module.ctx.interface()}; ifs = ifs ++ interface; } diff --git a/src/db/sqlite.zig b/src/db/sqlite.zig index 28d6487..d1accd5 100644 --- a/src/db/sqlite.zig +++ b/src/db/sqlite.zig @@ -5,7 +5,7 @@ const assert = std.debug.assert; pub fn Database(comptime T: type) type { const tables = switch (@typeInfo(T)) { .@"struct" => |s| s.fields, - else => @compileError("Database schema must be of type `struct`"), + else => @compileError("Database schema must be of type `struct`"), }; // NOTE: `std.math.log2_int_ceil(@TypeOf(tables.len), tables.len)` panics @@ -13,14 +13,14 @@ pub fn Database(comptime T: type) type { // a (useless) schema that does not define any tables. const TagIntTables = @Int(.unsigned, @bitSizeOf(@TypeOf(tables.len)) - @clz(tables.len)); - var table_names : [tables.len][]const u8 = undefined; - var table_types : [tables.len]type = undefined; + var table_names: [tables.len][]const u8 = undefined; + var table_types: [tables.len]type = undefined; var table_values: [tables.len]TagIntTables = undefined; inline for (&table_names, &table_types, &table_values, tables, 0..) |*table_name, *table_type, *table_value, table, table_index| { const columns = switch (@typeInfo(table.type)) { .@"struct" => |s| s.fields, - else => @compileError("Table `" ++ table.name ++ "` must be of type `struct`"), + else => @compileError("Table `" ++ table.name ++ "` must be of type `struct`"), }; // NOTE: A table must have at least one column @@ -29,28 +29,28 @@ pub fn Database(comptime T: type) type { const TagIntColumns = @Int(.unsigned, std.math.log2_int_ceil(@TypeOf(columns.len), columns.len)); - var column_names : [columns.len][]const u8 = undefined; - var column_types : [columns.len]type = undefined; + var column_names: [columns.len][]const u8 = undefined; + var column_types: [columns.len]type = undefined; var column_values: [columns.len]TagIntColumns = undefined; inline for (&column_names, &column_types, &column_values, columns, 0..) |*column_name, *column_type, *column_value, column, column_index| { // TODO: check column.type (of type `Type`) - column_name.* = column.name; - column_type.* = column.type; + column_name.* = column.name; + column_type.* = column.type; column_value.* = column_index; } const TagColumns = @Enum(TagIntColumns, .exhaustive, &column_names, &column_values); - const Columns = @Union(.auto, TagColumns, &column_names, &column_types, &@splat(.{})); + const Columns = @Union(.auto, TagColumns, &column_names, &column_types, &@splat(.{})); - table_name.* = table.name; - table_type.* = Columns; + table_name.* = table.name; + table_type.* = Columns; table_value.* = table_index; } const TagTables = @Enum(TagIntTables, .exhaustive, &table_names, &table_values); - const Tables = @Union(.auto, TagTables, &table_names, &table_types, &@splat(.{})); + const Tables = @Union(.auto, TagTables, &table_names, &table_types, &@splat(.{})); return struct { inner: Inner, @@ -80,7 +80,7 @@ pub fn Database(comptime T: type) type { }; const Schema = Tables; - const Self = @This(); + const Self = @This(); const Inner = struct { handle: *sqlite.sqlite3, @@ -89,14 +89,14 @@ pub fn Database(comptime T: type) type { pub fn init(kind: Kind) Error!Self { const name = switch (kind) { .temp_in_memory => ":memory:", - .temp_on_disk => "", - .path => |path| path, + .temp_on_disk => "", + .path => |path| path, }; var handle: ?*sqlite.sqlite3 = null; const result: Result = .from(sqlite.sqlite3_open(name.ptr, &handle)); const primary = result.asPrimary(); - + return if (primary == .ok) .{ .handle = handle.? } else result.toError(Error, handle); } @@ -132,8 +132,6 @@ fn Select(comptime Db: type, comptime table: std.meta.Tag(Db.Schema), comptime c }; } - - // pub const Type = struct { // string: String.Static, // type: type, @@ -266,7 +264,6 @@ const String = struct { // } // }; - /// SQLite result codes. /// /// Primary result codes define a broad category of results, whereas extended @@ -445,14 +442,14 @@ test { const Contacts = struct { contact_id: usize, // CHANGEME: to `Type` - const all = &.{ .contact_id }; + const all = &.{.contact_id}; }; }; - var db: Database(Schema) = try .init(.temp_in_memory); + var db: Database(Schema) = try .init(.temp_in_memory); defer assert(db.deinit() == .ok); - _ = db.select(.contacts, &.{ .contact_id }, .all); + _ = db.select(.contacts, &.{.contact_id}, .all); var ty = db.select(.contacts, Schema.Contacts.all, .all); ty.table = .foo; @@ -462,7 +459,7 @@ test { // db.select(.distinct(...)).from(...)... // // // If the simple SELECT is a SELECT ALL, then the entire set of result rows are returned by the SELECT. If neither ALL or DISTINCT are present, then the behavior is as if ALL were specified. - // db.select(enum { all, distinct }, + // db.select(enum { all, distinct }, // * == .all(), // table-name.* == .table(.table_name), // @@ -470,30 +467,30 @@ test { } // test { - // NOTE: we don't use `union(enum)` here as if a column has a default value, - // an integer tag type is needed explicitely, which, from a usability - // standpoint, is not great; we can calulate this value in `Schema.from`. - // - // CREATE TABLE contacts ( - // contact_id INTEGER PRIMARY KEY, - // first_name TEXT NOT NULL DEFAULT "Hans", - // last_name TEXT NOT NULL DEFAULT "Mustermann", - // nickname TEXT, - // email TEXT NOT NULL UNIQUE, - // phone TEXT NOT NULL UNIQUE - // ); - // const S = struct { - // contacts: struct { - // contact_id: PrimaryKey(Integer, .{}), - // first_name: VarChar(255, .{}) = "Hans", - // last_name: VarChar(255, .{}) = "Mustermann", - // nickname: ?VarChar(255, .{}), - // email: VarChar(255, .{.unique}), - // phone: VarChar(255, .{.unique}), - // }, - // }; - // - // // NOTE: init connects to DB and creates tables if not already present - // var db: Database(Schema(S)) = try .init(.temp_in_memory); - // db.deinit(); +// NOTE: we don't use `union(enum)` here as if a column has a default value, +// an integer tag type is needed explicitely, which, from a usability +// standpoint, is not great; we can calulate this value in `Schema.from`. +// +// CREATE TABLE contacts ( +// contact_id INTEGER PRIMARY KEY, +// first_name TEXT NOT NULL DEFAULT "Hans", +// last_name TEXT NOT NULL DEFAULT "Mustermann", +// nickname TEXT, +// email TEXT NOT NULL UNIQUE, +// phone TEXT NOT NULL UNIQUE +// ); +// const S = struct { +// contacts: struct { +// contact_id: PrimaryKey(Integer, .{}), +// first_name: VarChar(255, .{}) = "Hans", +// last_name: VarChar(255, .{}) = "Mustermann", +// nickname: ?VarChar(255, .{}), +// email: VarChar(255, .{.unique}), +// phone: VarChar(255, .{.unique}), +// }, +// }; +// +// // NOTE: init connects to DB and creates tables if not already present +// var db: Database(Schema(S)) = try .init(.temp_in_memory); +// db.deinit(); // } diff --git a/src/db/sqlite_new.zig b/src/db/sqlite_new.zig index 46706b8..36eaf61 100644 --- a/src/db/sqlite_new.zig +++ b/src/db/sqlite_new.zig @@ -25,7 +25,7 @@ pub fn Database(comptime T: type) type { // TODO: make functions out of `table` and `columns`: Table(Schema), []const Column(Schema, table) for readability? pub fn select(self: *Self, comptime table: std.meta.Tag(Schema), comptime columns: []const std.meta.Tag(@FieldType(Schema, @tagName(table)))) Select(Schema, table, columns) { _ = self; - std.debug.print("{any}\n", .{ columns }); + std.debug.print("{any}\n", .{columns}); return .{ // .handle = self.handle, }; @@ -152,17 +152,17 @@ fn Unique(comptime T: type) type { const Type = T.Type; // TODO: `contains(expected, str)` function const string = if (std.mem.count(u8, T.string, "UNIQUE") > 0) - @compileError(std.fmt.comptimePrint("Type `{s}` is already unique", .{ @typeName(T) })) + @compileError(std.fmt.comptimePrint("Type `{s}` is already unique", .{@typeName(T)})) else - std.fmt.comptimePrint("UNIQUE {s}", .{ T.string }); + std.fmt.comptimePrint("UNIQUE {s}", .{T.string}); }; } fn Default(comptime T: type, comptime value: T.Type) type { return struct { const Type = T.type; - const string = if(std.mem.count(u8, T.string, "DEFAULT") > 0) - @compileError(std.fmt.comptimePrint("Type `{s}` already has a default value", .{ @typeName(T) })) + const string = if (std.mem.count(u8, T.string, "DEFAULT") > 0) + @compileError(std.fmt.comptimePrint("Type `{s}` already has a default value", .{@typeName(T)})) else std.fmt.comptimePrint("{s} DEFAULT {}", .{ T.string, value }); }; @@ -202,12 +202,12 @@ fn Test(comptime Schema: type, comptime table: std.meta.Tag(Schema), comptime co fn Eql(comptime column: std.meta.Tag(ColumnUnion), value: @FieldType(ColumnUnion, @tagName(column))) type { return struct { - const fmt: []const u8 = std.fmt.comptimePrint("{s} = ", .{ @tagName(column) }); - const args: @Tuple(&.{ @TypeOf(value) }) = .{ value }; + const fmt: []const u8 = std.fmt.comptimePrint("{s} = ", .{@tagName(column)}); + const args: @Tuple(&.{@TypeOf(value)}) = .{value}; }; } // fn And() type { - // todo: concat fmt slices and args tuples + // todo: concat fmt slices and args tuples // } string: Node, @@ -236,10 +236,10 @@ test "schema" { const Schema = struct { contacts: Contacts, // contacts: struct { - // contact_id: Integer = .constrain(.primary_key), - // age: Integer = .unconstrained, - // contact_id: Integer(usize) = .constrain(.{ .primary_key = true }), - // first_name: Default(VarChar(255), "Hans"), + // contact_id: Integer = .constrain(.primary_key), + // age: Integer = .unconstrained, + // contact_id: Integer(usize) = .constrain(.{ .primary_key = true }), + // first_name: Default(VarChar(255), "Hans"), // }, }; @@ -254,10 +254,10 @@ test "schema" { const t: Test(@TypeOf(db).Schema, .contacts, &.{ .id, .first_name }) = .eql(.id, 42); // const t: Test(@TypeOf(db).Schema, .contacts, &.{ .id, .first_name }) = .op_and(.eql(.id, 42), .eql(.first_name, "name")); - std.debug.print("test output: {s}{}\n", switch (t.string) { .eql => |eq| .{ eq.fmt, eq.args } }); + std.debug.print("test output: {s}{}\n", switch (t.string) { + .eql => |eq| .{ eq.fmt, eq.args }, + }); const Int = Default(Unique(Integer(usize)), 42); - std.debug.print("default unique integer with default 42: {s}\n", .{ Int.string }); - - + std.debug.print("default unique integer with default 42: {s}\n", .{Int.string}); } diff --git a/src/html/Element.zig b/src/html/Element.zig index 8ecf31e..a9e5897 100644 --- a/src/html/Element.zig +++ b/src/html/Element.zig @@ -39,7 +39,7 @@ fn attributesFrom(attributes: anytype) []const Attribute { const name = field.name; const value = @field(attributes, name); - const next_attr: []const Attribute = &.{ .{ .name = name, .value = value } }; + const next_attr: []const Attribute = &.{.{ .name = name, .value = value }}; attrs = attrs ++ next_attr; } @@ -79,7 +79,6 @@ pub fn toHtml(comptime self: Self) []const u8 { break :string string; }; - switch (self.content) { .void => {}, .string => |s| { @@ -91,7 +90,7 @@ pub fn toHtml(comptime self: Self) []const u8 { } string = string ++ ""; - } + }, } string = string ++ if (self.tag.len == 0) "" else ""; diff --git a/src/main.zig b/src/main.zig index a429d14..176d3b2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -12,11 +12,7 @@ const web = @import("web/root.zig"); const routes: http.RouteSet = .init(api.interfaces ++ web.interfaces); pub fn main(init: std.process.Init) !void { - try http.serve( - init.gpa, - routes, - .{ - .address = try .parseLiteral("0.0.0.0:8080"), - } - ); + try http.serve(init.gpa, routes, .{ + .address = try .parseLiteral("0.0.0.0:8080"), + }); } diff --git a/src/web/root.zig b/src/web/root.zig index c9c1b08..d027dc8 100644 --- a/src/web/root.zig +++ b/src/web/root.zig @@ -24,4 +24,3 @@ const document: html.Element = .native(.html, .{}, .children(&.{ pub const interfaces: []const http.handler.Interface = &.{ Static(.index, document.toDocument()).interface(), }; - diff --git a/src/z/File.zig b/src/z/File.zig index cd7094f..cd19dba 100644 --- a/src/z/File.zig +++ b/src/z/File.zig @@ -15,4 +15,3 @@ pub fn load(comptime self: @This(), comptime content: []const u8) Element { return .transparent(.content(comptime self.minifier.minify(content))); } } - diff --git a/src/z/parser/builtin.zig b/src/z/parser/builtin.zig deleted file mode 100644 index fc96fc3..0000000 --- a/src/z/parser/builtin.zig +++ /dev/null @@ -1,17 +0,0 @@ -const std = @import("std"); - -const parser = @import("root.zig"); -const Parser = parser.Parser; -const Result = parser.Result; - -pub fn literal(lit: []const u8) Parser(void) { - return .fromType(struct { - pub fn parse(buffer: []const u8) !Result(void) { - if (std.mem.startsWith(u8, buffer, lit)) { - return .{ void{}, buffer[lit.len..] }; - } - - return error.Literal; - } - }); -} diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig new file mode 100644 index 0000000..038dbc5 --- /dev/null +++ b/src/z/parser/js/grammar/identifier.zig @@ -0,0 +1,4 @@ +pub const Identifier = enum { + public, + private, +}; diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig new file mode 100644 index 0000000..14c3272 --- /dev/null +++ b/src/z/parser/js/grammar/keyword.zig @@ -0,0 +1,48 @@ +pub const Keyword = enum { + @"break", + case, + @"catch", + class, + @"const", + @"continue", + debugger, + default, + delete, + do, + @"else", + @"export", + extends, + finally, + @"for", + function, + @"if", + import, + in, + instanceof, + new, + @"return", + super, + @"switch", + this, + throw, + @"try", + typeof, + @"var", + void, + @"while", + with, + yield, + let, + static, + await, + async, + of, + from, + get, + set, + meta, + @"enum", + null, + true, + false, +}; diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..a2ef97a --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,5 @@ +pub const Literal = enum { + numeric, + bigint, + string, +}; diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig new file mode 100644 index 0000000..81b2e6e --- /dev/null +++ b/src/z/parser/js/grammar/punctuator.zig @@ -0,0 +1,61 @@ +pub const Punctuator = enum { + @"+", + @"-", + @"*", + @"/", + @"%", + @"**", + @"++", + @"--", + @"=", + @"+=", + @"-=", + @"*=", + @"/=", + @"%=", + @"**=", + @"&=", + @"|=", + @"^=", + @"<<=", + @">>=", + @">>>=", + @"&&=", + @"||=", + @"??=", + @"==", + @"===", + @"!=", + @"!==", + @"<", + @"<=", + @">", + @">=", + @"&&", + @"||", + @"??", + @"&", + @"|", + @"^", + @"~", + @"<<", + @">>", + @">>>", + @"!", + @"?", + @":", + @";", + @",", + @".", + @"...", + @"=>", + @"(", + @")", + @"[", + @"]", + @"{", + @"}", + @"?.", + @"#", + @"@", +}; diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig new file mode 100644 index 0000000..e1a1e96 --- /dev/null +++ b/src/z/parser/js/grammar/root.zig @@ -0,0 +1,36 @@ +const std = @import("std"); + +const parser = @import("../../root.zig"); +pub const Lexer = parser.Lexer(Grammar); +pub const Token = parser.Token(Grammar); + +pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Literal = @import("literal.zig").Literal; +pub const Punctuator = @import("punctuator.zig").Punctuator; +pub const Identifier = @import("identifier.zig").Identifier; + +pub const Grammar = union(enum) { + whitespace: Whitespace, + literal: Literal, + punctuator: Punctuator, + identifier: Identifier, + end_of_file: void, + + pub fn tokenize(lexer: *Lexer) Lexer.Error!void { + while (lexer.peakChar()) |_| { + Whitespace.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; + }; + } + + lexer.push(lexer.token(0, .end_of_file)); + } +}; + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..c3435c4 --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,106 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(undefined); + + switch (lexer.peekChar() orelse 0) { + ' ', '\t', '\r', '\n' => { + token.kind = .{ .whitespace = .space }; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekCharAt(1) orelse 0) { + '*' => { + token.kind = .{ .whitespace = .block_comment }; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind = .{ .whitespace = .line_comment }; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); + + inline for (lexer.tokens) |token| { + std.debug.print("{f}\n", .{token}); + } +} diff --git a/src/z/parser/js/lexical_grammar.zig b/src/z/parser/js/lexical_grammar.zig deleted file mode 100644 index 7060766..0000000 --- a/src/z/parser/js/lexical_grammar.zig +++ /dev/null @@ -1,3 +0,0 @@ -const parser = @import("root.zig"); -const Result = parser.Result; - diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig index a6be9ca..e604855 100644 --- a/src/z/parser/js/root.zig +++ b/src/z/parser/js/root.zig @@ -1,10 +1,6 @@ const std = @import("std"); -pub const lexical_grammar = @import("lexical_grammar.zig"); - -pub const Result = @import("result.zig").Result; -pub const Parser = @import("parser.zig").Parser; -pub const builtin = @import("builtin.zig"); +pub const grammar = @import("./grammar/root.zig"); test { _ = std.testing.refAllDecls(@This()); diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig new file mode 100644 index 0000000..d9a2868 --- /dev/null +++ b/src/z/parser/lexer.zig @@ -0,0 +1,110 @@ +const std = @import("std"); + +pub fn Token(TokenKind: type) type { + return struct { + const Self = @This(); + + kind: TokenKind, + slice: []const u8, + + pub fn start(comptime lexer: *const Lexer(TokenKind), kind: TokenKind) Self { + return .{ + .kind = kind, + .slice = lexer.buffer, + }; + } + + pub inline fn stop(comptime self: Self, comptime lexer: *const Lexer(TokenKind)) Self { + comptime { + const length = self.slice.len - lexer.buffer.len; + return .{ + .kind = self.kind, + .slice = self.slice[0..length], + }; + } + } + + pub fn format(self: *const Self, writer: *std.Io.Writer) !void { + try writer.print("Token{}(\"{s}\")", .{self.kind, self.slice}); + } + }; +} + +pub fn Lexer(TokenKind: type) type { + return struct { + const Self = @This(); + + pub const Error = error{ + UnexpectedToken, + EndOfBuffer, + }; + + buffer: []const u8, + last_buffer: []const u8, + tokens: []const Token(TokenKind) = &.{}, + + pub fn init(buffer: []const u8) Self { + return .{ + .buffer = buffer, + .last_buffer = buffer, + }; + } + + pub inline fn peekChar(self: *Self) ?u8 { + return self.peekCharAt(0); + } + + pub inline fn peekCharAt(self: *Self, n: usize) ?u8 { + return if (n < self.buffer.len) self.buffer[n] else null; + } + + pub fn peekSlice(self: *Self, length: usize) ?[]const u8 { + return self.peekSliceAt(length, 0); + } + + pub fn peekSliceAt(self: *Self, length: usize, offset: usize) ?[]const u8 { + if (length + offset <= self.buffer.len) { + return self.buffer[offset .. offset + length]; + } + return null; + } + + pub fn skip(self: *Self) !void { + return self.skipTo(1); + } + + pub fn skipTo(self: *Self, n: usize) Error!void { + if (n <= self.buffer.len) { + self.buffer = self.buffer[n..]; + } else { + return Error.EndOfBuffer; + } + } + + pub fn skipUntil(self: *Self, literal: []const u8) Error!void { + if (std.mem.find(u8, self.buffer, literal)) |index| { + self.buffer = self.buffer[index + literal.len..]; + } else { + return Error.EndOfBuffer; + } + } + + pub fn skipToEnd(self: *Self) void { + self.buffer = self.buffer[self.buffer.len..]; + } + + pub fn start(self: *Self, kind: TokenKind) Token(TokenKind) { + return .start(self, kind); + } + + pub fn revert(self: *Self) void { + self.buffer = self.last_buffer; + } + + pub fn commit(self: *Self, tok: Token(TokenKind)) void { + self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; + self.last_buffer = self.buffer; + } + + }; +} diff --git a/src/z/parser/parser.zig b/src/z/parser/parser.zig deleted file mode 100644 index b73d895..0000000 --- a/src/z/parser/parser.zig +++ /dev/null @@ -1,117 +0,0 @@ -const std = @import("std"); - -const mod = @import("root.zig"); -const Result = mod.Result; - -pub fn Parser(T: type) type { - const R = Result(T); - - return struct { - parse: *const fn ([]const u8) anyerror!R, - - pub fn parseAll(self: *const @This(), buffer: []const u8) !T { - const value, const next = try self.parse(buffer); - - if (next.len > 0) { - return error.NotExhaustive; - } - - return value; - } - - pub fn fromType(P: type) @This() { - return .{ .parse = P.parse }; - } - - pub fn variants(V: type) @This() { - if (@typeInfo(T) != .@"union") { - @compileError("variants is only allowed with an union as result type"); - } - - return .{ - .parse = struct { - fn parse(buffer: []const u8) !R { - inline for (std.meta.fields(T)) |field| { - if (@field(V, field.name).parse(buffer)) |result| { - const value, const next = result; - return .{ - @unionInit(T, field.name, value), - next, - }; - } else |_| { } - } - - return error.Variants; - } - }.parse, - }; - } - - pub fn any(P: type, comptime parsers: []const struct { Parser(P), T }) @This() { - return .{ - .parse = struct { - fn parse(buffer: []const u8) !R { - inline for (parsers) |pair| { - const parser, const value = pair; - - if (parser.parse(buffer)) |result| { - _, const next = result; - return .{ value, next }; - } else |_| { } - } - return error.Any; - } - }.parse, - }; - } - }; -} - -test "variants" { - const Char = union(enum) { - upper: u8, - lower: u8, - space: void, - }; - - const parser: Parser(Char) = .variants(struct { - const upper: Parser(u8) = .any(void, &.{ - .{ mod.builtin.literal("A"), 'A' }, - .{ mod.builtin.literal("B"), 'B' }, - }); - - const lower: Parser(u8) = .any(void, &.{ - .{ mod.builtin.literal("a"), 'a' }, - .{ mod.builtin.literal("b"), 'b' }, - }); - - const space: Parser(void) = mod.builtin.literal(" "); - }); - - { - const result = try parser.parseAll("a"); - try std.testing.expect(result.lower == 'a'); - } - - { - const result = try parser.parseAll("b"); - try std.testing.expect(result.lower == 'b'); - } - - { - const result = try parser.parseAll("A"); - try std.testing.expect(result.upper == 'A'); - } - - { - const result = try parser.parseAll("B"); - try std.testing.expect(result.upper == 'B'); - } - - { - const result = try parser.parseAll(" "); - try std.testing.expect(result == .space); - } - - try std.testing.expect(parser.parseAll("c") == error.Variants); -} diff --git a/src/z/parser/result.zig b/src/z/parser/result.zig deleted file mode 100644 index 5cc5878..0000000 --- a/src/z/parser/result.zig +++ /dev/null @@ -1,4 +0,0 @@ - -pub fn Result(T: type) type { - return struct { T, []const u8 }; -} diff --git a/src/z/parser/root.zig b/src/z/parser/root.zig new file mode 100644 index 0000000..e0b67fb --- /dev/null +++ b/src/z/parser/root.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +const lexer = @import("lexer.zig"); +pub const Lexer = lexer.Lexer; +pub const Token = lexer.Token; + +pub const js = @import("js/root.zig"); + +pub fn Parser(TokenType: type) type { + return struct { + buffer: []const TokenType, + }; +} + +test { + _ = std.testing.refAllDecls(@This()); +} diff --git a/src/z/root.zig b/src/z/root.zig index b23ebc5..1fa8d99 100644 --- a/src/z/root.zig +++ b/src/z/root.zig @@ -3,6 +3,7 @@ const std = @import("std"); pub const Minifier = @import("Minifier.zig"); pub const Component = @import("Component.zig"); pub const File = @import("File.zig"); +pub const parser = @import("parser/root.zig"); const Element = @import("html").Element; -- cgit v1.2.3-70-g09d2