add tokenizer for whitespaces in js

23 files changed, 461 insertions, 226 deletions

diff --git a/src/api/hello-json.zig b/src/api/hello-json.zig
index 6c29ba7..12f1e72 100644
--- a/src/api/hello-json.zig
+++ b/src/api/hello-json.zig
@@ -2,7 +2,7 @@ const http = @import("http");
 const Context = http.handler.Context;
 const Response = http.Response;
 
-const Ctx = Context(.{ "hello-json" });
+const Ctx = Context(.{"hello-json"});
 pub const ctx: Ctx = .{
     .get = get,
 };
diff --git a/src/api/hello-param.zig b/src/api/hello-param.zig
index f4de1dd..b67212b 100644
--- a/src/api/hello-param.zig
+++ b/src/api/hello-param.zig
@@ -10,5 +10,7 @@ pub const ctx: Ctx = .{
 fn get(request: Ctx.Request) Response {
     const value = request.params.value.as(usize) catch return .badRequest(.string("parameter needs to be an int"));
 
-    return .okOrError(.json(request.arena, .{ .value = value, }));
+    return .okOrError(.json(request.arena, .{
+        .value = value,
+    }));
 }
diff --git a/src/api/root.zig b/src/api/root.zig
index 7570a76..2a54b91 100644
--- a/src/api/root.zig
+++ b/src/api/root.zig
@@ -16,7 +16,7 @@ pub const interfaces: []const Interface = ifs: {
     var ifs: []const Interface = &.{};
 
     for (modules) |module| {
-        const interface: []const Interface = &.{ module.ctx.interface() };
+        const interface: []const Interface = &.{module.ctx.interface()};
         ifs = ifs ++ interface;
     }
 
diff --git a/src/db/sqlite.zig b/src/db/sqlite.zig
index 28d6487..d1accd5 100644
--- a/src/db/sqlite.zig
+++ b/src/db/sqlite.zig
@@ -5,7 +5,7 @@ const assert = std.debug.assert;
 pub fn Database(comptime T: type) type {
     const tables = switch (@typeInfo(T)) {
         .@"struct" => |s| s.fields,
-        else       => @compileError("Database schema must be of type `struct`"),
+        else => @compileError("Database schema must be of type `struct`"),
     };
 
     // NOTE: `std.math.log2_int_ceil(@TypeOf(tables.len), tables.len)` panics
@@ -13,14 +13,14 @@ pub fn Database(comptime T: type) type {
     //       a (useless) schema that does not define any tables.
     const TagIntTables = @Int(.unsigned, @bitSizeOf(@TypeOf(tables.len)) - @clz(tables.len));
 
-    var table_names : [tables.len][]const u8   = undefined;
-    var table_types : [tables.len]type         = undefined;
+    var table_names: [tables.len][]const u8 = undefined;
+    var table_types: [tables.len]type = undefined;
     var table_values: [tables.len]TagIntTables = undefined;
 
     inline for (&table_names, &table_types, &table_values, tables, 0..) |*table_name, *table_type, *table_value, table, table_index| {
         const columns = switch (@typeInfo(table.type)) {
             .@"struct" => |s| s.fields,
-            else       => @compileError("Table `" ++ table.name ++ "` must be of type `struct`"),
+            else => @compileError("Table `" ++ table.name ++ "` must be of type `struct`"),
         };
 
         // NOTE: A table must have at least one column
@@ -29,28 +29,28 @@ pub fn Database(comptime T: type) type {
 
         const TagIntColumns = @Int(.unsigned, std.math.log2_int_ceil(@TypeOf(columns.len), columns.len));
 
-        var column_names : [columns.len][]const u8    = undefined;
-        var column_types : [columns.len]type          = undefined;
+        var column_names: [columns.len][]const u8 = undefined;
+        var column_types: [columns.len]type = undefined;
         var column_values: [columns.len]TagIntColumns = undefined;
 
         inline for (&column_names, &column_types, &column_values, columns, 0..) |*column_name, *column_type, *column_value, column, column_index| {
             // TODO: check column.type (of type `Type`)
 
-            column_name.*  = column.name;
-            column_type.*  = column.type;
+            column_name.* = column.name;
+            column_type.* = column.type;
             column_value.* = column_index;
         }
 
         const TagColumns = @Enum(TagIntColumns, .exhaustive, &column_names, &column_values);
-        const Columns    = @Union(.auto, TagColumns, &column_names, &column_types, &@splat(.{}));
+        const Columns = @Union(.auto, TagColumns, &column_names, &column_types, &@splat(.{}));
 
-        table_name.*  = table.name;
-        table_type.*  = Columns;
+        table_name.* = table.name;
+        table_type.* = Columns;
         table_value.* = table_index;
     }
 
     const TagTables = @Enum(TagIntTables, .exhaustive, &table_names, &table_values);
-    const Tables    = @Union(.auto, TagTables, &table_names, &table_types, &@splat(.{}));
+    const Tables = @Union(.auto, TagTables, &table_names, &table_types, &@splat(.{}));
 
     return struct {
         inner: Inner,
@@ -80,7 +80,7 @@ pub fn Database(comptime T: type) type {
         };
 
         const Schema = Tables;
-        const Self   = @This();
+        const Self = @This();
 
         const Inner = struct {
             handle: *sqlite.sqlite3,
@@ -89,14 +89,14 @@ pub fn Database(comptime T: type) type {
         pub fn init(kind: Kind) Error!Self {
             const name = switch (kind) {
                 .temp_in_memory => ":memory:",
-                .temp_on_disk   => "",
-                .path           => |path| path,
+                .temp_on_disk => "",
+                .path => |path| path,
             };
             var handle: ?*sqlite.sqlite3 = null;
 
             const result: Result = .from(sqlite.sqlite3_open(name.ptr, &handle));
             const primary = result.asPrimary();
-            
+
             return if (primary == .ok) .{ .handle = handle.? } else result.toError(Error, handle);
         }
 
@@ -132,8 +132,6 @@ fn Select(comptime Db: type, comptime table: std.meta.Tag(Db.Schema), comptime c
     };
 }
 
-
-
 // pub const Type = struct {
 //     string: String.Static,
 //     type: type,
@@ -266,7 +264,6 @@ const String = struct {
 //     }
 // };
 
-
 /// SQLite result codes.
 ///
 /// Primary result codes define a broad category of results, whereas extended
@@ -445,14 +442,14 @@ test {
         const Contacts = struct {
             contact_id: usize, // CHANGEME: to `Type`
 
-            const all = &.{ .contact_id };
+            const all = &.{.contact_id};
         };
     };
 
-    var   db: Database(Schema) = try .init(.temp_in_memory);
+    var db: Database(Schema) = try .init(.temp_in_memory);
     defer assert(db.deinit() == .ok);
 
-    _ = db.select(.contacts, &.{ .contact_id }, .all);
+    _ = db.select(.contacts, &.{.contact_id}, .all);
     var ty = db.select(.contacts, Schema.Contacts.all, .all);
     ty.table = .foo;
 
@@ -462,7 +459,7 @@ test {
     // db.select(.distinct(...)).from(...)...
     //
     // // If the simple SELECT is a SELECT ALL, then the entire set of result rows are returned by the SELECT. If neither ALL or DISTINCT are present, then the behavior is as if ALL were specified.
-    // db.select(enum { all, distinct }, 
+    // db.select(enum { all, distinct },
     //     * == .all(),
     //     table-name.* == .table(.table_name),
     //
@@ -470,30 +467,30 @@ test {
 }
 
 // test {
-    // NOTE: we don't use `union(enum)` here as if a column has a default value,
-    //       an integer tag type is needed explicitely, which, from a usability
-    //       standpoint, is not great; we can calulate this value in `Schema.from`.
-    //
-    // CREATE TABLE contacts (
-    //     contact_id INTEGER PRIMARY KEY,
-    //     first_name TEXT NOT NULL DEFAULT "Hans",
-    //     last_name  TEXT NOT NULL DEFAULT "Mustermann",
-    //     nickname   TEXT,
-    //     email      TEXT NOT NULL UNIQUE,
-    //     phone      TEXT NOT NULL UNIQUE
-    // );
-    // const S = struct {
-    //     contacts: struct {
-    //         contact_id: PrimaryKey(Integer, .{}),
-    //         first_name: VarChar(255, .{}) = "Hans",
-    //         last_name: VarChar(255, .{}) = "Mustermann",
-    //         nickname: ?VarChar(255, .{}),
-    //         email: VarChar(255, .{.unique}),
-    //         phone: VarChar(255, .{.unique}),
-    //     },
-    // };
-    //
-    // // NOTE: init connects to DB and creates tables if not already present
-    // var db: Database(Schema(S)) = try .init(.temp_in_memory);
-    // db.deinit();
+// NOTE: we don't use `union(enum)` here as if a column has a default value,
+//       an integer tag type is needed explicitely, which, from a usability
+//       standpoint, is not great; we can calulate this value in `Schema.from`.
+//
+// CREATE TABLE contacts (
+//     contact_id INTEGER PRIMARY KEY,
+//     first_name TEXT NOT NULL DEFAULT "Hans",
+//     last_name  TEXT NOT NULL DEFAULT "Mustermann",
+//     nickname   TEXT,
+//     email      TEXT NOT NULL UNIQUE,
+//     phone      TEXT NOT NULL UNIQUE
+// );
+// const S = struct {
+//     contacts: struct {
+//         contact_id: PrimaryKey(Integer, .{}),
+//         first_name: VarChar(255, .{}) = "Hans",
+//         last_name: VarChar(255, .{}) = "Mustermann",
+//         nickname: ?VarChar(255, .{}),
+//         email: VarChar(255, .{.unique}),
+//         phone: VarChar(255, .{.unique}),
+//     },
+// };
+//
+// // NOTE: init connects to DB and creates tables if not already present
+// var db: Database(Schema(S)) = try .init(.temp_in_memory);
+// db.deinit();
 // }
diff --git a/src/db/sqlite_new.zig b/src/db/sqlite_new.zig
index 46706b8..36eaf61 100644
--- a/src/db/sqlite_new.zig
+++ b/src/db/sqlite_new.zig
@@ -25,7 +25,7 @@ pub fn Database(comptime T: type) type {
         // TODO: make functions out of `table` and `columns`: Table(Schema), []const Column(Schema, table) for readability?
         pub fn select(self: *Self, comptime table: std.meta.Tag(Schema), comptime columns: []const std.meta.Tag(@FieldType(Schema, @tagName(table)))) Select(Schema, table, columns) {
             _ = self;
-            std.debug.print("{any}\n", .{ columns });
+            std.debug.print("{any}\n", .{columns});
             return .{
                 // .handle = self.handle,
             };
@@ -152,17 +152,17 @@ fn Unique(comptime T: type) type {
         const Type = T.Type;
         // TODO: `contains(expected, str)` function
         const string = if (std.mem.count(u8, T.string, "UNIQUE") > 0)
-            @compileError(std.fmt.comptimePrint("Type `{s}` is already unique", .{ @typeName(T) }))
+            @compileError(std.fmt.comptimePrint("Type `{s}` is already unique", .{@typeName(T)}))
         else
-            std.fmt.comptimePrint("UNIQUE {s}", .{ T.string });
+            std.fmt.comptimePrint("UNIQUE {s}", .{T.string});
     };
 }
 
 fn Default(comptime T: type, comptime value: T.Type) type {
     return struct {
         const Type = T.type;
-        const string = if(std.mem.count(u8, T.string, "DEFAULT") > 0)
-            @compileError(std.fmt.comptimePrint("Type `{s}` already has a default value", .{ @typeName(T) }))
+        const string = if (std.mem.count(u8, T.string, "DEFAULT") > 0)
+            @compileError(std.fmt.comptimePrint("Type `{s}` already has a default value", .{@typeName(T)}))
         else
             std.fmt.comptimePrint("{s} DEFAULT {}", .{ T.string, value });
     };
@@ -202,12 +202,12 @@ fn Test(comptime Schema: type, comptime table: std.meta.Tag(Schema), comptime co
 
         fn Eql(comptime column: std.meta.Tag(ColumnUnion), value: @FieldType(ColumnUnion, @tagName(column))) type {
             return struct {
-                const fmt: []const u8 = std.fmt.comptimePrint("{s} = ", .{ @tagName(column) });
-                const args: @Tuple(&.{ @TypeOf(value) }) = .{ value };
+                const fmt: []const u8 = std.fmt.comptimePrint("{s} = ", .{@tagName(column)});
+                const args: @Tuple(&.{@TypeOf(value)}) = .{value};
             };
         }
         // fn And() type {
-            // todo: concat fmt slices and args tuples
+        // todo: concat fmt slices and args tuples
         // }
 
         string: Node,
@@ -236,10 +236,10 @@ test "schema" {
     const Schema = struct {
         contacts: Contacts,
         // contacts: struct {
-            // contact_id: Integer = .constrain(.primary_key),
-            // age: Integer = .unconstrained,
-            // contact_id: Integer(usize) = .constrain(.{ .primary_key = true }),
-            // first_name: Default(VarChar(255), "Hans"),
+        // contact_id: Integer = .constrain(.primary_key),
+        // age: Integer = .unconstrained,
+        // contact_id: Integer(usize) = .constrain(.{ .primary_key = true }),
+        // first_name: Default(VarChar(255), "Hans"),
         // },
     };
 
@@ -254,10 +254,10 @@ test "schema" {
 
     const t: Test(@TypeOf(db).Schema, .contacts, &.{ .id, .first_name }) = .eql(.id, 42);
     // const t: Test(@TypeOf(db).Schema, .contacts, &.{ .id, .first_name }) = .op_and(.eql(.id, 42), .eql(.first_name, "name"));
-    std.debug.print("test output: {s}{}\n", switch (t.string) { .eql => |eq| .{ eq.fmt, eq.args } });
+    std.debug.print("test output: {s}{}\n", switch (t.string) {
+        .eql => |eq| .{ eq.fmt, eq.args },
+    });
 
     const Int = Default(Unique(Integer(usize)), 42);
-    std.debug.print("default unique integer with default 42: {s}\n", .{ Int.string });
-
-
+    std.debug.print("default unique integer with default 42: {s}\n", .{Int.string});
 }
diff --git a/src/html/Element.zig b/src/html/Element.zig
index 8ecf31e..a9e5897 100644
--- a/src/html/Element.zig
+++ b/src/html/Element.zig
@@ -39,7 +39,7 @@ fn attributesFrom(attributes: anytype) []const Attribute {
         const name = field.name;
         const value = @field(attributes, name);
 
-        const next_attr: []const Attribute = &.{ .{ .name = name, .value = value } };
+        const next_attr: []const Attribute = &.{.{ .name = name, .value = value }};
         attrs = attrs ++ next_attr;
     }
 
@@ -79,7 +79,6 @@ pub fn toHtml(comptime self: Self) []const u8 {
         break :string string;
     };
 
-
     switch (self.content) {
         .void => {},
         .string => |s| {
@@ -91,7 +90,7 @@ pub fn toHtml(comptime self: Self) []const u8 {
             }
 
             string = string ++ "</" ++ self.tag ++ ">";
-        }
+        },
     }
 
     string = string ++ if (self.tag.len == 0) "" else "</" ++ self.tag ++ ">";
diff --git a/src/main.zig b/src/main.zig
index a429d14..176d3b2 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -12,11 +12,7 @@ const web = @import("web/root.zig");
 const routes: http.RouteSet = .init(api.interfaces ++ web.interfaces);
 
 pub fn main(init: std.process.Init) !void {
-    try http.serve(
-        init.gpa,
-        routes,
-        .{
-            .address = try .parseLiteral("0.0.0.0:8080"),
-        }
-    );
+    try http.serve(init.gpa, routes, .{
+        .address = try .parseLiteral("0.0.0.0:8080"),
+    });
 }
diff --git a/src/web/root.zig b/src/web/root.zig
index c9c1b08..d027dc8 100644
--- a/src/web/root.zig
+++ b/src/web/root.zig
@@ -24,4 +24,3 @@ const document: html.Element = .native(.html, .{}, .children(&.{
 pub const interfaces: []const http.handler.Interface = &.{
     Static(.index, document.toDocument()).interface(),
 };
-
diff --git a/src/z/File.zig b/src/z/File.zig
index cd7094f..cd19dba 100644
--- a/src/z/File.zig
+++ b/src/z/File.zig
@@ -15,4 +15,3 @@ pub fn load(comptime self: @This(), comptime content: []const u8) Element {
         return .transparent(.content(comptime self.minifier.minify(content)));
     }
 }
-
diff --git a/src/z/parser/builtin.zig b/src/z/parser/builtin.zig
deleted file mode 100644
index fc96fc3..0000000
--- a/src/z/parser/builtin.zig
+++ /dev/null
@@ -1,17 +0,0 @@
-const std = @import("std");
-
-const parser = @import("root.zig");
-const Parser = parser.Parser;
-const Result = parser.Result;
-
-pub fn literal(lit: []const u8) Parser(void) {
-    return .fromType(struct {
-        pub fn parse(buffer: []const u8) !Result(void) {
-            if (std.mem.startsWith(u8, buffer, lit)) {
-                return .{ void{}, buffer[lit.len..] };
-            }
-
-            return error.Literal;
-        }
-    });
-}
diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig
new file mode 100644
index 0000000..038dbc5
--- /dev/null
+++ b/src/z/parser/js/grammar/identifier.zig
@@ -0,0 +1,4 @@
+pub const Identifier = enum {
+    public,
+    private,
+};
diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig
new file mode 100644
index 0000000..14c3272
--- /dev/null
+++ b/src/z/parser/js/grammar/keyword.zig
@@ -0,0 +1,48 @@
+pub const Keyword = enum {
+    @"break",
+    case,
+    @"catch",
+    class,
+    @"const",
+    @"continue",
+    debugger,
+    default,
+    delete,
+    do,
+    @"else",
+    @"export",
+    extends,
+    finally,
+    @"for",
+    function,
+    @"if",
+    import,
+    in,
+    instanceof,
+    new,
+    @"return",
+    super,
+    @"switch",
+    this,
+    throw,
+    @"try",
+    typeof,
+    @"var",
+    void,
+    @"while",
+    with,
+    yield,
+    let,
+    static,
+    await,
+    async,
+    of,
+    from,
+    get,
+    set,
+    meta,
+    @"enum",
+    null,
+    true,
+    false,
+};
diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
new file mode 100644
index 0000000..a2ef97a
--- /dev/null
+++ b/src/z/parser/js/grammar/literal.zig
@@ -0,0 +1,5 @@
+pub const Literal = enum {
+    numeric,
+    bigint,
+    string,
+};
diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig
new file mode 100644
index 0000000..81b2e6e
--- /dev/null
+++ b/src/z/parser/js/grammar/punctuator.zig
@@ -0,0 +1,61 @@
+pub const Punctuator = enum {
+    @"+",
+    @"-",
+    @"*",
+    @"/",
+    @"%",
+    @"**",
+    @"++",
+    @"--",
+    @"=",
+    @"+=",
+    @"-=",
+    @"*=",
+    @"/=",
+    @"%=",
+    @"**=",
+    @"&=",
+    @"|=",
+    @"^=",
+    @"<<=",
+    @">>=",
+    @">>>=",
+    @"&&=",
+    @"||=",
+    @"??=",
+    @"==",
+    @"===",
+    @"!=",
+    @"!==",
+    @"<",
+    @"<=",
+    @">",
+    @">=",
+    @"&&",
+    @"||",
+    @"??",
+    @"&",
+    @"|",
+    @"^",
+    @"~",
+    @"<<",
+    @">>",
+    @">>>",
+    @"!",
+    @"?",
+    @":",
+    @";",
+    @",",
+    @".",
+    @"...",
+    @"=>",
+    @"(",
+    @")",
+    @"[",
+    @"]",
+    @"{",
+    @"}",
+    @"?.",
+    @"#",
+    @"@",
+};
diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
new file mode 100644
index 0000000..e1a1e96
--- /dev/null
+++ b/src/z/parser/js/grammar/root.zig
@@ -0,0 +1,36 @@
+const std = @import("std");
+
+const parser = @import("../../root.zig");
+pub const Lexer = parser.Lexer(Grammar);
+pub const Token = parser.Token(Grammar);
+
+pub const Whitespace = @import("whitespace.zig").Whitespace;
+pub const Literal = @import("literal.zig").Literal;
+pub const Punctuator = @import("punctuator.zig").Punctuator;
+pub const Identifier = @import("identifier.zig").Identifier;
+
+pub const Grammar = union(enum) {
+    whitespace: Whitespace,
+    literal: Literal,
+    punctuator: Punctuator,
+    identifier: Identifier,
+    end_of_file: void,
+
+    pub fn tokenize(lexer: *Lexer) Lexer.Error!void {
+        while (lexer.peakChar()) |_| {
+            Whitespace.tokenize(lexer) catch {
+                Literal.tokenize(lexer) catch {
+                    Punctuator.tokenize(lexer) catch {
+                        try Identifier.tokenize(lexer);
+                    };
+                };
+            };
+        }
+
+        lexer.push(lexer.token(0, .end_of_file));
+    }
+};
+
+test {
+    _ = std.testing.refAllDecls(@This());
+}
diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig
new file mode 100644
index 0000000..c3435c4
--- /dev/null
+++ b/src/z/parser/js/grammar/whitespace.zig
@@ -0,0 +1,106 @@
+const std = @import("std");
+const Lexer = @import("root.zig").Lexer;
+const Token = @import("root.zig").Token;
+
+pub const Whitespace = enum {
+    space,
+    line_comment,
+    block_comment,
+
+    pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+        comptime {
+            errdefer lexer.revert();
+
+            var token = lexer.start(undefined);
+
+            switch (lexer.peekChar() orelse 0) {
+                ' ', '\t', '\r', '\n' => {
+                    token.kind = .{ .whitespace = .space };
+
+                    while (lexer.peekChar()) |ch| {
+                        switch (ch) {
+                            ' ', '\t', '\r', '\n' => try lexer.skip(),
+                            else => break,
+                        }
+                    }
+                },
+                '/' => {
+                    switch (lexer.peekCharAt(1) orelse 0) {
+                        '*' => {
+                            token.kind = .{ .whitespace = .block_comment };
+                            try lexer.skipUntil("*/");
+                        },
+                        '/' => {
+                            token.kind = .{ .whitespace = .line_comment };
+                            lexer.skipUntil("\n") catch lexer.skipToEnd();
+                        },
+                        else => return Lexer.Error.UnexpectedToken,
+                    }
+                },
+                else => return Lexer.Error.UnexpectedToken,
+            }
+
+            lexer.commit(token);
+        }
+    }
+};
+
+test "space" {
+    {
+        comptime var lexer: Lexer = .init(" ");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(1, lexer.tokens[0].slice.len);
+    }
+
+    {
+        comptime var lexer: Lexer = .init(" \t\t   \n\r  ");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqual(10, lexer.tokens[0].slice.len);
+    }
+}
+
+test "line-comment" {
+    {
+        comptime var lexer: Lexer = .init("// some comment");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice);
+    }
+
+    {
+        comptime var lexer: Lexer = .init("// some comment\n");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice);
+    }
+}
+
+test "block-comment" {
+    {
+        comptime var lexer: Lexer = .init("/* some comment */");
+        try Whitespace.tokenize(&lexer);
+        try std.testing.expectEqual(1, lexer.tokens.len);
+        try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice);
+    }
+}
+
+test "mixed" {
+    comptime var lexer: Lexer = .init(
+        \\  // some comment
+        \\/* other comment here */     // comment or so
+        \\            /* multi
+        \\             * line
+        \\             * comment
+        \\             */
+    );
+    inline while (Whitespace.tokenize(&lexer)) {
+    } else |_| {}
+
+    try std.testing.expectEqual(7, lexer.tokens.len);
+
+    inline for (lexer.tokens) |token| {
+        std.debug.print("{f}\n", .{token});
+    }
+}
diff --git a/src/z/parser/js/lexical_grammar.zig b/src/z/parser/js/lexical_grammar.zig
deleted file mode 100644
index 7060766..0000000
--- a/src/z/parser/js/lexical_grammar.zig
+++ /dev/null
@@ -1,3 +0,0 @@
-const parser = @import("root.zig");
-const Result = parser.Result;
-
diff --git a/src/z/parser/js/root.zig b/src/z/parser/js/root.zig
index a6be9ca..e604855 100644
--- a/src/z/parser/js/root.zig
+++ b/src/z/parser/js/root.zig
@@ -1,10 +1,6 @@
 const std = @import("std");
 
-pub const lexical_grammar = @import("lexical_grammar.zig");
-
-pub const Result = @import("result.zig").Result;
-pub const Parser = @import("parser.zig").Parser;
-pub const builtin = @import("builtin.zig");
+pub const grammar = @import("./grammar/root.zig");
 
 test {
     _ = std.testing.refAllDecls(@This());
diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig
new file mode 100644
index 0000000..d9a2868
--- /dev/null
+++ b/src/z/parser/lexer.zig
@@ -0,0 +1,110 @@
+const std = @import("std");
+
+pub fn Token(TokenKind: type) type {
+    return struct {
+        const Self = @This();
+
+        kind: TokenKind,
+        slice: []const u8,
+
+        pub fn start(comptime lexer: *const Lexer(TokenKind), kind: TokenKind) Self {
+            return .{
+                .kind = kind,
+                .slice = lexer.buffer,
+            };
+        }
+
+        pub inline fn stop(comptime self: Self, comptime lexer: *const Lexer(TokenKind)) Self {
+            comptime {
+                const length = self.slice.len - lexer.buffer.len;
+                return .{
+                    .kind = self.kind,
+                    .slice = self.slice[0..length],
+                };
+            }
+        }
+
+        pub fn format(self: *const Self, writer: *std.Io.Writer) !void {
+            try writer.print("Token{}(\"{s}\")", .{self.kind, self.slice});
+        }
+    };
+}
+
+pub fn Lexer(TokenKind: type) type {
+    return struct {
+        const Self = @This();
+
+        pub const Error = error{
+            UnexpectedToken,
+            EndOfBuffer,
+        };
+
+        buffer: []const u8,
+        last_buffer: []const u8,
+        tokens: []const Token(TokenKind) = &.{},
+
+        pub fn init(buffer: []const u8) Self {
+            return .{
+                .buffer = buffer,
+                .last_buffer = buffer,
+            };
+        }
+
+        pub inline fn peekChar(self: *Self) ?u8 {
+            return self.peekCharAt(0);
+        }
+
+        pub inline fn peekCharAt(self: *Self, n: usize) ?u8 {
+            return if (n < self.buffer.len) self.buffer[n] else null;
+        }
+
+        pub fn peekSlice(self: *Self, length: usize) ?[]const u8 {
+            return self.peekSliceAt(length, 0);
+        }
+
+        pub fn peekSliceAt(self: *Self, length: usize, offset: usize) ?[]const u8 {
+            if (length + offset <= self.buffer.len) {
+                return self.buffer[offset .. offset + length];
+            }
+            return null;
+        }
+
+        pub fn skip(self: *Self) !void {
+            return self.skipTo(1);
+        }
+
+        pub fn skipTo(self: *Self, n: usize) Error!void {
+            if (n <= self.buffer.len) {
+                self.buffer = self.buffer[n..];
+            } else {
+                return Error.EndOfBuffer;
+            }
+        }
+
+        pub fn skipUntil(self: *Self, literal: []const u8) Error!void {
+            if (std.mem.find(u8, self.buffer, literal)) |index| {
+                self.buffer = self.buffer[index + literal.len..];
+            } else {
+                return Error.EndOfBuffer;
+            }
+        }
+
+        pub fn skipToEnd(self: *Self) void {
+            self.buffer = self.buffer[self.buffer.len..];
+        }
+
+        pub fn start(self: *Self, kind: TokenKind) Token(TokenKind) {
+            return .start(self, kind);
+        }
+
+        pub fn revert(self: *Self) void {
+            self.buffer = self.last_buffer;
+        }
+
+        pub fn commit(self: *Self, tok: Token(TokenKind)) void {
+            self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)};
+            self.last_buffer = self.buffer;
+        }
+
+    };
+}
diff --git a/src/z/parser/parser.zig b/src/z/parser/parser.zig
deleted file mode 100644
index b73d895..0000000
--- a/src/z/parser/parser.zig
+++ /dev/null
@@ -1,117 +0,0 @@
-const std = @import("std");
-
-const mod = @import("root.zig");
-const Result = mod.Result;
-
-pub fn Parser(T: type) type {
-    const R = Result(T);
-
-    return struct {
-        parse: *const fn ([]const u8) anyerror!R,
-
-        pub fn parseAll(self: *const @This(), buffer: []const u8) !T {
-            const value, const next = try self.parse(buffer);
-
-            if (next.len > 0) {
-                return error.NotExhaustive;
-            }
-
-            return value;
-        }
-
-        pub fn fromType(P: type) @This() {
-            return .{ .parse = P.parse };
-        }
-
-        pub fn variants(V: type) @This() {
-            if (@typeInfo(T) != .@"union") {
-                @compileError("variants is only allowed with an union as result type");
-            }
-
-            return .{
-                .parse = struct {
-                    fn parse(buffer: []const u8) !R {
-                        inline for (std.meta.fields(T)) |field| {
-                            if (@field(V, field.name).parse(buffer)) |result| {
-                                const value, const next = result;
-                                return .{
-                                    @unionInit(T, field.name, value),
-                                    next,
-                                };
-                            } else |_| { }
-                        }
-
-                        return error.Variants;
-                    }
-                }.parse,
-            };
-        }
-
-        pub fn any(P: type, comptime parsers: []const struct { Parser(P), T }) @This() {
-            return .{
-                .parse = struct {
-                    fn parse(buffer: []const u8) !R {
-                        inline for (parsers) |pair| {
-                            const parser, const value = pair;
-
-                            if (parser.parse(buffer)) |result| {
-                                _, const next = result;
-                                return .{ value, next };
-                            } else |_| { }
-                        }
-                        return error.Any;
-                    }
-                }.parse,
-            };
-        }
-    };
-}
-
-test "variants" {
-    const Char = union(enum) {
-        upper: u8,
-        lower: u8,
-        space: void,
-    };
-
-    const parser: Parser(Char) = .variants(struct {
-        const upper: Parser(u8) = .any(void, &.{
-            .{ mod.builtin.literal("A"), 'A' },
-            .{ mod.builtin.literal("B"), 'B' },
-        });
-
-        const lower: Parser(u8) = .any(void, &.{
-            .{ mod.builtin.literal("a"), 'a' },
-            .{ mod.builtin.literal("b"), 'b' },
-        });
-
-        const space: Parser(void) = mod.builtin.literal(" ");
-    });
-
-    {
-        const result = try parser.parseAll("a");
-        try std.testing.expect(result.lower == 'a');
-    }
-
-    {
-        const result = try parser.parseAll("b");
-        try std.testing.expect(result.lower == 'b');
-    }
-
-    {
-        const result = try parser.parseAll("A");
-        try std.testing.expect(result.upper == 'A');
-    }
-
-    {
-        const result = try parser.parseAll("B");
-        try std.testing.expect(result.upper == 'B');
-    }
-
-    {
-        const result = try parser.parseAll(" ");
-        try std.testing.expect(result == .space);
-    }
-
-    try std.testing.expect(parser.parseAll("c") == error.Variants);
-}
diff --git a/src/z/parser/result.zig b/src/z/parser/result.zig
deleted file mode 100644
index 5cc5878..0000000
--- a/src/z/parser/result.zig
+++ /dev/null
@@ -1,4 +0,0 @@
-
-pub fn Result(T: type) type {
-    return struct { T, []const u8 };
-}
diff --git a/src/z/parser/root.zig b/src/z/parser/root.zig
new file mode 100644
index 0000000..e0b67fb
--- /dev/null
+++ b/src/z/parser/root.zig
@@ -0,0 +1,17 @@
+const std = @import("std");
+
+const lexer = @import("lexer.zig");
+pub const Lexer = lexer.Lexer;
+pub const Token = lexer.Token;
+
+pub const js = @import("js/root.zig");
+
+pub fn Parser(TokenType: type) type {
+    return struct {
+        buffer: []const TokenType,
+    };
+}
+
+test {
+    _ = std.testing.refAllDecls(@This());
+}
diff --git a/src/z/root.zig b/src/z/root.zig
index b23ebc5..1fa8d99 100644
--- a/src/z/root.zig
+++ b/src/z/root.zig
@@ -3,6 +3,7 @@ const std = @import("std");
 pub const Minifier = @import("Minifier.zig");
 pub const Component = @import("Component.zig");
 pub const File = @import("File.zig");
+pub const parser = @import("parser/root.zig");
 
 const Element = @import("html").Element;