const std = @import("std");

const parser = @import("../../root.zig");
pub const Lexer = parser.Lexer(Grammar);
pub const Token = parser.Token(Grammar);

pub const Whitespace = @import("whitespace.zig").Whitespace;
pub const Literal = @import("literal.zig").Literal;
pub const Punctuator = @import("punctuator.zig").Punctuator;
pub const Identifier = @import("identifier.zig").Identifier;

pub const Grammar = union(enum) {
    whitespace: Whitespace,
    literal: Literal,
    punctuator: Punctuator,
    identifier: Identifier,
    end_of_file: void,

    pub fn tokenize(lexer: *Lexer) Lexer.Error!void {
        while (lexer.peakChar()) |_| {
            Whitespace.tokenize(lexer) catch {
                Literal.tokenize(lexer) catch {
                    Punctuator.tokenize(lexer) catch {
                        try Identifier.tokenize(lexer);
                    };
                };
            };
        }

        lexer.push(lexer.token(0, .end_of_file));
    }
};

test {
    _ = std.testing.refAllDecls(@This());
}