const std = @import("std"); const Lexer = @import("root.zig").Lexer; const Token = @import("root.zig").Token; pub const Whitespace = enum { space, line_comment, block_comment, pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { comptime { errdefer lexer.revert(); var token = lexer.start(.{ .whitespace = undefined }); switch (try lexer.consume()) { ' ', '\t', '\r', '\n' => { token.kind.whitespace = .space; while (lexer.peekChar()) |ch| { switch (ch) { ' ', '\t', '\r', '\n' => try lexer.skip(), else => break, } } }, '/' => { switch (lexer.peekChar() orelse 0) { '*' => { token.kind.whitespace = .block_comment; try lexer.skipUntil("*/"); }, '/' => { token.kind.whitespace = .line_comment; lexer.skipUntil("\n") catch lexer.skipToEnd(); }, else => return Lexer.Error.UnexpectedToken, } }, else => return Lexer.Error.UnexpectedToken, } lexer.commit(token); } } }; test "space" { { comptime var lexer: Lexer = .init(" "); try Whitespace.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(1, lexer.tokens[0].slice.len); } { comptime var lexer: Lexer = .init(" \t\t \n\r "); try Whitespace.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqual(10, lexer.tokens[0].slice.len); } } test "line-comment" { { comptime var lexer: Lexer = .init("// some comment"); try Whitespace.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); } { comptime var lexer: Lexer = .init("// some comment\n"); try Whitespace.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); } } test "block-comment" { { comptime var lexer: Lexer = .init("/* some comment */"); try Whitespace.tokenize(&lexer); try std.testing.expectEqual(1, lexer.tokens.len); try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); } } test "mixed" { comptime var lexer: Lexer = .init( \\ // some comment \\/* other comment here */ // comment or so \\ /* multi \\ * line \\ * comment \\ */ ); inline while (Whitespace.tokenize(&lexer)) { } else |_| {} try std.testing.expectEqual(7, lexer.tokens.len); }