diff options
| author | Alexander Rolley <arolley35@gmail.com> | 2026-08-05 17:25:27 +0200 |
|---|---|---|
| committer | Alexander Rolley <arolley35@gmail.com> | 2026-08-05 17:25:27 +0200 |
| commit | dc90bc811648da695e4cb47826ea6479cfd22a06 (patch) | |
| tree | 3621725aadb61d267f1e88afb570a4ab3c824ab8 /src/z/parser/js/grammar/whitespace.zig | |
| parent | 23ac91196552068d44bab5677a09a106b8172e27 (diff) | |
| parent | 9b63c0f568832085b75ea6e3e983dc6cbeccfea4 (diff) | |
Merge branch 'dev' of https://git.nathanreiner.xyz/memora into develop
Diffstat (limited to 'src/z/parser/js/grammar/whitespace.zig')
| -rw-r--r-- | src/z/parser/js/grammar/whitespace.zig | 102 |
1 files changed, 102 insertions, 0 deletions
diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..08d031f --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,102 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(.{ .whitespace = undefined }); + + switch (try lexer.consume()) { + ' ', '\t', '\r', '\n' => { + token.kind.whitespace = .space; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekChar() orelse 0) { + '*' => { + token.kind.whitespace = .block_comment; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind.whitespace = .line_comment; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); +} |