diff options
Diffstat (limited to 'src/z/parser/js/grammar')
| -rw-r--r-- | src/z/parser/js/grammar/identifier.zig | 60 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/keyword.zig | 115 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/literal.zig | 192 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/punctuator.zig | 122 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/root.zig | 66 | ||||
| -rw-r--r-- | src/z/parser/js/grammar/whitespace.zig | 102 |
6 files changed, 657 insertions, 0 deletions
diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig new file mode 100644 index 0000000..ccc5268 --- /dev/null +++ b/src/z/parser/js/grammar/identifier.zig @@ -0,0 +1,60 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + +pub const Identifier = enum { + public, + private, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + defer lexer.revert(); + + lexer.context.regex_possible = false; + + const token = tok: { + if (lexer.peekChar() == '#') { + const t = lexer.start(.{ .identifier = .private }); + lexer.skip() catch unreachable; + break :tok t; + } else { + break :tok lexer.start(.{ .identifier = .public }); + } + }; + + switch (try lexer.consume()) { + 'a'...'z', 'A'...'Z', '_' => {}, + else => return Lexer.Error.UnexpectedToken, + } + + while (lexer.peekChar()) |ch| { + switch (ch) { + 'a'...'z', 'A'...'Z', '0'...'9', '_' => {}, + else => break, + } + + try lexer.skip(); + } + + lexer.commit(token); + } + } +}; + +test "simple" { + { + comptime var lexer: Lexer = .init("some_literal9"); + try Identifier.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(13, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init("#some_literal9"); + try Identifier.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(14, lexer.tokens[0].slice.len); + } +} diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig new file mode 100644 index 0000000..8e99b4a --- /dev/null +++ b/src/z/parser/js/grammar/keyword.zig @@ -0,0 +1,115 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + +pub const Keyword = enum { + @"break", + case, + @"catch", + class, + @"const", + @"continue", + debugger, + default, + delete, + do, + @"else", + @"export", + extends, + finally, + @"for", + function, + @"if", + import, + in, + instanceof, + new, + @"return", + super, + @"switch", + this, + throw, + @"try", + typeof, + @"var", + void, + @"while", + with, + yield, + let, + static, + await, + async, + of, + from, + get, + set, + meta, + @"enum", + null, + true, + false, + + const Self = @This(); + + const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: { + var keywords: []const struct { []const u8, Self } = &.{}; + + for (@typeInfo(Self).@"enum".fields) |field| { + const name: []const u8 = field.name; + const value: Self = @field(Self, field.name); + + keywords = keywords ++ .{ .{ name, value } }; + } + + break :kws keywords; + }); + + pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void { + comptime { + @setEvalBranchQuota(10000); + errdefer lexer.revert(); + + lexer.context.regex_possible = true; + + const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len; + + if (index == 0) return Lexer.Error.UnexpectedToken; + + if (keywordmap.get(lexer.buffer[0..index])) |tag| { + const token = lexer.start(.{ .keyword = tag }); + try lexer.skipTo(index); + lexer.commit(token); + } else { + return Lexer.Error.UnexpectedToken; + } + } + } +}; + +test "simple" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key); + try Keyword.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "ending" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "."); + try Keyword.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "subword" { + inline for (comptime Keyword.keywordmap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "a"); + try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer)); + } +} diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig new file mode 100644 index 0000000..4316dee --- /dev/null +++ b/src/z/parser/js/grammar/literal.zig @@ -0,0 +1,192 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + +pub const Literal = union(enum) { + // NOTE: currently only decimal literals are tokenized + numeric: void, + bigint: void, + string: void, + template: enum { + whole, + start, + middle, + end, + }, + regex: void, + + const digits = "_0123456789abcdef"; + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(.{ .literal = undefined }); + + const regex_possible = lexer.context.regex_possible; + lexer.context.regex_possible = false; + + switch (try lexer.consume()) { + '0'...'9', '.' => |c| { + const base = if (c != '0') 10 else switch (lexer.peekChar() orelse 0) { + 'x' => 16, + 'o' => 8, + 'b' => 2, + else => 10, + }; + + if (base != 10) lexer.skip() catch unreachable; + + { + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + + + if (index == 0 and base != 10) return Lexer.Error.UnexpectedToken; + } + + if (base == 10 and lexer.peekChar() == '.') { + try lexer.skip(); + const index = std.mem.findNone( + u8, + lexer.buffer, + digits[0 .. base + 1], + ) orelse lexer.buffer.len; + try lexer.skipTo(index); + } + + if (lexer.peekChar() == 'n') { + lexer.skip() catch unreachable; + token.kind.literal = .bigint; + } else { + token.kind.literal = .numeric; + } + }, + '\'', '"' => |quote| { + token.kind.literal = .string; + while (true) { + switch (try lexer.consume()) { + quote => break, + '\\' => try lexer.skip(), + else => {}, + } + } + }, + '`' => { + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .whole; + break; + }, + '\\' => try lexer.skip(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .start; + lexer.context.regex_possible = true; + break; + }, + else => {}, + } + } + }, + '}' => if (lexer.context.template_nesting > 0) { + lexer.context.template_nesting -= 1; + token.kind.literal = .{ .template = undefined }; + + while (true) { + switch (try lexer.consume()) { + '`' => { + token.kind.literal.template = .end; + break; + }, + '\\' => try lexer.skip(), + '$' => if (lexer.peekChar() == '{') { + lexer.skip() catch unreachable; + lexer.context.template_nesting += 1; + token.kind.literal.template = .middle; + lexer.context.regex_possible = true; + break; + }, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, + '/' => if (regex_possible) { + token.kind.literal = .regex; + while (true) { + switch (try lexer.consume()) { + '/' => break, + '\\' => try lexer.skip(), + '\n' => return Lexer.Error.UnexpectedToken, + else => {}, + } + } + } else return Lexer.Error.UnexpectedToken, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "numeric" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("1234", 4); + try Simple.run("1.234", 5); + try Simple.run("1_2_3_4", 7); + try Simple.run("0b101011", 8); + try Simple.run("0xabcf01", 8); + try Simple.run("0o731734", 8); + try Simple.run("0o731734n", 9); +} + +test "string" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("'hello'", 7); + try Simple.run("\"hello\"", 7); + try std.testing.expectError( + Lexer.Error.EndOfBuffer, + Simple.run("'hello", 0), + ); +} + +test "regex" { + const Simple = struct { + pub fn run(comptime buffer: []const u8, n: usize) !void { + comptime var lexer: Lexer = .init(buffer); + try Literal.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(n, lexer.tokens[0].slice.len); + } + }; + + try Simple.run("/hello/", 7); + try Simple.run("/some string\\/ here/", 20); +} diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig new file mode 100644 index 0000000..b085ce1 --- /dev/null +++ b/src/z/parser/js/grammar/punctuator.zig @@ -0,0 +1,122 @@ +const std = @import("std"); + +const root = @import("root.zig"); +const Lexer = root.Lexer; +const Token = root.Token; + +pub const Punctuator = enum { + @"+", + @"-", + @"*", + @"/", + @"%", + @"**", + @"++", + @"--", + @"=", + @"+=", + @"-=", + @"*=", + @"/=", + @"%=", + @"**=", + @"&=", + @"|=", + @"^=", + @"<<=", + @">>=", + @">>>=", + @"&&=", + @"||=", + @"??=", + @"==", + @"===", + @"!=", + @"!==", + @"<", + @"<=", + @">", + @">=", + @"&&", + @"||", + @"??", + @"&", + @"|", + @"^", + @"~", + @"<<", + @">>", + @">>>", + @"!", + @"?", + @":", + @";", + @",", + @".", + @"...", + @"=>", + @"(", + @")", + @"[", + @"]", + @"{", + @"}", + @"?.", + @"#", + @"@", + + const Self = @This(); + + const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: { + var punctuators: []const struct { []const u8, Self } = &.{}; + + for (@typeInfo(Self).@"enum".fields) |field| { + const name: []const u8 = field.name; + const value: Self = @field(Self, field.name); + + punctuators = punctuators ++ .{.{ name, value }}; + } + + break :kws punctuators; + }); + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + @setEvalBranchQuota(10_000); + defer lexer.revert(); + + lexer.context.regex_possible = true; + + if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| { + + if (std.mem.eql(u8, pair.key, "}") and lexer.context.template_nesting > 0) { + return Lexer.Error.UnexpectedToken; + } + + const token = lexer.start(.{ .punctuator = pair.value }); + try lexer.skipTo(pair.key.len); + lexer.commit(token); + } else { + return Lexer.Error.UnexpectedToken; + } + } + } +}; + +test "simple" { + inline for (comptime Punctuator.punctuatormap.keys()) |key| { + comptime var lexer: Lexer = .init(key); + try Punctuator.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} + +test "ending" { + inline for (comptime Punctuator.punctuatormap.keys()) |key| { + comptime var lexer: Lexer = .init(key ++ "a"); + try Punctuator.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len); + } +} diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig new file mode 100644 index 0000000..f732b7f --- /dev/null +++ b/src/z/parser/js/grammar/root.zig @@ -0,0 +1,66 @@ +//! This is a Javascript parser and tokenizer. +//! It is and probably will never be complete, +//! since js is a very strange language and I want +//! to move forward and code on more interesting things +//! I started something called mujs (micro-js). It's a +//! subset of the javascript language. It does not support +//! the regex literal and currently also not template strings. +//! It is chosen like that to make parsing easier and also less +//! context-dependent. + +const std = @import("std"); + +const parser = @import("../../root.zig"); + +pub const Lexer = parser.Lexer(Grammar); +pub const Token = parser.Token(Grammar); + +pub const Whitespace = @import("whitespace.zig").Whitespace; +pub const Keyword = @import("keyword.zig").Keyword; +pub const Literal = @import("literal.zig").Literal; +pub const Punctuator = @import("punctuator.zig").Punctuator; +pub const Identifier = @import("identifier.zig").Identifier; + +pub const Grammar = union(enum) { + whitespace: Whitespace, + keyword: Keyword, + literal: Literal, + punctuator: Punctuator, + identifier: Identifier, + end_of_file: void, + + pub const Context = struct { + template_nesting: usize = 0, + regex_possible: bool = true, + }; + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + @setEvalBranchQuota(lexer.buffer.len * 1000); + comptime { + while (lexer.peekChar()) |_| { + Whitespace.tokenize(lexer) catch { + Keyword.tokenize(lexer) catch { + Literal.tokenize(lexer) catch { + Punctuator.tokenize(lexer) catch { + try Identifier.tokenize(lexer); + }; + }; + }; + }; + } + + lexer.commit(lexer.start(.end_of_file)); + } + } +}; + +test { + _ = std.testing.refAllDecls(@This()); +} + +test "z.js" { + comptime var lexer: Lexer = .init(@embedFile("../../../z.js")); + Grammar.tokenize(&lexer) catch |err| { + std.debug.print("error: {} at {s}", .{ err, lexer.buffer }); + }; +} diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig new file mode 100644 index 0000000..08d031f --- /dev/null +++ b/src/z/parser/js/grammar/whitespace.zig @@ -0,0 +1,102 @@ +const std = @import("std"); +const Lexer = @import("root.zig").Lexer; +const Token = @import("root.zig").Token; + +pub const Whitespace = enum { + space, + line_comment, + block_comment, + + pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void { + comptime { + errdefer lexer.revert(); + + var token = lexer.start(.{ .whitespace = undefined }); + + switch (try lexer.consume()) { + ' ', '\t', '\r', '\n' => { + token.kind.whitespace = .space; + + while (lexer.peekChar()) |ch| { + switch (ch) { + ' ', '\t', '\r', '\n' => try lexer.skip(), + else => break, + } + } + }, + '/' => { + switch (lexer.peekChar() orelse 0) { + '*' => { + token.kind.whitespace = .block_comment; + try lexer.skipUntil("*/"); + }, + '/' => { + token.kind.whitespace = .line_comment; + lexer.skipUntil("\n") catch lexer.skipToEnd(); + }, + else => return Lexer.Error.UnexpectedToken, + } + }, + else => return Lexer.Error.UnexpectedToken, + } + + lexer.commit(token); + } + } +}; + +test "space" { + { + comptime var lexer: Lexer = .init(" "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(1, lexer.tokens[0].slice.len); + } + + { + comptime var lexer: Lexer = .init(" \t\t \n\r "); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqual(10, lexer.tokens[0].slice.len); + } +} + +test "line-comment" { + { + comptime var lexer: Lexer = .init("// some comment"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment", lexer.tokens[0].slice); + } + + { + comptime var lexer: Lexer = .init("// some comment\n"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("// some comment\n", lexer.tokens[0].slice); + } +} + +test "block-comment" { + { + comptime var lexer: Lexer = .init("/* some comment */"); + try Whitespace.tokenize(&lexer); + try std.testing.expectEqual(1, lexer.tokens.len); + try std.testing.expectEqualStrings("/* some comment */", lexer.tokens[0].slice); + } +} + +test "mixed" { + comptime var lexer: Lexer = .init( + \\ // some comment + \\/* other comment here */ // comment or so + \\ /* multi + \\ * line + \\ * comment + \\ */ + ); + inline while (Whitespace.tokenize(&lexer)) { + } else |_| {} + + try std.testing.expectEqual(7, lexer.tokens.len); +} |