aboutsummaryrefslogtreecommitdiff
path: root/src/z
diff options
context:
space:
mode:
Diffstat (limited to 'src/z')
-rw-r--r--src/z/parser/js/grammar/identifier.zig54
-rw-r--r--src/z/parser/js/grammar/keyword.zig65
-rw-r--r--src/z/parser/js/grammar/literal.zig81
-rw-r--r--src/z/parser/js/grammar/punctuator.zig54
-rw-r--r--src/z/parser/js/grammar/root.zig34
-rw-r--r--src/z/parser/js/grammar/whitespace.zig16
-rw-r--r--src/z/parser/lexer.zig14
7 files changed, 299 insertions, 19 deletions
diff --git a/src/z/parser/js/grammar/identifier.zig b/src/z/parser/js/grammar/identifier.zig
index 038dbc5..905f937 100644
--- a/src/z/parser/js/grammar/identifier.zig
+++ b/src/z/parser/js/grammar/identifier.zig
@@ -1,4 +1,58 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
pub const Identifier = enum {
public,
private,
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ defer lexer.revert();
+
+ const token = tok: {
+ if (lexer.peekChar() == '#') {
+ const t = lexer.start(.{ .identifier = .private });
+ lexer.skip() catch unreachable;
+ break :tok t;
+ } else {
+ break :tok lexer.start(.{ .identifier = .public });
+ }
+ };
+
+ switch (try lexer.consume()) {
+ 'a'...'z', 'A'...'Z', '_' => {},
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ while (lexer.peekChar()) |ch| {
+ switch (ch) {
+ 'a'...'z', 'A'...'Z', '0'...'9', '_' => {},
+ else => break,
+ }
+
+ try lexer.skip();
+ }
+
+ lexer.commit(token);
+ }
+ }
};
+
+test "simple" {
+ {
+ comptime var lexer: Lexer = .init("some_literal9");
+ try Identifier.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(13, lexer.tokens[0].slice.len);
+ }
+
+ {
+ comptime var lexer: Lexer = .init("#some_literal9");
+ try Identifier.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(14, lexer.tokens[0].slice.len);
+ }
+}
diff --git a/src/z/parser/js/grammar/keyword.zig b/src/z/parser/js/grammar/keyword.zig
index 14c3272..c3b32ac 100644
--- a/src/z/parser/js/grammar/keyword.zig
+++ b/src/z/parser/js/grammar/keyword.zig
@@ -1,3 +1,9 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
pub const Keyword = enum {
@"break",
case,
@@ -45,4 +51,63 @@ pub const Keyword = enum {
null,
true,
false,
+
+ const Self = @This();
+
+ const keywordmap: std.StaticStringMap(Self) = .initComptime(kws: {
+ var keywords: []const struct { []const u8, Self } = &.{};
+
+ for (@typeInfo(Self).@"enum".fields) |field| {
+ const name: []const u8 = field.name;
+ const value: Self = @field(Self, field.name);
+
+ keywords = keywords ++ .{ .{ name, value } };
+ }
+
+ break :kws keywords;
+ });
+
+ pub inline fn tokenize(lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ @setEvalBranchQuota(10000);
+ errdefer lexer.revert();
+
+ const index = std.mem.findNone(u8, lexer.buffer, std.ascii.lowercase) orelse lexer.buffer.len;
+
+ if (index == 0) return Lexer.Error.UnexpectedToken;
+
+ if (keywordmap.get(lexer.buffer[0..index])) |tag| {
+ const token = lexer.start(.{ .keyword = tag });
+ try lexer.skipTo(index);
+ lexer.commit(token);
+ } else {
+ return Lexer.Error.UnexpectedToken;
+ }
+ }
+ }
};
+
+test "simple" {
+ inline for (comptime Keyword.keywordmap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key);
+ try Keyword.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
+
+test "ending" {
+ inline for (comptime Keyword.keywordmap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key ++ ".");
+ try Keyword.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
+
+test "subword" {
+ inline for (comptime Keyword.keywordmap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key ++ "a");
+ try std.testing.expectError(Lexer.Error.UnexpectedToken, Keyword.tokenize(&lexer));
+ }
+}
diff --git a/src/z/parser/js/grammar/literal.zig b/src/z/parser/js/grammar/literal.zig
index a2ef97a..7f92240 100644
--- a/src/z/parser/js/grammar/literal.zig
+++ b/src/z/parser/js/grammar/literal.zig
@@ -1,5 +1,86 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
pub const Literal = enum {
+ // NOTE: currently only decimal literals are tokenized
numeric,
bigint,
string,
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ errdefer lexer.revert();
+
+ var token = lexer.start(.{ .literal = undefined });
+
+ switch (try lexer.consume()) {
+ '0'...'9', '.' => |c| {
+ token.kind.literal = .numeric;
+
+ var had_period = c == '.';
+
+ while (lexer.peekChar()) |ch| {
+ switch (ch) {
+ '0'...'9' => {},
+ '.' => if (had_period) {
+ return Lexer.Error.UnexpectedToken;
+ } else {
+ had_period = true;
+ },
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ try lexer.skip();
+ }
+ },
+ '\'', '"' => |quote| while (try lexer.consume() != quote) {
+ if (quote == '\\') {
+ try lexer.consume();
+ }
+ },
+ else => return Lexer.Error.UnexpectedToken,
+ }
+
+ lexer.commit(token);
+ }
+ }
};
+
+test "numeric" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("1234", 4);
+ try Simple.run("1.234", 5);
+ try std.testing.expectError(
+ Lexer.Error.UnexpectedToken,
+ Simple.run("12.3.4", 0),
+ );
+}
+
+test "string" {
+ const Simple = struct {
+ pub fn run(comptime buffer: []const u8, n: usize) !void {
+ comptime var lexer: Lexer = .init(buffer);
+ try Literal.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(n, lexer.tokens[0].slice.len);
+ }
+ };
+
+ try Simple.run("'hello'", 7);
+ try Simple.run("\"hello\"", 7);
+ try std.testing.expectError(
+ Lexer.Error.EndOfBuffer,
+ Simple.run("'hello", 0),
+ );
+}
diff --git a/src/z/parser/js/grammar/punctuator.zig b/src/z/parser/js/grammar/punctuator.zig
index 81b2e6e..a72ab8a 100644
--- a/src/z/parser/js/grammar/punctuator.zig
+++ b/src/z/parser/js/grammar/punctuator.zig
@@ -1,3 +1,9 @@
+const std = @import("std");
+
+const root = @import("root.zig");
+const Lexer = root.Lexer;
+const Token = root.Token;
+
pub const Punctuator = enum {
@"+",
@"-",
@@ -58,4 +64,52 @@ pub const Punctuator = enum {
@"?.",
@"#",
@"@",
+
+ const Self = @This();
+
+ const punctuatormap: std.StaticStringMap(Self) = .initComptime(kws: {
+ var punctuators: []const struct { []const u8, Self } = &.{};
+
+ for (@typeInfo(Self).@"enum".fields) |field| {
+ const name: []const u8 = field.name;
+ const value: Self = @field(Self, field.name);
+
+ punctuators = punctuators ++ .{.{ name, value }};
+ }
+
+ break :kws punctuators;
+ });
+
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ @setEvalBranchQuota(10_000);
+ defer lexer.revert();
+
+ if (punctuatormap.getLongestPrefix(lexer.buffer)) |pair| {
+ const token = lexer.start(.{ .punctuator = pair.value });
+ try lexer.skipTo(pair.key.len);
+ lexer.commit(token);
+ } else {
+ return Lexer.Error.UnexpectedToken;
+ }
+ }
+ }
};
+
+test "simple" {
+ inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key);
+ try Punctuator.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
+
+test "ending" {
+ inline for (comptime Punctuator.punctuatormap.keys()) |key| {
+ comptime var lexer: Lexer = .init(key ++ "a");
+ try Punctuator.tokenize(&lexer);
+ try std.testing.expectEqual(1, lexer.tokens.len);
+ try std.testing.expectEqual(key.len, lexer.tokens[0].slice.len);
+ }
+}
diff --git a/src/z/parser/js/grammar/root.zig b/src/z/parser/js/grammar/root.zig
index e1a1e96..03b386e 100644
--- a/src/z/parser/js/grammar/root.zig
+++ b/src/z/parser/js/grammar/root.zig
@@ -1,3 +1,13 @@
+//! This is a Javascript parser and tokenizer.
+//! It is and probably will never be complete,
+//! since js is a very strange language and I want
+//! to move forward and code on more interesting things
+//! I started something called mujs (micro-js). It's a
+//! subset of the javascript language. It does not support
+//! the regex literal and currently also not template strings.
+//! It is chosen like that to make parsing easier and also less
+//! context-dependent.
+
const std = @import("std");
const parser = @import("../../root.zig");
@@ -5,29 +15,35 @@ pub const Lexer = parser.Lexer(Grammar);
pub const Token = parser.Token(Grammar);
pub const Whitespace = @import("whitespace.zig").Whitespace;
+pub const Keyword = @import("keyword.zig").Keyword;
pub const Literal = @import("literal.zig").Literal;
pub const Punctuator = @import("punctuator.zig").Punctuator;
pub const Identifier = @import("identifier.zig").Identifier;
pub const Grammar = union(enum) {
whitespace: Whitespace,
+ keyword: Keyword,
literal: Literal,
punctuator: Punctuator,
identifier: Identifier,
end_of_file: void,
- pub fn tokenize(lexer: *Lexer) Lexer.Error!void {
- while (lexer.peakChar()) |_| {
- Whitespace.tokenize(lexer) catch {
- Literal.tokenize(lexer) catch {
- Punctuator.tokenize(lexer) catch {
- try Identifier.tokenize(lexer);
+ pub inline fn tokenize(comptime lexer: *Lexer) Lexer.Error!void {
+ comptime {
+ while (lexer.peekChar()) |_| {
+ Whitespace.tokenize(lexer) catch {
+ Keyword.tokenize(lexer) catch {
+ Literal.tokenize(lexer) catch {
+ Punctuator.tokenize(lexer) catch {
+ try Identifier.tokenize(lexer);
+ };
+ };
};
};
- };
- }
+ }
- lexer.push(lexer.token(0, .end_of_file));
+ lexer.commit(lexer.start(.end_of_file));
+ }
}
};
diff --git a/src/z/parser/js/grammar/whitespace.zig b/src/z/parser/js/grammar/whitespace.zig
index c3435c4..08d031f 100644
--- a/src/z/parser/js/grammar/whitespace.zig
+++ b/src/z/parser/js/grammar/whitespace.zig
@@ -11,11 +11,11 @@ pub const Whitespace = enum {
comptime {
errdefer lexer.revert();
- var token = lexer.start(undefined);
+ var token = lexer.start(.{ .whitespace = undefined });
- switch (lexer.peekChar() orelse 0) {
+ switch (try lexer.consume()) {
' ', '\t', '\r', '\n' => {
- token.kind = .{ .whitespace = .space };
+ token.kind.whitespace = .space;
while (lexer.peekChar()) |ch| {
switch (ch) {
@@ -25,13 +25,13 @@ pub const Whitespace = enum {
}
},
'/' => {
- switch (lexer.peekCharAt(1) orelse 0) {
+ switch (lexer.peekChar() orelse 0) {
'*' => {
- token.kind = .{ .whitespace = .block_comment };
+ token.kind.whitespace = .block_comment;
try lexer.skipUntil("*/");
},
'/' => {
- token.kind = .{ .whitespace = .line_comment };
+ token.kind.whitespace = .line_comment;
lexer.skipUntil("\n") catch lexer.skipToEnd();
},
else => return Lexer.Error.UnexpectedToken,
@@ -99,8 +99,4 @@ test "mixed" {
} else |_| {}
try std.testing.expectEqual(7, lexer.tokens.len);
-
- inline for (lexer.tokens) |token| {
- std.debug.print("{f}\n", .{token});
- }
}
diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig
index d9a2868..40e1209 100644
--- a/src/z/parser/lexer.zig
+++ b/src/z/parser/lexer.zig
@@ -69,6 +69,20 @@ pub fn Lexer(TokenKind: type) type {
return null;
}
+ pub fn consume(self: *Self) !u8 {
+ return (try self.consumeTo(1))[0];
+ }
+
+ pub fn consumeTo(self: *Self, n: usize) ![]const u8 {
+ if (n <= self.buffer.len) {
+ const slice = self.buffer[0..n];
+ self.buffer = self.buffer[n..];
+ return slice;
+ } else {
+ return Error.EndOfBuffer;
+ }
+ }
+
pub fn skip(self: *Self) !void {
return self.skipTo(1);
}
generated by cgit v1.2.3-70-g09d2 (git 2.53.0) at 2026-09-16 12:57:12 +0000