add tokenizer for whitespaces in js

1 files changed, 110 insertions, 0 deletions

diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig
new file mode 100644
index 0000000..d9a2868
--- /dev/null
+++ b/src/z/parser/lexer.zig
@@ -0,0 +1,110 @@
+const std = @import("std");
+
+pub fn Token(TokenKind: type) type {
+    return struct {
+        const Self = @This();
+
+        kind: TokenKind,
+        slice: []const u8,
+
+        pub fn start(comptime lexer: *const Lexer(TokenKind), kind: TokenKind) Self {
+            return .{
+                .kind = kind,
+                .slice = lexer.buffer,
+            };
+        }
+
+        pub inline fn stop(comptime self: Self, comptime lexer: *const Lexer(TokenKind)) Self {
+            comptime {
+                const length = self.slice.len - lexer.buffer.len;
+                return .{
+                    .kind = self.kind,
+                    .slice = self.slice[0..length],
+                };
+            }
+        }
+
+        pub fn format(self: *const Self, writer: *std.Io.Writer) !void {
+            try writer.print("Token{}(\"{s}\")", .{self.kind, self.slice});
+        }
+    };
+}
+
+pub fn Lexer(TokenKind: type) type {
+    return struct {
+        const Self = @This();
+
+        pub const Error = error{
+            UnexpectedToken,
+            EndOfBuffer,
+        };
+
+        buffer: []const u8,
+        last_buffer: []const u8,
+        tokens: []const Token(TokenKind) = &.{},
+
+        pub fn init(buffer: []const u8) Self {
+            return .{
+                .buffer = buffer,
+                .last_buffer = buffer,
+            };
+        }
+
+        pub inline fn peekChar(self: *Self) ?u8 {
+            return self.peekCharAt(0);
+        }
+
+        pub inline fn peekCharAt(self: *Self, n: usize) ?u8 {
+            return if (n < self.buffer.len) self.buffer[n] else null;
+        }
+
+        pub fn peekSlice(self: *Self, length: usize) ?[]const u8 {
+            return self.peekSliceAt(length, 0);
+        }
+
+        pub fn peekSliceAt(self: *Self, length: usize, offset: usize) ?[]const u8 {
+            if (length + offset <= self.buffer.len) {
+                return self.buffer[offset .. offset + length];
+            }
+            return null;
+        }
+
+        pub fn skip(self: *Self) !void {
+            return self.skipTo(1);
+        }
+
+        pub fn skipTo(self: *Self, n: usize) Error!void {
+            if (n <= self.buffer.len) {
+                self.buffer = self.buffer[n..];
+            } else {
+                return Error.EndOfBuffer;
+            }
+        }
+
+        pub fn skipUntil(self: *Self, literal: []const u8) Error!void {
+            if (std.mem.find(u8, self.buffer, literal)) |index| {
+                self.buffer = self.buffer[index + literal.len..];
+            } else {
+                return Error.EndOfBuffer;
+            }
+        }
+
+        pub fn skipToEnd(self: *Self) void {
+            self.buffer = self.buffer[self.buffer.len..];
+        }
+
+        pub fn start(self: *Self, kind: TokenKind) Token(TokenKind) {
+            return .start(self, kind);
+        }
+
+        pub fn revert(self: *Self) void {
+            self.buffer = self.last_buffer;
+        }
+
+        pub fn commit(self: *Self, tok: Token(TokenKind)) void {
+            self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)};
+            self.last_buffer = self.buffer;
+        }
+
+    };
+}