add tokenizer for all other variants

1 files changed, 14 insertions, 0 deletions

diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig
index d9a2868..40e1209 100644
--- a/src/z/parser/lexer.zig
+++ b/src/z/parser/lexer.zig
@@ -69,6 +69,20 @@ pub fn Lexer(TokenKind: type) type {
             return null;
         }
 
+        pub fn consume(self: *Self) !u8 {
+            return (try self.consumeTo(1))[0];
+        }
+
+        pub fn consumeTo(self: *Self, n: usize) ![]const u8 {
+            if (n <= self.buffer.len) {
+                const slice = self.buffer[0..n];
+                self.buffer = self.buffer[n..];
+                return slice;
+            } else {
+                return Error.EndOfBuffer;
+            }
+        }
+
         pub fn skip(self: *Self) !void {
             return self.skipTo(1);
         }