From a10b091d78208907397a6d49791b27e04e675161 Mon Sep 17 00:00:00 2001
From: Nathan Reiner <nathan@nathanreiner.xyz>
Date: Tue, 4 Aug 2026 19:02:27 +0200
Subject: add tokenizer for all other variants

---
 src/z/parser/lexer.zig | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

(limited to 'src/z/parser/lexer.zig')

diff --git a/src/z/parser/lexer.zig b/src/z/parser/lexer.zig
index d9a2868..40e1209 100644
--- a/src/z/parser/lexer.zig
+++ b/src/z/parser/lexer.zig
@@ -69,6 +69,20 @@ pub fn Lexer(TokenKind: type) type {
             return null;
         }
 
+        pub fn consume(self: *Self) !u8 {
+            return (try self.consumeTo(1))[0];
+        }
+
+        pub fn consumeTo(self: *Self, n: usize) ![]const u8 {
+            if (n <= self.buffer.len) {
+                const slice = self.buffer[0..n];
+                self.buffer = self.buffer[n..];
+                return slice;
+            } else {
+                return Error.EndOfBuffer;
+            }
+        }
+
         pub fn skip(self: *Self) !void {
             return self.skipTo(1);
         }
-- 
cgit v1.2.3-70-g09d2