const std = @import("std"); pub fn Token(TokenKind: type) type { return struct { const Self = @This(); kind: TokenKind, slice: []const u8, pub fn start(comptime lexer: *const Lexer(TokenKind), kind: TokenKind) Self { return .{ .kind = kind, .slice = lexer.buffer, }; } pub inline fn stop(comptime self: Self, comptime lexer: *const Lexer(TokenKind)) Self { comptime { const length = self.slice.len - lexer.buffer.len; return .{ .kind = self.kind, .slice = self.slice[0..length], }; } } pub fn format(self: *const Self, writer: *std.Io.Writer) !void { try writer.print("Token{}(\"{s}\")", .{self.kind, self.slice}); } }; } pub fn Lexer(TokenKind: type) type { return struct { const Self = @This(); pub const Error = error{ UnexpectedToken, EndOfBuffer, }; buffer: []const u8, last_buffer: []const u8, tokens: []const Token(TokenKind) = &.{}, pub fn init(buffer: []const u8) Self { return .{ .buffer = buffer, .last_buffer = buffer, }; } pub inline fn peekChar(self: *Self) ?u8 { return self.peekCharAt(0); } pub inline fn peekCharAt(self: *Self, n: usize) ?u8 { return if (n < self.buffer.len) self.buffer[n] else null; } pub fn peekSlice(self: *Self, length: usize) ?[]const u8 { return self.peekSliceAt(length, 0); } pub fn peekSliceAt(self: *Self, length: usize, offset: usize) ?[]const u8 { if (length + offset <= self.buffer.len) { return self.buffer[offset .. offset + length]; } return null; } pub fn skip(self: *Self) !void { return self.skipTo(1); } pub fn skipTo(self: *Self, n: usize) Error!void { if (n <= self.buffer.len) { self.buffer = self.buffer[n..]; } else { return Error.EndOfBuffer; } } pub fn skipUntil(self: *Self, literal: []const u8) Error!void { if (std.mem.find(u8, self.buffer, literal)) |index| { self.buffer = self.buffer[index + literal.len..]; } else { return Error.EndOfBuffer; } } pub fn skipToEnd(self: *Self) void { self.buffer = self.buffer[self.buffer.len..]; } pub fn start(self: *Self, kind: TokenKind) Token(TokenKind) { return .start(self, kind); } pub fn revert(self: *Self) void { self.buffer = self.last_buffer; } pub fn commit(self: *Self, tok: Token(TokenKind)) void { self.tokens = self.tokens ++ [_]Token(TokenKind){tok.stop(self)}; self.last_buffer = self.buffer; } }; }