const std = @import("std"); const eostre = @import("eostre"); const Cursor = eostre.cursor.Cursor; const Parser = eostre.parser.Parser; const Context = eostre.parser.Context; const Literal = eostre.parser.common.Literal; const ParseFn = eostre.parser.ParseFn; const Keyword = enum { if_statement, other }; const String = struct { }; const Token = union(enum) { openparenthesis: void, closeparenthesis: void, openscope: void, closescope: void, backslash: void, pipe: void, semicolon: void, logical_and: void, logical_or: void, assign: void, keyword: Keyword, string: String, }; pub fn parse_string(ctx: *Context(u8)) !Token { const snapshot = ctx.cursor.snapshot(); errdefer ctx.cursor.rollback(snapshot); const first_char = (try ctx.cursor.consume(1))[0]; if (first_char == ' ') { return error.NotAString; } if (first_char == '"') { while (ctx.cursor.consume(1) catch null) |c| { if (c[0] == '\"') { return Token.string; } else if (c[0] == '{') { while (ctx.cursor.consume(1) catch null) |inner| { if (inner[0] == '}') { break; } } else { return error.UnterminatedString; } } } else { return error.UnterminatedString; } } else if (first_char == '\'') { while (ctx.cursor.consume(1) catch null) |c| { if (c[0] == '\'') { return Token.string; } } else { return error.UnterminatedString; } } while (ctx.cursor.consume(1) catch null) |c| { if (c[0] == ' ') { return Token.string; } else if (c[0] == '{') { while (ctx.cursor.consume(1) catch null) |inner| { if (inner[0] == '}') { break; } } else { return error.UnterminatedString; } } } else { return error.UnterminatedString; } } const unterminated_keyword = Parser(u8, Keyword, keyword_literal("if", Keyword.if_statement)); fn keyword(ctx: *Context(u8)) !Token { const snapshot = ctx.cursor.snapshot(); errdefer ctx.cursor.rollback(snapshot); const token = try unterminated_keyword.parse(ctx); const next_char = (try ctx.cursor.peek(1))[0]; switch (next_char) { '(', ')', '{', '}', '\\', '&', '|', ';', '=', ' ' => return .{ .keyword = token }, else => return error.UnterminatedKeyword, } } const literal = Literal(u8, Token); const keyword_literal = Literal(u8, Keyword); const skip_literal = Literal(u8, void); const Tokenizer = Parser(u8, Token, literal("(", Token.openparenthesis)) .or_else(literal(")", Token.closeparenthesis)) .or_else(literal("{", Token.openscope)) .or_else(literal("}", Token.closescope)) .or_else(literal("\\", Token.backslash)) .or_else(literal("&&", Token.logical_and)) .or_else(literal("||", Token.logical_or)) .or_else(literal("|", Token.pipe)) .or_else(literal(";", Token.semicolon)) .or_else(literal("=", Token.assign)) .or_else(keyword) .or_else(parse_string) .and_skip(Parser(u8, void, skip_literal(" ", {})).zero_or_more().parse) .zero_or_more() .then_skip(Parser(u8, void, skip_literal(" ", {})).zero_or_more().parse) .end(); pub fn main() void { std.debug.print("Hello, World!\n", .{}); } test "tokenizer" { const example = " | 'asdf' ( || \"hello\" ) or_something &&{ =}\\ ; if "; var ctx = Context(u8) { .cursor = Cursor(u8).init(example), .allocator = std.testing.allocator, }; const tokens = try Tokenizer.parse(&ctx); defer tokens.deinit(); try std.testing.expectEqualSlices( Token, &[_]Token { .pipe, .string, .openparenthesis, .logical_or, .string, .closeparenthesis, .string, .logical_and, .openscope, .assign, .closescope, .backslash, .semicolon, .{ .keyword = Keyword.if_statement } }, tokens.items ); }