#include #include #include #include #define NOB_IMPLEMENTATION #include "nob.h" typedef enum e_Token_Kind { TOKEN_KIND_PAR_LEFT, TOKEN_KIND_PAR_RIGHT, TOKEN_KIND_NUMBER, TOKEN_KIND_SYMBOL, } TokenKind; typedef struct s_Token { TokenKind kind; Nob_String_View data; } Token; typedef struct s_Tokens { Token *items; size_t count; size_t capacity; } Tokens; void tokenize(Nob_String_View content, Tokens *tokens) { for (size_t i = 0; i < content.count; i++) { char c = content.data[i]; int kind = -1; switch (c) { case '(': { kind = TOKEN_KIND_PAR_LEFT; } break; case ')': { kind = TOKEN_KIND_PAR_RIGHT; } break; } if (kind != -1) { Token tok = (Token){.kind = kind, .data = {0}}; nob_da_append(tokens, tok); continue; } Nob_String_Builder sb = {0}; while (isdigit(c)) { nob_sb_appendf(&sb, "%c", c); c = content.data[++i]; } if (sb.count > 0) { i--; Token tok = (Token){.kind = TOKEN_KIND_NUMBER, .data = nob_sb_to_sv(sb)}; nob_da_append(tokens, tok); continue; } // symbols shouldn't start with a digit bool condition = isascii(c) && !isdigit(c) && !isblank(c) && c != '(' && c != ')'; while (condition) { nob_sb_appendf(&sb, "%c", c); c = content.data[++i]; condition = isascii(c) && !isblank(c) && c != '(' && c != ')'; } if (sb.count > 0) { i--; Token tok = (Token){.kind = TOKEN_KIND_SYMBOL, .data = nob_sb_to_sv(sb)}; nob_da_append(tokens, tok); } } } const char *token_kind_name(TokenKind kind) { switch (kind) { case TOKEN_KIND_PAR_LEFT: return "PAR_LEFT"; case TOKEN_KIND_PAR_RIGHT: return "PAR_RIGHT"; case TOKEN_KIND_NUMBER: return "NUMBER"; case TOKEN_KIND_SYMBOL: return "SYMBOL"; default: UNREACHABLE("Unknown token type"); } } int main(int argc, char **argv) { if (argc < 2) return 1; argv++; const char *line = *argv++; Nob_String_View view = nob_sv_from_cstr(line); Tokens tokens = {0}; tokenize(view, &tokens); for (size_t i = 0; i < tokens.count; i++) { Token tok = tokens.items[i]; printf("Token(kind=%s, data=" SV_Fmt ")\n", token_kind_name(tok.kind), SV_Arg(tok.data)); } return 0; }