This commit is contained in:
2026-07-20 22:56:58 +02:00
commit e9404ea473
3 changed files with 3513 additions and 0 deletions
+114
View File
@@ -0,0 +1,114 @@
#include <ctype.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>
#define NOB_IMPLEMENTATION
#include "nob.h"
typedef enum e_Token_Kind {
TOKEN_KIND_PAR_LEFT,
TOKEN_KIND_PAR_RIGHT,
TOKEN_KIND_NUMBER,
TOKEN_KIND_SYMBOL,
} TokenKind;
typedef struct s_Token {
TokenKind kind;
Nob_String_View data;
} Token;
typedef struct s_Tokens {
Token *items;
size_t count;
size_t capacity;
} Tokens;
void tokenize(Nob_String_View content, Tokens *tokens) {
for (size_t i = 0; i < content.count; i++) {
char c = content.data[i];
int kind = -1;
switch (c) {
case '(': {
kind = TOKEN_KIND_PAR_LEFT;
} break;
case ')': {
kind = TOKEN_KIND_PAR_RIGHT;
} break;
}
if (kind != -1) {
Token tok = (Token){.kind = kind, .data = {0}};
nob_da_append(tokens, tok);
continue;
}
Nob_String_Builder sb = {0};
while (isdigit(c)) {
nob_sb_appendf(&sb, "%c", c);
c = content.data[++i];
}
if (sb.count > 0) {
i--;
Token tok =
(Token){.kind = TOKEN_KIND_NUMBER, .data = nob_sb_to_sv(sb)};
nob_da_append(tokens, tok);
continue;
}
// symbols shouldn't start with a digit
bool condition =
isascii(c) && !isdigit(c) && !isblank(c) && c != '(' && c != ')';
while (condition) {
nob_sb_appendf(&sb, "%c", c);
c = content.data[++i];
condition = isascii(c) && !isblank(c) && c != '(' && c != ')';
}
if (sb.count > 0) {
i--;
Token tok =
(Token){.kind = TOKEN_KIND_SYMBOL, .data = nob_sb_to_sv(sb)};
nob_da_append(tokens, tok);
}
}
}
const char *token_kind_name(TokenKind kind) {
switch (kind) {
case TOKEN_KIND_PAR_LEFT:
return "PAR_LEFT";
case TOKEN_KIND_PAR_RIGHT:
return "PAR_RIGHT";
case TOKEN_KIND_NUMBER:
return "NUMBER";
case TOKEN_KIND_SYMBOL:
return "SYMBOL";
default:
UNREACHABLE("Unknown token type");
}
}
int main(int argc, char **argv) {
if (argc < 2)
return 1;
argv++;
const char *line = *argv++;
Nob_String_View view = nob_sv_from_cstr(line);
Tokens tokens = {0};
tokenize(view, &tokens);
for (size_t i = 0; i < tokens.count; i++) {
Token tok = tokens.items[i];
printf("Token(kind=%s, data=" SV_Fmt ")\n", token_kind_name(tok.kind),
SV_Arg(tok.data));
}
return 0;
}