add c and jai

c/lexer/lexer.c

@@ -0,0 +1,98 @@
+// lexer.c -- Tokenizer infrastructure, color schemes, common helpers
+
+#include "lexer/lexer.h"
+#include <ctype.h>
+#include <string.h>
+
+const char *g_lang_names[LANG_COUNT] = {
+    "Plain Text",   // LANG_PLAIN_TEXT
+    "C",            // LANG_C
+    "Go",           // LANG_GO
+    "JavaScript",   // LANG_JS
+    "Lua",          // LANG_LUA
+    "SQL",          // LANG_SQL
+};
+
+////////////////////////////////
+// Tokenizer helpers
+
+static void tokenizer_init(Tokenizer *tok, const char *data, S32 length) {
+    tok->buf   = data;
+    tok->t     = data;
+    tok->max_t = data + length;
+    tok->start_t = data;
+}
+
+static void tokenizer_eat_whitespace(Tokenizer *tok) {
+    while (tok->t < tok->max_t && (*tok->t == ' ' || *tok->t == '\t' ||
+           *tok->t == '\n' || *tok->t == '\r'))
+        tok->t++;
+}
+
+static void tokenizer_eat_until_newline(Tokenizer *tok) {
+    while (tok->t < tok->max_t && *tok->t != '\n')
+        tok->t++;
+}
+
+////////////////////////////////
+// Paint helper -- fill out_tokens[start..start+len) with a token type
+
+static void paint_token(U8 *out_tokens, S32 start, S32 len, Token_Type type) {
+    memset(out_tokens + start, (U8)type, len);
+}
+
+////////////////////////////////
+// Language detection
+
+static Lang lexer_detect_lang(const char *filename) {
+    if (!filename) return LANG_PLAIN_TEXT;
+    const char *dot = strrchr(filename, '.');
+    if (!dot) return LANG_PLAIN_TEXT;
+    dot++; // skip the '.'
+
+    if (strcmp(dot, "c") == 0 || strcmp(dot, "h") == 0 ||
+        strcmp(dot, "C") == 0 || strcmp(dot, "H") == 0)
+        return LANG_C;
+
+    if (strcmp(dot, "go") == 0)
+        return LANG_GO;
+
+    if (strcmp(dot, "js") == 0 || strcmp(dot, "mjs") == 0 ||
+        strcmp(dot, "cjs") == 0 || strcmp(dot, "jsx") == 0)
+        return LANG_JS;
+
+    if (strcmp(dot, "lua") == 0)
+        return LANG_LUA;
+
+    if (strcmp(dot, "sql") == 0 || strcmp(dot, "ddl") == 0 ||
+        strcmp(dot, "dml") == 0 || strcmp(dot, "pgsql") == 0 ||
+        strcmp(dot, "plsql") == 0 || strcmp(dot, "psql") == 0)
+        return LANG_SQL;
+
+    return LANG_PLAIN_TEXT;
+}
+
+////////////////////////////////
+// Forward-declare language tokenizers
+
+static void tokenize_c(const char *data, S32 length, U8 *out_tokens);
+static void tokenize_go(const char *data, S32 length, U8 *out_tokens);
+static void tokenize_js(const char *data, S32 length, U8 *out_tokens);
+static void tokenize_lua(const char *data, S32 length, U8 *out_tokens);
+static void tokenize_sql(const char *data, S32 length, U8 *out_tokens);
+
+static void tokenize_plain(const char *data, S32 length, U8 *out_tokens) {
+    (void)data;
+    memset(out_tokens, TOK_DEFAULT, length);
+}
+
+static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang) {
+    switch (lang) {
+    case LANG_C:   return tokenize_c;
+    case LANG_GO:  return tokenize_go;
+    case LANG_JS:  return tokenize_js;
+    case LANG_LUA: return tokenize_lua;
+    case LANG_SQL: return tokenize_sql;
+    default:       return tokenize_plain;
+    }
+}