add c and jai

c/lexer/lexer.h

@@ -0,0 +1,98 @@
+#pragma once
+// lexer.h -- Token types, tokenizer interface, and color scheme
+//
+// Follows the Focus editor pattern: a universal Token_Type enum shared by all
+// languages, a per-byte token-type array on each buffer, and language-specific
+// tokenizer functions that fill that array.
+
+#include "base/base_core.h"
+
+////////////////////////////////
+// Token types (universal across all languages)
+//
+// The enum value doubles as an index into the color map, so CODE_DEFAULT
+// must be 0.  Keep this list ordered -- rendering indexes directly.
+
+typedef enum Token_Type {
+    TOK_DEFAULT = 0,
+
+    TOK_COMMENT,
+    TOK_MULTILINE_COMMENT,
+
+    TOK_STRING_LITERAL,
+    TOK_CHAR_LITERAL,
+
+    TOK_NUMBER,
+    TOK_IDENTIFIER,
+    TOK_FUNCTION,
+
+    TOK_KEYWORD,
+    TOK_TYPE,
+    TOK_VALUE,          // true, false, NULL, nullptr
+    TOK_MODIFIER,       // const, static, volatile, extern ...
+
+    TOK_DIRECTIVE,      // #include, #define, ...
+    TOK_PUNCTUATION,
+    TOK_OPERATION,
+
+    TOK_INVALID,
+
+    TOK_COUNT
+} Token_Type;
+
+////////////////////////////////
+// Language enum
+
+typedef enum Lang {
+    LANG_PLAIN_TEXT = 0,
+    LANG_C,
+    LANG_GO,
+    LANG_JS,
+    LANG_LUA,
+    LANG_SQL,
+    LANG_COUNT
+} Lang;
+
+// Human-readable language names (indexed by Lang)
+extern const char *g_lang_names[LANG_COUNT];
+
+////////////////////////////////
+// Tokenizer state (shared by all language tokenizers)
+
+typedef struct Tokenizer {
+    const char *buf;     // start of buffer data
+    const char *t;       // current scan cursor
+    const char *max_t;   // one past end
+    const char *start_t; // cursor at start of current token
+} Tokenizer;
+
+////////////////////////////////
+// Token (returned by get_next_token functions)
+
+typedef struct Token {
+    S32        start;    // byte offset into buffer
+    S32        len;      // byte length of token
+    Token_Type type;
+} Token;
+
+////////////////////////////////
+// Language tokenizer function signature
+//
+// A tokenizer function takes a buffer's data + length, and writes token types
+// into the `out_tokens` array (one byte per source byte, same length as data).
+// This is the Focus "paint the token array" approach.
+
+typedef void (*LexerTokenizeFn)(const char *data, S32 length, U8 *out_tokens);
+
+// Get the tokenizer function for a language.
+static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang);
+
+// Detect language from a file name / extension.
+static Lang lexer_detect_lang(const char *filename);
+
+////////////////////////////////
+// Common tokenizer helpers (usable by all language tokenizers)
+
+static void tokenizer_init(Tokenizer *tok, const char *data, S32 length);
+static void tokenizer_eat_whitespace(Tokenizer *tok);
+static void tokenizer_eat_until_newline(Tokenizer *tok);