99 lines
2.9 KiB
C
99 lines
2.9 KiB
C
// lexer.c -- Tokenizer infrastructure, color schemes, common helpers
|
|
|
|
#include "lexer/lexer.h"
|
|
#include <ctype.h>
|
|
#include <string.h>
|
|
|
|
const char *g_lang_names[LANG_COUNT] = {
|
|
"Plain Text", // LANG_PLAIN_TEXT
|
|
"C", // LANG_C
|
|
"Go", // LANG_GO
|
|
"JavaScript", // LANG_JS
|
|
"Lua", // LANG_LUA
|
|
"SQL", // LANG_SQL
|
|
};
|
|
|
|
////////////////////////////////
|
|
// Tokenizer helpers
|
|
|
|
static void tokenizer_init(Tokenizer *tok, const char *data, S32 length) {
|
|
tok->buf = data;
|
|
tok->t = data;
|
|
tok->max_t = data + length;
|
|
tok->start_t = data;
|
|
}
|
|
|
|
static void tokenizer_eat_whitespace(Tokenizer *tok) {
|
|
while (tok->t < tok->max_t && (*tok->t == ' ' || *tok->t == '\t' ||
|
|
*tok->t == '\n' || *tok->t == '\r'))
|
|
tok->t++;
|
|
}
|
|
|
|
static void tokenizer_eat_until_newline(Tokenizer *tok) {
|
|
while (tok->t < tok->max_t && *tok->t != '\n')
|
|
tok->t++;
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Paint helper -- fill out_tokens[start..start+len) with a token type
|
|
|
|
static void paint_token(U8 *out_tokens, S32 start, S32 len, Token_Type type) {
|
|
memset(out_tokens + start, (U8)type, len);
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Language detection
|
|
|
|
static Lang lexer_detect_lang(const char *filename) {
|
|
if (!filename) return LANG_PLAIN_TEXT;
|
|
const char *dot = strrchr(filename, '.');
|
|
if (!dot) return LANG_PLAIN_TEXT;
|
|
dot++; // skip the '.'
|
|
|
|
if (strcmp(dot, "c") == 0 || strcmp(dot, "h") == 0 ||
|
|
strcmp(dot, "C") == 0 || strcmp(dot, "H") == 0)
|
|
return LANG_C;
|
|
|
|
if (strcmp(dot, "go") == 0)
|
|
return LANG_GO;
|
|
|
|
if (strcmp(dot, "js") == 0 || strcmp(dot, "mjs") == 0 ||
|
|
strcmp(dot, "cjs") == 0 || strcmp(dot, "jsx") == 0)
|
|
return LANG_JS;
|
|
|
|
if (strcmp(dot, "lua") == 0)
|
|
return LANG_LUA;
|
|
|
|
if (strcmp(dot, "sql") == 0 || strcmp(dot, "ddl") == 0 ||
|
|
strcmp(dot, "dml") == 0 || strcmp(dot, "pgsql") == 0 ||
|
|
strcmp(dot, "plsql") == 0 || strcmp(dot, "psql") == 0)
|
|
return LANG_SQL;
|
|
|
|
return LANG_PLAIN_TEXT;
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Forward-declare language tokenizers
|
|
|
|
static void tokenize_c(const char *data, S32 length, U8 *out_tokens);
|
|
static void tokenize_go(const char *data, S32 length, U8 *out_tokens);
|
|
static void tokenize_js(const char *data, S32 length, U8 *out_tokens);
|
|
static void tokenize_lua(const char *data, S32 length, U8 *out_tokens);
|
|
static void tokenize_sql(const char *data, S32 length, U8 *out_tokens);
|
|
|
|
static void tokenize_plain(const char *data, S32 length, U8 *out_tokens) {
|
|
(void)data;
|
|
memset(out_tokens, TOK_DEFAULT, length);
|
|
}
|
|
|
|
static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang) {
|
|
switch (lang) {
|
|
case LANG_C: return tokenize_c;
|
|
case LANG_GO: return tokenize_go;
|
|
case LANG_JS: return tokenize_js;
|
|
case LANG_LUA: return tokenize_lua;
|
|
case LANG_SQL: return tokenize_sql;
|
|
default: return tokenize_plain;
|
|
}
|
|
}
|