add c and jai
This commit is contained in:
98
c/lexer/lexer.c
Normal file
98
c/lexer/lexer.c
Normal file
@@ -0,0 +1,98 @@
|
||||
// lexer.c -- Tokenizer infrastructure, color schemes, common helpers
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
const char *g_lang_names[LANG_COUNT] = {
|
||||
"Plain Text", // LANG_PLAIN_TEXT
|
||||
"C", // LANG_C
|
||||
"Go", // LANG_GO
|
||||
"JavaScript", // LANG_JS
|
||||
"Lua", // LANG_LUA
|
||||
"SQL", // LANG_SQL
|
||||
};
|
||||
|
||||
////////////////////////////////
|
||||
// Tokenizer helpers
|
||||
|
||||
static void tokenizer_init(Tokenizer *tok, const char *data, S32 length) {
|
||||
tok->buf = data;
|
||||
tok->t = data;
|
||||
tok->max_t = data + length;
|
||||
tok->start_t = data;
|
||||
}
|
||||
|
||||
static void tokenizer_eat_whitespace(Tokenizer *tok) {
|
||||
while (tok->t < tok->max_t && (*tok->t == ' ' || *tok->t == '\t' ||
|
||||
*tok->t == '\n' || *tok->t == '\r'))
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
static void tokenizer_eat_until_newline(Tokenizer *tok) {
|
||||
while (tok->t < tok->max_t && *tok->t != '\n')
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Paint helper -- fill out_tokens[start..start+len) with a token type
|
||||
|
||||
static void paint_token(U8 *out_tokens, S32 start, S32 len, Token_Type type) {
|
||||
memset(out_tokens + start, (U8)type, len);
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Language detection
|
||||
|
||||
static Lang lexer_detect_lang(const char *filename) {
|
||||
if (!filename) return LANG_PLAIN_TEXT;
|
||||
const char *dot = strrchr(filename, '.');
|
||||
if (!dot) return LANG_PLAIN_TEXT;
|
||||
dot++; // skip the '.'
|
||||
|
||||
if (strcmp(dot, "c") == 0 || strcmp(dot, "h") == 0 ||
|
||||
strcmp(dot, "C") == 0 || strcmp(dot, "H") == 0)
|
||||
return LANG_C;
|
||||
|
||||
if (strcmp(dot, "go") == 0)
|
||||
return LANG_GO;
|
||||
|
||||
if (strcmp(dot, "js") == 0 || strcmp(dot, "mjs") == 0 ||
|
||||
strcmp(dot, "cjs") == 0 || strcmp(dot, "jsx") == 0)
|
||||
return LANG_JS;
|
||||
|
||||
if (strcmp(dot, "lua") == 0)
|
||||
return LANG_LUA;
|
||||
|
||||
if (strcmp(dot, "sql") == 0 || strcmp(dot, "ddl") == 0 ||
|
||||
strcmp(dot, "dml") == 0 || strcmp(dot, "pgsql") == 0 ||
|
||||
strcmp(dot, "plsql") == 0 || strcmp(dot, "psql") == 0)
|
||||
return LANG_SQL;
|
||||
|
||||
return LANG_PLAIN_TEXT;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Forward-declare language tokenizers
|
||||
|
||||
static void tokenize_c(const char *data, S32 length, U8 *out_tokens);
|
||||
static void tokenize_go(const char *data, S32 length, U8 *out_tokens);
|
||||
static void tokenize_js(const char *data, S32 length, U8 *out_tokens);
|
||||
static void tokenize_lua(const char *data, S32 length, U8 *out_tokens);
|
||||
static void tokenize_sql(const char *data, S32 length, U8 *out_tokens);
|
||||
|
||||
static void tokenize_plain(const char *data, S32 length, U8 *out_tokens) {
|
||||
(void)data;
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
}
|
||||
|
||||
static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang) {
|
||||
switch (lang) {
|
||||
case LANG_C: return tokenize_c;
|
||||
case LANG_GO: return tokenize_go;
|
||||
case LANG_JS: return tokenize_js;
|
||||
case LANG_LUA: return tokenize_lua;
|
||||
case LANG_SQL: return tokenize_sql;
|
||||
default: return tokenize_plain;
|
||||
}
|
||||
}
|
||||
98
c/lexer/lexer.h
Normal file
98
c/lexer/lexer.h
Normal file
@@ -0,0 +1,98 @@
|
||||
#pragma once
|
||||
// lexer.h -- Token types, tokenizer interface, and color scheme
|
||||
//
|
||||
// Follows the Focus editor pattern: a universal Token_Type enum shared by all
|
||||
// languages, a per-byte token-type array on each buffer, and language-specific
|
||||
// tokenizer functions that fill that array.
|
||||
|
||||
#include "base/base_core.h"
|
||||
|
||||
////////////////////////////////
|
||||
// Token types (universal across all languages)
|
||||
//
|
||||
// The enum value doubles as an index into the color map, so CODE_DEFAULT
|
||||
// must be 0. Keep this list ordered -- rendering indexes directly.
|
||||
|
||||
typedef enum Token_Type {
|
||||
TOK_DEFAULT = 0,
|
||||
|
||||
TOK_COMMENT,
|
||||
TOK_MULTILINE_COMMENT,
|
||||
|
||||
TOK_STRING_LITERAL,
|
||||
TOK_CHAR_LITERAL,
|
||||
|
||||
TOK_NUMBER,
|
||||
TOK_IDENTIFIER,
|
||||
TOK_FUNCTION,
|
||||
|
||||
TOK_KEYWORD,
|
||||
TOK_TYPE,
|
||||
TOK_VALUE, // true, false, NULL, nullptr
|
||||
TOK_MODIFIER, // const, static, volatile, extern ...
|
||||
|
||||
TOK_DIRECTIVE, // #include, #define, ...
|
||||
TOK_PUNCTUATION,
|
||||
TOK_OPERATION,
|
||||
|
||||
TOK_INVALID,
|
||||
|
||||
TOK_COUNT
|
||||
} Token_Type;
|
||||
|
||||
////////////////////////////////
|
||||
// Language enum
|
||||
|
||||
typedef enum Lang {
|
||||
LANG_PLAIN_TEXT = 0,
|
||||
LANG_C,
|
||||
LANG_GO,
|
||||
LANG_JS,
|
||||
LANG_LUA,
|
||||
LANG_SQL,
|
||||
LANG_COUNT
|
||||
} Lang;
|
||||
|
||||
// Human-readable language names (indexed by Lang)
|
||||
extern const char *g_lang_names[LANG_COUNT];
|
||||
|
||||
////////////////////////////////
|
||||
// Tokenizer state (shared by all language tokenizers)
|
||||
|
||||
typedef struct Tokenizer {
|
||||
const char *buf; // start of buffer data
|
||||
const char *t; // current scan cursor
|
||||
const char *max_t; // one past end
|
||||
const char *start_t; // cursor at start of current token
|
||||
} Tokenizer;
|
||||
|
||||
////////////////////////////////
|
||||
// Token (returned by get_next_token functions)
|
||||
|
||||
typedef struct Token {
|
||||
S32 start; // byte offset into buffer
|
||||
S32 len; // byte length of token
|
||||
Token_Type type;
|
||||
} Token;
|
||||
|
||||
////////////////////////////////
|
||||
// Language tokenizer function signature
|
||||
//
|
||||
// A tokenizer function takes a buffer's data + length, and writes token types
|
||||
// into the `out_tokens` array (one byte per source byte, same length as data).
|
||||
// This is the Focus "paint the token array" approach.
|
||||
|
||||
typedef void (*LexerTokenizeFn)(const char *data, S32 length, U8 *out_tokens);
|
||||
|
||||
// Get the tokenizer function for a language.
|
||||
static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang);
|
||||
|
||||
// Detect language from a file name / extension.
|
||||
static Lang lexer_detect_lang(const char *filename);
|
||||
|
||||
////////////////////////////////
|
||||
// Common tokenizer helpers (usable by all language tokenizers)
|
||||
|
||||
static void tokenizer_init(Tokenizer *tok, const char *data, S32 length);
|
||||
static void tokenizer_eat_whitespace(Tokenizer *tok);
|
||||
static void tokenizer_eat_until_newline(Tokenizer *tok);
|
||||
423
c/lexer/lexer_c.c
Normal file
423
c/lexer/lexer_c.c
Normal file
@@ -0,0 +1,423 @@
|
||||
// lexer_c.c -- C language tokenizer
|
||||
//
|
||||
// Follows the Focus editor pattern: a get_next_token() loop that advances
|
||||
// a cursor through the source, producing Token structs. After each token,
|
||||
// we paint the per-byte token-type array with the token's type.
|
||||
// A second pass retroactively marks identifiers followed by '(' as functions.
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
////////////////////////////////
|
||||
// C keyword tables
|
||||
|
||||
typedef struct KeywordEntry {
|
||||
const char *word;
|
||||
Token_Type type;
|
||||
} KeywordEntry;
|
||||
|
||||
static const KeywordEntry c_keywords[] = {
|
||||
// Control-flow & language keywords
|
||||
{"auto", TOK_KEYWORD},
|
||||
{"break", TOK_KEYWORD},
|
||||
{"case", TOK_KEYWORD},
|
||||
{"continue", TOK_KEYWORD},
|
||||
{"default", TOK_KEYWORD},
|
||||
{"do", TOK_KEYWORD},
|
||||
{"else", TOK_KEYWORD},
|
||||
{"enum", TOK_KEYWORD},
|
||||
{"for", TOK_KEYWORD},
|
||||
{"goto", TOK_KEYWORD},
|
||||
{"if", TOK_KEYWORD},
|
||||
{"inline", TOK_KEYWORD},
|
||||
{"restrict", TOK_KEYWORD},
|
||||
{"return", TOK_KEYWORD},
|
||||
{"sizeof", TOK_KEYWORD},
|
||||
{"struct", TOK_KEYWORD},
|
||||
{"switch", TOK_KEYWORD},
|
||||
{"typedef", TOK_KEYWORD},
|
||||
{"union", TOK_KEYWORD},
|
||||
{"while", TOK_KEYWORD},
|
||||
{"_Alignas", TOK_KEYWORD},
|
||||
{"alignas", TOK_KEYWORD},
|
||||
{"_Alignof", TOK_KEYWORD},
|
||||
{"alignof", TOK_KEYWORD},
|
||||
{"_Atomic", TOK_KEYWORD},
|
||||
{"_Generic", TOK_KEYWORD},
|
||||
{"_Noreturn", TOK_KEYWORD},
|
||||
{"static_assert", TOK_KEYWORD},
|
||||
{"_Static_assert", TOK_KEYWORD},
|
||||
|
||||
// Types
|
||||
{"char", TOK_TYPE},
|
||||
{"double", TOK_TYPE},
|
||||
{"float", TOK_TYPE},
|
||||
{"int", TOK_TYPE},
|
||||
{"long", TOK_TYPE},
|
||||
{"short", TOK_TYPE},
|
||||
{"void", TOK_TYPE},
|
||||
{"bool", TOK_TYPE},
|
||||
{"_Bool", TOK_TYPE},
|
||||
{"_Complex", TOK_TYPE},
|
||||
{"_Imaginary", TOK_TYPE},
|
||||
// stdint
|
||||
{"int8_t", TOK_TYPE},
|
||||
{"int16_t", TOK_TYPE},
|
||||
{"int32_t", TOK_TYPE},
|
||||
{"int64_t", TOK_TYPE},
|
||||
{"uint8_t", TOK_TYPE},
|
||||
{"uint16_t", TOK_TYPE},
|
||||
{"uint32_t", TOK_TYPE},
|
||||
{"uint64_t", TOK_TYPE},
|
||||
{"size_t", TOK_TYPE},
|
||||
{"ssize_t", TOK_TYPE},
|
||||
{"ptrdiff_t", TOK_TYPE},
|
||||
{"intptr_t", TOK_TYPE},
|
||||
{"uintptr_t", TOK_TYPE},
|
||||
{"nullptr_t", TOK_TYPE},
|
||||
{"FILE", TOK_TYPE},
|
||||
// Project types
|
||||
{"U8", TOK_TYPE},
|
||||
{"U16", TOK_TYPE},
|
||||
{"U32", TOK_TYPE},
|
||||
{"U64", TOK_TYPE},
|
||||
{"S8", TOK_TYPE},
|
||||
{"S16", TOK_TYPE},
|
||||
{"S32", TOK_TYPE},
|
||||
{"S64", TOK_TYPE},
|
||||
{"B32", TOK_TYPE},
|
||||
{"F32", TOK_TYPE},
|
||||
{"F64", TOK_TYPE},
|
||||
|
||||
// Values
|
||||
{"false", TOK_VALUE},
|
||||
{"true", TOK_VALUE},
|
||||
{"NULL", TOK_VALUE},
|
||||
{"nullptr", TOK_VALUE},
|
||||
|
||||
// Modifiers
|
||||
{"const", TOK_MODIFIER},
|
||||
{"constexpr", TOK_MODIFIER},
|
||||
{"extern", TOK_MODIFIER},
|
||||
{"register", TOK_MODIFIER},
|
||||
{"signed", TOK_MODIFIER},
|
||||
{"static", TOK_MODIFIER},
|
||||
{"unsigned", TOK_MODIFIER},
|
||||
{"volatile", TOK_MODIFIER},
|
||||
{"thread_local", TOK_MODIFIER},
|
||||
{"_Thread_local", TOK_MODIFIER},
|
||||
};
|
||||
|
||||
#define C_KEYWORD_COUNT (S32)(sizeof(c_keywords) / sizeof(c_keywords[0]))
|
||||
|
||||
static Token_Type c_lookup_keyword(const char *word, S32 len) {
|
||||
for (S32 i = 0; i < C_KEYWORD_COUNT; i++) {
|
||||
const char *kw = c_keywords[i].word;
|
||||
S32 kwlen = (S32)strlen(kw);
|
||||
if (kwlen == len && memcmp(kw, word, len) == 0)
|
||||
return c_keywords[i].type;
|
||||
}
|
||||
return TOK_IDENTIFIER;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Character helpers
|
||||
|
||||
static B32 c_is_ident_start(char c) {
|
||||
return isalpha((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 c_is_ident_char(char c) {
|
||||
return isalnum((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 c_is_hex(char c) {
|
||||
return isdigit((unsigned char)c) ||
|
||||
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Individual token parsers
|
||||
|
||||
static Token c_parse_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
const char *begin = tok->t;
|
||||
while (tok->t < tok->max_t && c_is_ident_char(*tok->t))
|
||||
tok->t++;
|
||||
S32 len = (S32)(tok->t - begin);
|
||||
token.type = c_lookup_keyword(begin, len);
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
char start_char = *tok->t;
|
||||
tok->t++;
|
||||
if (tok->t >= tok->max_t) goto done;
|
||||
|
||||
if (start_char == '0' && tok->t < tok->max_t) {
|
||||
if (*tok->t == 'x' || *tok->t == 'X') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && c_is_hex(*tok->t)) tok->t++;
|
||||
goto suffixes;
|
||||
}
|
||||
if (*tok->t == 'b' || *tok->t == 'B') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (*tok->t == '0' || *tok->t == '1')) tok->t++;
|
||||
goto suffixes;
|
||||
}
|
||||
}
|
||||
|
||||
// Decimal / float
|
||||
{
|
||||
B32 seen_dot = 0;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '.')) {
|
||||
if (*tok->t == '.') {
|
||||
if (seen_dot) break;
|
||||
seen_dot = 1;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
// Float suffix
|
||||
if (tok->t < tok->max_t && seen_dot) {
|
||||
if (*tok->t == 'f' || *tok->t == 'F' || *tok->t == 'l' || *tok->t == 'L')
|
||||
tok->t++;
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
|
||||
suffixes:
|
||||
// Integer suffixes: u, l, ll, ul, ull, lu, llu
|
||||
if (tok->t < tok->max_t && (*tok->t == 'u' || *tok->t == 'U')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) tok->t++;
|
||||
}
|
||||
} else if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'u' || *tok->t == 'U')) tok->t++;
|
||||
}
|
||||
|
||||
done:
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_string(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
B32 escape = 0;
|
||||
tok->t++; // skip opening "
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') {
|
||||
if (*tok->t == '"' && !escape) { tok->t++; break; }
|
||||
escape = !escape && (*tok->t == '\\');
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_char_literal(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_CHAR_LITERAL;
|
||||
|
||||
tok->t++; // skip opening '
|
||||
if (tok->t < tok->max_t && *tok->t == '\\') {
|
||||
tok->t++; // escape char
|
||||
if (tok->t < tok->max_t && *tok->t != '\n') tok->t++; // the escaped char
|
||||
} else if (tok->t < tok->max_t && *tok->t != '\n' && *tok->t != '\'') {
|
||||
tok->t++; // the char
|
||||
}
|
||||
if (tok->t < tok->max_t && *tok->t == '\'') tok->t++; // closing '
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_slash_or_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
|
||||
tok->t++; // skip '/'
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.type = TOK_OPERATION;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
|
||||
if (*tok->t == '/') {
|
||||
// Line comment
|
||||
token.type = TOK_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
} else if (*tok->t == '*') {
|
||||
// Block comment
|
||||
token.type = TOK_MULTILINE_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '*' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '/') {
|
||||
tok->t += 2;
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
} else if (*tok->t == '=') {
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
} else {
|
||||
token.type = TOK_OPERATION;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_directive(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_DIRECTIVE;
|
||||
|
||||
tok->t++; // skip '#'
|
||||
// Skip whitespace between # and directive name
|
||||
while (tok->t < tok->max_t && (*tok->t == ' ' || *tok->t == '\t')) tok->t++;
|
||||
// Read directive name
|
||||
while (tok->t < tok->max_t && isalpha((unsigned char)*tok->t)) tok->t++;
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Two-char operator check
|
||||
static Token c_parse_operator(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_OPERATION;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (tok->t < tok->max_t) {
|
||||
char n = *tok->t;
|
||||
// Two-character operators
|
||||
if ((c == '=' && n == '=') || (c == '!' && n == '=') ||
|
||||
(c == '<' && n == '=') || (c == '>' && n == '=') ||
|
||||
(c == '+' && n == '=') || (c == '-' && n == '=') ||
|
||||
(c == '*' && n == '=') || (c == '%' && n == '=') ||
|
||||
(c == '&' && n == '=') || (c == '|' && n == '=') ||
|
||||
(c == '^' && n == '=') || (c == '+' && n == '+') ||
|
||||
(c == '-' && n == '-') || (c == '&' && n == '&') ||
|
||||
(c == '|' && n == '|') || (c == '<' && n == '<') ||
|
||||
(c == '>' && n == '>') || (c == '-' && n == '>')) {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle negative number literal: operator '-' followed by digit
|
||||
// (not done here -- handled by caller context if needed)
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Main get_next_token
|
||||
|
||||
static Token c_get_next_token(Tokenizer *tok) {
|
||||
tokenizer_eat_whitespace(tok);
|
||||
|
||||
Token token = {0};
|
||||
token.start = (S32)(tok->t - tok->buf);
|
||||
token.type = TOK_DEFAULT;
|
||||
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.len = 0;
|
||||
return token; // EOF
|
||||
}
|
||||
|
||||
tok->start_t = tok->t;
|
||||
char c = *tok->t;
|
||||
|
||||
if (c_is_ident_start(c)) {
|
||||
return c_parse_identifier(tok);
|
||||
}
|
||||
if (isdigit((unsigned char)c)) {
|
||||
return c_parse_number(tok);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '"': return c_parse_string(tok);
|
||||
case '\'': return c_parse_char_literal(tok);
|
||||
case '/': return c_parse_slash_or_comment(tok);
|
||||
case '#': return c_parse_directive(tok);
|
||||
|
||||
// Punctuation
|
||||
case ';': case ',': case '.':
|
||||
case '{': case '}': case '(': case ')': case '[': case ']':
|
||||
case '\\':
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Operators
|
||||
case '=': case '!': case '<': case '>':
|
||||
case '+': case '-': case '*': case '%':
|
||||
case '&': case '|': case '^': case '~':
|
||||
case '?': case ':':
|
||||
return c_parse_operator(tok);
|
||||
|
||||
default:
|
||||
token.type = TOK_INVALID;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// tokenize_c -- main entry point
|
||||
//
|
||||
// Tokenizes the full buffer and paints out_tokens[] with token types.
|
||||
// Second pass: retroactively marks identifiers before '(' as functions.
|
||||
|
||||
static void tokenize_c(const char *data, S32 length, U8 *out_tokens) {
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
|
||||
Tokenizer tok;
|
||||
tokenizer_init(&tok, data, length);
|
||||
|
||||
Token prev = {0};
|
||||
prev.type = TOK_DEFAULT;
|
||||
|
||||
while (tok.t < tok.max_t) {
|
||||
Token token = c_get_next_token(&tok);
|
||||
if (token.len == 0) break;
|
||||
|
||||
paint_token(out_tokens, token.start, token.len, token.type);
|
||||
|
||||
// Retroactively mark identifier before '(' as a function
|
||||
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
||||
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
||||
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
||||
}
|
||||
|
||||
prev = token;
|
||||
}
|
||||
}
|
||||
489
c/lexer/lexer_go.c
Normal file
489
c/lexer/lexer_go.c
Normal file
@@ -0,0 +1,489 @@
|
||||
// lexer_go.c -- Go language tokenizer
|
||||
//
|
||||
// Same pattern as lexer_c.c: a get_next_token() loop that advances a cursor
|
||||
// through the source, producing Token structs. After each token we paint
|
||||
// the per-byte token-type array. A second pass marks identifiers before '('
|
||||
// as functions.
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
////////////////////////////////
|
||||
// Go keyword tables
|
||||
|
||||
typedef struct GoKeywordEntry {
|
||||
const char *word;
|
||||
Token_Type type;
|
||||
} GoKeywordEntry;
|
||||
|
||||
static const GoKeywordEntry go_keywords[] = {
|
||||
// Control-flow & language keywords
|
||||
{"break", TOK_KEYWORD},
|
||||
{"case", TOK_KEYWORD},
|
||||
{"chan", TOK_KEYWORD},
|
||||
{"const", TOK_KEYWORD},
|
||||
{"continue", TOK_KEYWORD},
|
||||
{"default", TOK_KEYWORD},
|
||||
{"defer", TOK_KEYWORD},
|
||||
{"else", TOK_KEYWORD},
|
||||
{"fallthrough", TOK_KEYWORD},
|
||||
{"for", TOK_KEYWORD},
|
||||
{"func", TOK_KEYWORD},
|
||||
{"go", TOK_KEYWORD},
|
||||
{"goto", TOK_KEYWORD},
|
||||
{"if", TOK_KEYWORD},
|
||||
{"import", TOK_KEYWORD},
|
||||
{"interface", TOK_KEYWORD},
|
||||
{"map", TOK_KEYWORD},
|
||||
{"package", TOK_KEYWORD},
|
||||
{"range", TOK_KEYWORD},
|
||||
{"return", TOK_KEYWORD},
|
||||
{"select", TOK_KEYWORD},
|
||||
{"struct", TOK_KEYWORD},
|
||||
{"switch", TOK_KEYWORD},
|
||||
{"type", TOK_KEYWORD},
|
||||
{"var", TOK_KEYWORD},
|
||||
|
||||
// Built-in types
|
||||
{"bool", TOK_TYPE},
|
||||
{"byte", TOK_TYPE},
|
||||
{"complex64", TOK_TYPE},
|
||||
{"complex128", TOK_TYPE},
|
||||
{"error", TOK_TYPE},
|
||||
{"float32", TOK_TYPE},
|
||||
{"float64", TOK_TYPE},
|
||||
{"int", TOK_TYPE},
|
||||
{"int8", TOK_TYPE},
|
||||
{"int16", TOK_TYPE},
|
||||
{"int32", TOK_TYPE},
|
||||
{"int64", TOK_TYPE},
|
||||
{"rune", TOK_TYPE},
|
||||
{"string", TOK_TYPE},
|
||||
{"uint", TOK_TYPE},
|
||||
{"uint8", TOK_TYPE},
|
||||
{"uint16", TOK_TYPE},
|
||||
{"uint32", TOK_TYPE},
|
||||
{"uint64", TOK_TYPE},
|
||||
{"uintptr", TOK_TYPE},
|
||||
{"any", TOK_TYPE},
|
||||
{"comparable", TOK_TYPE},
|
||||
|
||||
// Built-in values
|
||||
{"true", TOK_VALUE},
|
||||
{"false", TOK_VALUE},
|
||||
{"nil", TOK_VALUE},
|
||||
{"iota", TOK_VALUE},
|
||||
|
||||
// Built-in functions (treated as modifiers for distinct coloring)
|
||||
{"append", TOK_MODIFIER},
|
||||
{"cap", TOK_MODIFIER},
|
||||
{"clear", TOK_MODIFIER},
|
||||
{"close", TOK_MODIFIER},
|
||||
{"complex", TOK_MODIFIER},
|
||||
{"copy", TOK_MODIFIER},
|
||||
{"delete", TOK_MODIFIER},
|
||||
{"imag", TOK_MODIFIER},
|
||||
{"len", TOK_MODIFIER},
|
||||
{"make", TOK_MODIFIER},
|
||||
{"max", TOK_MODIFIER},
|
||||
{"min", TOK_MODIFIER},
|
||||
{"new", TOK_MODIFIER},
|
||||
{"panic", TOK_MODIFIER},
|
||||
{"print", TOK_MODIFIER},
|
||||
{"println", TOK_MODIFIER},
|
||||
{"real", TOK_MODIFIER},
|
||||
{"recover", TOK_MODIFIER},
|
||||
};
|
||||
|
||||
#define GO_KEYWORD_COUNT (S32)(sizeof(go_keywords) / sizeof(go_keywords[0]))
|
||||
|
||||
static Token_Type go_lookup_keyword(const char *word, S32 len) {
|
||||
for (S32 i = 0; i < GO_KEYWORD_COUNT; i++) {
|
||||
const char *kw = go_keywords[i].word;
|
||||
S32 kwlen = (S32)strlen(kw);
|
||||
if (kwlen == len && memcmp(kw, word, len) == 0)
|
||||
return go_keywords[i].type;
|
||||
}
|
||||
return TOK_IDENTIFIER;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Character helpers
|
||||
|
||||
static B32 go_is_ident_start(char c) {
|
||||
return isalpha((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 go_is_ident_char(char c) {
|
||||
return isalnum((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 go_is_hex(char c) {
|
||||
return isdigit((unsigned char)c) ||
|
||||
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
static B32 go_is_octal(char c) {
|
||||
return c >= '0' && c <= '7';
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Individual token parsers
|
||||
|
||||
static Token go_parse_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
const char *begin = tok->t;
|
||||
while (tok->t < tok->max_t && go_is_ident_char(*tok->t))
|
||||
tok->t++;
|
||||
S32 len = (S32)(tok->t - begin);
|
||||
token.type = go_lookup_keyword(begin, len);
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token go_parse_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
char start_char = *tok->t;
|
||||
tok->t++;
|
||||
if (tok->t >= tok->max_t) goto done;
|
||||
|
||||
if (start_char == '0' && tok->t < tok->max_t) {
|
||||
// Hex: 0x or 0X
|
||||
if (*tok->t == 'x' || *tok->t == 'X') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (go_is_hex(*tok->t) || *tok->t == '_')) tok->t++;
|
||||
// Hex float: optional .hex_digits, optional p exponent
|
||||
if (tok->t < tok->max_t && *tok->t == '.') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (go_is_hex(*tok->t) || *tok->t == '_')) tok->t++;
|
||||
}
|
||||
if (tok->t < tok->max_t && (*tok->t == 'p' || *tok->t == 'P')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_')) tok->t++;
|
||||
}
|
||||
goto imaginary;
|
||||
}
|
||||
// Octal: 0o or 0O
|
||||
if (*tok->t == 'o' || *tok->t == 'O') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (go_is_octal(*tok->t) || *tok->t == '_')) tok->t++;
|
||||
goto imaginary;
|
||||
}
|
||||
// Binary: 0b or 0B
|
||||
if (*tok->t == 'b' || *tok->t == 'B') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (*tok->t == '0' || *tok->t == '1' || *tok->t == '_')) tok->t++;
|
||||
goto imaginary;
|
||||
}
|
||||
}
|
||||
|
||||
// Decimal or float (also handles legacy octal like 0755)
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
|
||||
// Decimal float: .digits
|
||||
if (tok->t < tok->max_t && *tok->t == '.') {
|
||||
// Check next char is digit to avoid consuming '..' or method calls
|
||||
if ((tok->t + 1) < tok->max_t && isdigit((unsigned char)*(tok->t + 1))) {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_')) tok->t++;
|
||||
} else if ((tok->t + 1) >= tok->max_t || !go_is_ident_start(*(tok->t + 1))) {
|
||||
// Trailing dot like "1." -- still a float in Go
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_')) tok->t++;
|
||||
}
|
||||
|
||||
imaginary:
|
||||
// Imaginary suffix
|
||||
if (tok->t < tok->max_t && *tok->t == 'i')
|
||||
tok->t++;
|
||||
|
||||
done:
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Dot-starting float: .5, .123e4
|
||||
static Token go_parse_dot_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
tok->t++; // skip '.'
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_')) tok->t++;
|
||||
}
|
||||
// Imaginary
|
||||
if (tok->t < tok->max_t && *tok->t == 'i')
|
||||
tok->t++;
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token go_parse_string(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
B32 escape = 0;
|
||||
tok->t++; // skip opening "
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') {
|
||||
if (*tok->t == '"' && !escape) { tok->t++; break; }
|
||||
escape = !escape && (*tok->t == '\\');
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Raw string literal: `...`
|
||||
static Token go_parse_raw_string(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
tok->t++; // skip opening `
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '`') { tok->t++; break; }
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Rune literal: 'x', '\n', '\u0041', etc.
|
||||
static Token go_parse_rune_literal(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_CHAR_LITERAL;
|
||||
|
||||
tok->t++; // skip opening '
|
||||
if (tok->t < tok->max_t && *tok->t == '\\') {
|
||||
tok->t++; // skip backslash
|
||||
// Consume escape sequence characters
|
||||
if (tok->t < tok->max_t) {
|
||||
char esc = *tok->t;
|
||||
tok->t++;
|
||||
if (esc == 'x') {
|
||||
// \xNN
|
||||
for (int i = 0; i < 2 && tok->t < tok->max_t && go_is_hex(*tok->t); i++) tok->t++;
|
||||
} else if (esc == 'u') {
|
||||
// \uNNNN
|
||||
for (int i = 0; i < 4 && tok->t < tok->max_t && go_is_hex(*tok->t); i++) tok->t++;
|
||||
} else if (esc == 'U') {
|
||||
// \UNNNNNNNN
|
||||
for (int i = 0; i < 8 && tok->t < tok->max_t && go_is_hex(*tok->t); i++) tok->t++;
|
||||
} else if (go_is_octal(esc)) {
|
||||
// \NNN
|
||||
for (int i = 0; i < 2 && tok->t < tok->max_t && go_is_octal(*tok->t); i++) tok->t++;
|
||||
}
|
||||
// else: simple escape like \n, \t, \\, \' -- already consumed
|
||||
}
|
||||
} else if (tok->t < tok->max_t && *tok->t != '\n' && *tok->t != '\'') {
|
||||
tok->t++; // the rune character
|
||||
}
|
||||
if (tok->t < tok->max_t && *tok->t == '\'') tok->t++; // closing '
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token go_parse_slash_or_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
|
||||
tok->t++; // skip '/'
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.type = TOK_OPERATION;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
|
||||
if (*tok->t == '/') {
|
||||
// Line comment
|
||||
token.type = TOK_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
} else if (*tok->t == '*') {
|
||||
// Block comment
|
||||
token.type = TOK_MULTILINE_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '*' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '/') {
|
||||
tok->t += 2;
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
} else if (*tok->t == '=') {
|
||||
// /=
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
} else {
|
||||
token.type = TOK_OPERATION;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token go_parse_operator(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_OPERATION;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (tok->t < tok->max_t) {
|
||||
char n = *tok->t;
|
||||
// Three-character operators
|
||||
if (c == '<' && n == '<' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '=') {
|
||||
tok->t += 2; goto done;
|
||||
}
|
||||
if (c == '>' && n == '>' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '=') {
|
||||
tok->t += 2; goto done;
|
||||
}
|
||||
if (c == '&' && n == '^' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '=') {
|
||||
tok->t += 2; goto done;
|
||||
}
|
||||
// := (short variable declaration)
|
||||
if (c == ':' && n == '=') { tok->t++; goto done; }
|
||||
// Two-character operators
|
||||
if ((c == '=' && n == '=') || (c == '!' && n == '=') ||
|
||||
(c == '<' && n == '=') || (c == '>' && n == '=') ||
|
||||
(c == '+' && n == '=') || (c == '-' && n == '=') ||
|
||||
(c == '*' && n == '=') || (c == '%' && n == '=') ||
|
||||
(c == '&' && n == '=') || (c == '|' && n == '=') ||
|
||||
(c == '^' && n == '=') || (c == '+' && n == '+') ||
|
||||
(c == '-' && n == '-') || (c == '&' && n == '&') ||
|
||||
(c == '|' && n == '|') || (c == '<' && n == '<') ||
|
||||
(c == '>' && n == '>') || (c == '-' && n == '>') ||
|
||||
(c == '<' && n == '-') || (c == '&' && n == '^')) {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Main get_next_token
|
||||
|
||||
static Token go_get_next_token(Tokenizer *tok) {
|
||||
tokenizer_eat_whitespace(tok);
|
||||
|
||||
Token token = {0};
|
||||
token.start = (S32)(tok->t - tok->buf);
|
||||
token.type = TOK_DEFAULT;
|
||||
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.len = 0;
|
||||
return token; // EOF
|
||||
}
|
||||
|
||||
tok->start_t = tok->t;
|
||||
char c = *tok->t;
|
||||
|
||||
if (go_is_ident_start(c)) {
|
||||
return go_parse_identifier(tok);
|
||||
}
|
||||
if (isdigit((unsigned char)c)) {
|
||||
return go_parse_number(tok);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '"': return go_parse_string(tok);
|
||||
case '`': return go_parse_raw_string(tok);
|
||||
case '\'': return go_parse_rune_literal(tok);
|
||||
case '/': return go_parse_slash_or_comment(tok);
|
||||
|
||||
// Dot: could start a float literal like .5
|
||||
case '.':
|
||||
if ((tok->t + 1) < tok->max_t && isdigit((unsigned char)*(tok->t + 1))) {
|
||||
return go_parse_dot_number(tok);
|
||||
}
|
||||
// Ellipsis ... or just punctuation
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && *tok->t == '.' &&
|
||||
(tok->t + 1) < tok->max_t && *(tok->t + 1) == '.') {
|
||||
tok->t += 2; // consume ...
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
|
||||
// Punctuation
|
||||
case ';': case ',':
|
||||
case '{': case '}': case '(': case ')': case '[': case ']':
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Operators
|
||||
case '=': case '!': case '<': case '>':
|
||||
case '+': case '-': case '*': case '%':
|
||||
case '&': case '|': case '^': case '~':
|
||||
case ':':
|
||||
return go_parse_operator(tok);
|
||||
|
||||
default:
|
||||
token.type = TOK_INVALID;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// tokenize_go -- main entry point
|
||||
//
|
||||
// Tokenizes the full buffer and paints out_tokens[] with token types.
|
||||
// Second pass: retroactively marks identifiers before '(' as functions.
|
||||
|
||||
static void tokenize_go(const char *data, S32 length, U8 *out_tokens) {
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
|
||||
Tokenizer tok;
|
||||
tokenizer_init(&tok, data, length);
|
||||
|
||||
Token prev = {0};
|
||||
prev.type = TOK_DEFAULT;
|
||||
|
||||
while (tok.t < tok.max_t) {
|
||||
Token token = go_get_next_token(&tok);
|
||||
if (token.len == 0) break;
|
||||
|
||||
paint_token(out_tokens, token.start, token.len, token.type);
|
||||
|
||||
// Retroactively mark identifier before '(' as a function
|
||||
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
||||
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
||||
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
||||
}
|
||||
|
||||
prev = token;
|
||||
}
|
||||
}
|
||||
651
c/lexer/lexer_js.c
Normal file
651
c/lexer/lexer_js.c
Normal file
@@ -0,0 +1,651 @@
|
||||
// lexer_js.c -- JavaScript language tokenizer
|
||||
//
|
||||
// Supports nested tagged template literals with ${...} interpolation.
|
||||
// Uses a depth stack to track template literal nesting so that expressions
|
||||
// inside ${...} can themselves contain template literals at arbitrary depth.
|
||||
//
|
||||
// Example: html`outer ${css`inner ${x}`} rest`
|
||||
// ^^^^ ^^^^^^ ^^^ ^^^^^^ ^ ^ ^^^^^^
|
||||
// func string func str id str string
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
////////////////////////////////
|
||||
// JavaScript keyword tables
|
||||
|
||||
typedef struct JSKeywordEntry {
|
||||
const char *word;
|
||||
Token_Type type;
|
||||
} JSKeywordEntry;
|
||||
|
||||
static const JSKeywordEntry js_keywords[] = {
|
||||
// Control-flow & language keywords
|
||||
{"break", TOK_KEYWORD},
|
||||
{"case", TOK_KEYWORD},
|
||||
{"catch", TOK_KEYWORD},
|
||||
{"class", TOK_KEYWORD},
|
||||
{"const", TOK_KEYWORD},
|
||||
{"continue", TOK_KEYWORD},
|
||||
{"debugger", TOK_KEYWORD},
|
||||
{"default", TOK_KEYWORD},
|
||||
{"delete", TOK_KEYWORD},
|
||||
{"do", TOK_KEYWORD},
|
||||
{"else", TOK_KEYWORD},
|
||||
{"extends", TOK_KEYWORD},
|
||||
{"finally", TOK_KEYWORD},
|
||||
{"for", TOK_KEYWORD},
|
||||
{"function", TOK_KEYWORD},
|
||||
{"if", TOK_KEYWORD},
|
||||
{"in", TOK_KEYWORD},
|
||||
{"instanceof", TOK_KEYWORD},
|
||||
{"let", TOK_KEYWORD},
|
||||
{"new", TOK_KEYWORD},
|
||||
{"of", TOK_KEYWORD},
|
||||
{"return", TOK_KEYWORD},
|
||||
{"switch", TOK_KEYWORD},
|
||||
{"throw", TOK_KEYWORD},
|
||||
{"try", TOK_KEYWORD},
|
||||
{"typeof", TOK_KEYWORD},
|
||||
{"var", TOK_KEYWORD},
|
||||
{"void", TOK_KEYWORD},
|
||||
{"while", TOK_KEYWORD},
|
||||
{"with", TOK_KEYWORD},
|
||||
{"yield", TOK_KEYWORD},
|
||||
{"async", TOK_KEYWORD},
|
||||
{"await", TOK_KEYWORD},
|
||||
|
||||
// Module keywords (directive-style coloring)
|
||||
{"import", TOK_DIRECTIVE},
|
||||
{"export", TOK_DIRECTIVE},
|
||||
{"from", TOK_DIRECTIVE},
|
||||
{"as", TOK_DIRECTIVE},
|
||||
|
||||
// Values
|
||||
{"true", TOK_VALUE},
|
||||
{"false", TOK_VALUE},
|
||||
{"null", TOK_VALUE},
|
||||
{"undefined", TOK_VALUE},
|
||||
{"NaN", TOK_VALUE},
|
||||
{"Infinity", TOK_VALUE},
|
||||
{"this", TOK_VALUE},
|
||||
{"super", TOK_VALUE},
|
||||
|
||||
// Modifiers / contextual keywords
|
||||
{"static", TOK_MODIFIER},
|
||||
{"get", TOK_MODIFIER},
|
||||
{"set", TOK_MODIFIER},
|
||||
};
|
||||
|
||||
#define JS_KEYWORD_COUNT (S32)(sizeof(js_keywords) / sizeof(js_keywords[0]))
|
||||
|
||||
static Token_Type js_lookup_keyword(const char *word, S32 len) {
|
||||
for (S32 i = 0; i < JS_KEYWORD_COUNT; i++) {
|
||||
const char *kw = js_keywords[i].word;
|
||||
S32 kwlen = (S32)strlen(kw);
|
||||
if (kwlen == len && memcmp(kw, word, len) == 0)
|
||||
return js_keywords[i].type;
|
||||
}
|
||||
return TOK_IDENTIFIER;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Character helpers
|
||||
|
||||
static B32 js_is_ident_start(char c) {
|
||||
return isalpha((unsigned char)c) || c == '_' || c == '$';
|
||||
}
|
||||
|
||||
static B32 js_is_ident_char(char c) {
|
||||
return isalnum((unsigned char)c) || c == '_' || c == '$';
|
||||
}
|
||||
|
||||
static B32 js_is_hex(char c) {
|
||||
return isdigit((unsigned char)c) ||
|
||||
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Individual token parsers
|
||||
|
||||
static Token js_parse_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
const char *begin = tok->t;
|
||||
while (tok->t < tok->max_t && js_is_ident_char(*tok->t))
|
||||
tok->t++;
|
||||
S32 len = (S32)(tok->t - begin);
|
||||
token.type = js_lookup_keyword(begin, len);
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token js_parse_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
char start_char = *tok->t;
|
||||
tok->t++;
|
||||
if (tok->t >= tok->max_t) goto done;
|
||||
|
||||
if (start_char == '0' && tok->t < tok->max_t) {
|
||||
// Hex: 0x / 0X
|
||||
if (*tok->t == 'x' || *tok->t == 'X') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (js_is_hex(*tok->t) || *tok->t == '_')) tok->t++;
|
||||
if (tok->t < tok->max_t && *tok->t == 'n') tok->t++; // BigInt
|
||||
goto done;
|
||||
}
|
||||
// Octal: 0o / 0O
|
||||
if (*tok->t == 'o' || *tok->t == 'O') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && ((*tok->t >= '0' && *tok->t <= '7') || *tok->t == '_')) tok->t++;
|
||||
if (tok->t < tok->max_t && *tok->t == 'n') tok->t++;
|
||||
goto done;
|
||||
}
|
||||
// Binary: 0b / 0B
|
||||
if (*tok->t == 'b' || *tok->t == 'B') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (*tok->t == '0' || *tok->t == '1' || *tok->t == '_')) tok->t++;
|
||||
if (tok->t < tok->max_t && *tok->t == 'n') tok->t++;
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
|
||||
// Decimal digits
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
|
||||
// Float: .digits
|
||||
if (tok->t < tok->max_t && *tok->t == '.') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
// BigInt suffix
|
||||
if (tok->t < tok->max_t && *tok->t == 'n') tok->t++;
|
||||
|
||||
done:
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Float starting with dot: .5, .123e4
|
||||
static Token js_parse_dot_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
tok->t++; // skip '.'
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '_'))
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// String literal with given quote character (" or ')
|
||||
static Token js_parse_string(Tokenizer *tok, char quote) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
B32 escape = 0;
|
||||
tok->t++; // skip opening quote
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') {
|
||||
if (*tok->t == quote && !escape) { tok->t++; break; }
|
||||
escape = !escape && (*tok->t == '\\');
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// / followed by / or * (comments), or /= (divide-assign), or bare / (divide)
|
||||
static Token js_parse_slash_or_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
|
||||
tok->t++; // skip '/'
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.type = TOK_OPERATION;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
|
||||
if (*tok->t == '/') {
|
||||
// Line comment
|
||||
token.type = TOK_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
} else if (*tok->t == '*') {
|
||||
// Block comment
|
||||
token.type = TOK_MULTILINE_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '*' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '/') {
|
||||
tok->t += 2;
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
} else if (*tok->t == '=') {
|
||||
// /=
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
} else {
|
||||
// bare / (division)
|
||||
token.type = TOK_OPERATION;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Regex literal: /pattern/flags
|
||||
static Token js_parse_regex(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL; // color regex like strings
|
||||
|
||||
B32 escape = 0;
|
||||
B32 in_class = 0; // inside character class [...]
|
||||
tok->t++; // skip opening /
|
||||
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') {
|
||||
if (escape) {
|
||||
escape = 0;
|
||||
tok->t++;
|
||||
continue;
|
||||
}
|
||||
if (*tok->t == '\\') {
|
||||
escape = 1;
|
||||
tok->t++;
|
||||
continue;
|
||||
}
|
||||
if (*tok->t == '[') { in_class = 1; tok->t++; continue; }
|
||||
if (*tok->t == ']') { in_class = 0; tok->t++; continue; }
|
||||
if (*tok->t == '/' && !in_class) {
|
||||
tok->t++; // closing /
|
||||
// Consume flags: d, g, i, m, s, u, v, y
|
||||
while (tok->t < tok->max_t && isalpha((unsigned char)*tok->t)) tok->t++;
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Operators (handles multi-character sequences)
|
||||
static Token js_parse_operator(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_OPERATION;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (tok->t < tok->max_t) {
|
||||
char n = *tok->t;
|
||||
|
||||
// Three-character (and four-character) operators
|
||||
if ((tok->t + 1) < tok->max_t) {
|
||||
char nn = *(tok->t + 1);
|
||||
// >>> and >>>=
|
||||
if (c == '>' && n == '>' && nn == '>') {
|
||||
tok->t += 2;
|
||||
if (tok->t < tok->max_t && *tok->t == '=') tok->t++;
|
||||
goto done;
|
||||
}
|
||||
// ===
|
||||
if (c == '=' && n == '=' && nn == '=') { tok->t += 2; goto done; }
|
||||
// !==
|
||||
if (c == '!' && n == '=' && nn == '=') { tok->t += 2; goto done; }
|
||||
// **=
|
||||
if (c == '*' && n == '*' && nn == '=') { tok->t += 2; goto done; }
|
||||
// <<=
|
||||
if (c == '<' && n == '<' && nn == '=') { tok->t += 2; goto done; }
|
||||
// >>=
|
||||
if (c == '>' && n == '>' && nn == '=') { tok->t += 2; goto done; }
|
||||
// &&=
|
||||
if (c == '&' && n == '&' && nn == '=') { tok->t += 2; goto done; }
|
||||
// ||=
|
||||
if (c == '|' && n == '|' && nn == '=') { tok->t += 2; goto done; }
|
||||
// ??=
|
||||
if (c == '?' && n == '?' && nn == '=') { tok->t += 2; goto done; }
|
||||
}
|
||||
|
||||
// Two-character operators
|
||||
if ((c == '=' && n == '=') || (c == '!' && n == '=') ||
|
||||
(c == '<' && n == '=') || (c == '>' && n == '=') ||
|
||||
(c == '+' && n == '=') || (c == '-' && n == '=') ||
|
||||
(c == '*' && n == '=') || (c == '%' && n == '=') ||
|
||||
(c == '&' && n == '=') || (c == '|' && n == '=') ||
|
||||
(c == '^' && n == '=') || (c == '+' && n == '+') ||
|
||||
(c == '-' && n == '-') || (c == '&' && n == '&') ||
|
||||
(c == '|' && n == '|') || (c == '<' && n == '<') ||
|
||||
(c == '>' && n == '>') || (c == '=' && n == '>') ||
|
||||
(c == '*' && n == '*') || (c == '?' && n == '?') ||
|
||||
(c == '?' && n == '.')) {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Regex vs division heuristic
|
||||
//
|
||||
// After identifiers, numbers, values, strings, ) and ] -- slash is division.
|
||||
// After keywords, operators, most punctuation, start of file -- slash is regex.
|
||||
|
||||
static B32 js_slash_is_regex(Token prev, const char *data) {
|
||||
switch (prev.type) {
|
||||
case TOK_DEFAULT: return 1; // start of file / no previous token
|
||||
case TOK_KEYWORD: return 1; // e.g. return /regex/
|
||||
case TOK_DIRECTIVE: return 1; // after import/export
|
||||
case TOK_OPERATION:
|
||||
// After ++ or --, it's division (x++ / y)
|
||||
if (prev.len == 2) {
|
||||
char c0 = data[prev.start];
|
||||
char c1 = data[prev.start + 1];
|
||||
if ((c0 == '+' && c1 == '+') || (c0 == '-' && c1 == '-'))
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
case TOK_PUNCTUATION:
|
||||
// After ) or ], it's division
|
||||
if (prev.len == 1) {
|
||||
char c = data[prev.start];
|
||||
if (c == ')' || c == ']') return 0;
|
||||
}
|
||||
return 1;
|
||||
case TOK_IDENTIFIER:
|
||||
case TOK_FUNCTION:
|
||||
case TOK_NUMBER:
|
||||
case TOK_STRING_LITERAL:
|
||||
case TOK_CHAR_LITERAL:
|
||||
case TOK_VALUE:
|
||||
return 0; // division
|
||||
default:
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Template literal content scanner
|
||||
//
|
||||
// Scans the string content inside a template literal (the text between
|
||||
// backticks, or between a closing } and the next ${ or closing backtick).
|
||||
// Paints each byte as TOK_STRING_LITERAL.
|
||||
//
|
||||
// Returns 1 if we hit ${ (caller should enter expression mode),
|
||||
// 0 if we hit closing ` or EOF (template is done).
|
||||
|
||||
static int js_scan_template_content(Tokenizer *tok, U8 *out_tokens) {
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '`') {
|
||||
// Closing backtick -- template done
|
||||
S32 pos = (S32)(tok->t - tok->buf);
|
||||
out_tokens[pos] = TOK_STRING_LITERAL;
|
||||
tok->t++;
|
||||
return 0;
|
||||
}
|
||||
if (*tok->t == '$' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '{') {
|
||||
// Interpolation start -- paint ${ as punctuation
|
||||
S32 pos = (S32)(tok->t - tok->buf);
|
||||
out_tokens[pos] = TOK_PUNCTUATION;
|
||||
out_tokens[pos + 1] = TOK_PUNCTUATION;
|
||||
tok->t += 2;
|
||||
return 1;
|
||||
}
|
||||
if (*tok->t == '\\' && (tok->t + 1) < tok->max_t) {
|
||||
// Escape sequence -- paint both chars as string
|
||||
S32 pos = (S32)(tok->t - tok->buf);
|
||||
out_tokens[pos] = TOK_STRING_LITERAL;
|
||||
out_tokens[pos + 1] = TOK_STRING_LITERAL;
|
||||
tok->t += 2;
|
||||
continue;
|
||||
}
|
||||
// Regular string character
|
||||
S32 pos = (S32)(tok->t - tok->buf);
|
||||
out_tokens[pos] = TOK_STRING_LITERAL;
|
||||
tok->t++;
|
||||
}
|
||||
return 0; // EOF -- unclosed template
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Get next non-template token
|
||||
//
|
||||
// Called by the main loop for normal expression parsing. Template backticks
|
||||
// and interpolation-closing braces are handled directly by the main loop
|
||||
// before this function is reached.
|
||||
|
||||
static Token js_get_next_token(Tokenizer *tok, Token prev, const char *data) {
|
||||
tokenizer_eat_whitespace(tok);
|
||||
|
||||
Token token = {0};
|
||||
token.start = (S32)(tok->t - tok->buf);
|
||||
token.type = TOK_DEFAULT;
|
||||
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.len = 0;
|
||||
return token;
|
||||
}
|
||||
|
||||
tok->start_t = tok->t;
|
||||
char c = *tok->t;
|
||||
|
||||
if (js_is_ident_start(c)) {
|
||||
return js_parse_identifier(tok);
|
||||
}
|
||||
if (isdigit((unsigned char)c)) {
|
||||
return js_parse_number(tok);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '"': return js_parse_string(tok, '"');
|
||||
case '\'': return js_parse_string(tok, '\'');
|
||||
|
||||
case '/':
|
||||
// Check for comment first (// or /*)
|
||||
if ((tok->t + 1) < tok->max_t) {
|
||||
char n = *(tok->t + 1);
|
||||
if (n == '/' || n == '*')
|
||||
return js_parse_slash_or_comment(tok);
|
||||
}
|
||||
// Regex or division based on context
|
||||
if (js_slash_is_regex(prev, data))
|
||||
return js_parse_regex(tok);
|
||||
return js_parse_slash_or_comment(tok);
|
||||
|
||||
// Dot: float literal (.5), spread (...), or member access
|
||||
case '.':
|
||||
if ((tok->t + 1) < tok->max_t && isdigit((unsigned char)*(tok->t + 1))) {
|
||||
return js_parse_dot_number(tok);
|
||||
}
|
||||
if ((tok->t + 2) < tok->max_t && *(tok->t + 1) == '.' && *(tok->t + 2) == '.') {
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t += 3;
|
||||
token.len = 3;
|
||||
return token;
|
||||
}
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Punctuation
|
||||
case ';': case ',':
|
||||
case '(': case ')': case '[': case ']':
|
||||
case '{': case '}':
|
||||
case ':':
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Operators
|
||||
case '=': case '!': case '<': case '>':
|
||||
case '+': case '-': case '*': case '%':
|
||||
case '&': case '|': case '^': case '~':
|
||||
case '?':
|
||||
return js_parse_operator(tok);
|
||||
|
||||
// Private class fields: #name
|
||||
case '#':
|
||||
token.type = TOK_IDENTIFIER;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && js_is_ident_char(*tok->t)) tok->t++;
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
|
||||
// Decorators: @name
|
||||
case '@':
|
||||
token.type = TOK_DIRECTIVE;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && js_is_ident_char(*tok->t)) tok->t++;
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
|
||||
default:
|
||||
token.type = TOK_INVALID;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// tokenize_js -- main entry point
|
||||
//
|
||||
// Tokenizes the full buffer and paints out_tokens[] with token types.
|
||||
//
|
||||
// Template literal nesting is tracked with a depth stack. Each level
|
||||
// records how many unmatched { braces exist in the current interpolation
|
||||
// expression. When a } is encountered and the brace count is zero, it
|
||||
// closes the interpolation and we resume scanning template string content.
|
||||
//
|
||||
// html`text ${obj.x} more ${css`inner ${y}`} end`
|
||||
// |str ||expr | str || |str || || str|
|
||||
// ^punc ^punc ^punc ^ ^^
|
||||
// ${ } ${ } }` (nesting!)
|
||||
|
||||
#define JS_MAX_TEMPLATE_DEPTH 32
|
||||
|
||||
static void tokenize_js(const char *data, S32 length, U8 *out_tokens) {
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
|
||||
Tokenizer tok;
|
||||
tokenizer_init(&tok, data, length);
|
||||
|
||||
S32 tmpl_depth = 0;
|
||||
S32 brace_count[JS_MAX_TEMPLATE_DEPTH];
|
||||
memset(brace_count, 0, sizeof(brace_count));
|
||||
|
||||
Token prev = {0};
|
||||
prev.type = TOK_DEFAULT;
|
||||
|
||||
while (tok.t < tok.max_t) {
|
||||
tokenizer_eat_whitespace(&tok);
|
||||
if (tok.t >= tok.max_t) break;
|
||||
|
||||
char c = *tok.t;
|
||||
|
||||
// ---- Template literal: opening backtick ----
|
||||
// This handles both top-level template literals and nested ones
|
||||
// (e.g. a tagged template inside a ${...} interpolation).
|
||||
if (c == '`') {
|
||||
S32 pos = (S32)(tok.t - tok.buf);
|
||||
out_tokens[pos] = TOK_STRING_LITERAL;
|
||||
tok.t++;
|
||||
|
||||
int result = js_scan_template_content(&tok, out_tokens);
|
||||
if (result == 1) {
|
||||
// Hit ${ -- push template depth
|
||||
if (tmpl_depth < JS_MAX_TEMPLATE_DEPTH) {
|
||||
brace_count[tmpl_depth] = 0;
|
||||
tmpl_depth++;
|
||||
}
|
||||
}
|
||||
// result == 0: self-contained template (no interpolation, or
|
||||
// all interpolations already resolved recursively)
|
||||
|
||||
prev.type = TOK_STRING_LITERAL;
|
||||
prev.start = pos;
|
||||
prev.len = (S32)(tok.t - tok.buf) - pos;
|
||||
continue;
|
||||
}
|
||||
|
||||
// ---- Closing brace that ends a template interpolation ----
|
||||
// When we're inside a template expression and the brace count is
|
||||
// zero, this } closes the ${...} and we resume string scanning.
|
||||
if (c == '}' && tmpl_depth > 0 && brace_count[tmpl_depth - 1] == 0) {
|
||||
S32 pos = (S32)(tok.t - tok.buf);
|
||||
out_tokens[pos] = TOK_PUNCTUATION;
|
||||
tok.t++;
|
||||
tmpl_depth--;
|
||||
|
||||
// Resume template string content
|
||||
int result = js_scan_template_content(&tok, out_tokens);
|
||||
if (result == 1) {
|
||||
// Hit another ${ -- re-enter expression mode
|
||||
if (tmpl_depth < JS_MAX_TEMPLATE_DEPTH) {
|
||||
brace_count[tmpl_depth] = 0;
|
||||
tmpl_depth++;
|
||||
}
|
||||
}
|
||||
// result == 0: template closed with backtick
|
||||
|
||||
prev.type = TOK_STRING_LITERAL;
|
||||
prev.start = pos;
|
||||
prev.len = (S32)(tok.t - tok.buf) - pos;
|
||||
continue;
|
||||
}
|
||||
|
||||
// ---- Normal token ----
|
||||
tok.start_t = tok.t;
|
||||
Token token = js_get_next_token(&tok, prev, data);
|
||||
if (token.len == 0) break;
|
||||
|
||||
paint_token(out_tokens, token.start, token.len, token.type);
|
||||
|
||||
// Track brace depth for template interpolation
|
||||
if (tmpl_depth > 0 && token.type == TOK_PUNCTUATION && token.len == 1) {
|
||||
char tc = data[token.start];
|
||||
if (tc == '{') brace_count[tmpl_depth - 1]++;
|
||||
else if (tc == '}') brace_count[tmpl_depth - 1]--;
|
||||
}
|
||||
|
||||
// Retroactively mark identifier before '(' as function
|
||||
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
||||
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
||||
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
||||
}
|
||||
|
||||
prev = token;
|
||||
}
|
||||
}
|
||||
422
c/lexer/lexer_lua.c
Normal file
422
c/lexer/lexer_lua.c
Normal file
@@ -0,0 +1,422 @@
|
||||
// lexer_lua.c -- Lua language tokenizer
|
||||
//
|
||||
// Same pattern as lexer_go.c: a get_next_token() loop that advances a cursor
|
||||
// through the source, producing Token structs. After each token we paint
|
||||
// the per-byte token-type array. A second pass marks identifiers before '('
|
||||
// as functions.
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
////////////////////////////////
|
||||
// Lua keyword tables
|
||||
|
||||
typedef struct LuaKeywordEntry {
|
||||
const char *word;
|
||||
Token_Type type;
|
||||
} LuaKeywordEntry;
|
||||
|
||||
static const LuaKeywordEntry lua_keywords[] = {
|
||||
// Keywords
|
||||
{"and", TOK_KEYWORD},
|
||||
{"break", TOK_KEYWORD},
|
||||
{"do", TOK_KEYWORD},
|
||||
{"else", TOK_KEYWORD},
|
||||
{"elseif", TOK_KEYWORD},
|
||||
{"end", TOK_KEYWORD},
|
||||
{"for", TOK_KEYWORD},
|
||||
{"function", TOK_KEYWORD},
|
||||
{"goto", TOK_KEYWORD},
|
||||
{"if", TOK_KEYWORD},
|
||||
{"in", TOK_KEYWORD},
|
||||
{"local", TOK_KEYWORD},
|
||||
{"not", TOK_KEYWORD},
|
||||
{"or", TOK_KEYWORD},
|
||||
{"repeat", TOK_KEYWORD},
|
||||
{"return", TOK_KEYWORD},
|
||||
{"then", TOK_KEYWORD},
|
||||
{"until", TOK_KEYWORD},
|
||||
{"while", TOK_KEYWORD},
|
||||
|
||||
// Values
|
||||
{"true", TOK_VALUE},
|
||||
{"false", TOK_VALUE},
|
||||
{"nil", TOK_VALUE},
|
||||
|
||||
// Built-in globals / types (use TOK_TYPE for distinct coloring)
|
||||
{"self", TOK_TYPE},
|
||||
|
||||
// Built-in functions (TOK_MODIFIER for distinct coloring)
|
||||
{"assert", TOK_MODIFIER},
|
||||
{"collectgarbage", TOK_MODIFIER},
|
||||
{"dofile", TOK_MODIFIER},
|
||||
{"error", TOK_MODIFIER},
|
||||
{"getmetatable", TOK_MODIFIER},
|
||||
{"ipairs", TOK_MODIFIER},
|
||||
{"load", TOK_MODIFIER},
|
||||
{"loadfile", TOK_MODIFIER},
|
||||
{"next", TOK_MODIFIER},
|
||||
{"pairs", TOK_MODIFIER},
|
||||
{"pcall", TOK_MODIFIER},
|
||||
{"print", TOK_MODIFIER},
|
||||
{"rawequal", TOK_MODIFIER},
|
||||
{"rawget", TOK_MODIFIER},
|
||||
{"rawlen", TOK_MODIFIER},
|
||||
{"rawset", TOK_MODIFIER},
|
||||
{"require", TOK_MODIFIER},
|
||||
{"select", TOK_MODIFIER},
|
||||
{"setmetatable", TOK_MODIFIER},
|
||||
{"tonumber", TOK_MODIFIER},
|
||||
{"tostring", TOK_MODIFIER},
|
||||
{"type", TOK_MODIFIER},
|
||||
{"unpack", TOK_MODIFIER},
|
||||
{"xpcall", TOK_MODIFIER},
|
||||
};
|
||||
|
||||
#define LUA_KEYWORD_COUNT (S32)(sizeof(lua_keywords) / sizeof(lua_keywords[0]))
|
||||
|
||||
static Token_Type lua_lookup_keyword(const char *word, S32 len) {
|
||||
for (S32 i = 0; i < LUA_KEYWORD_COUNT; i++) {
|
||||
const char *kw = lua_keywords[i].word;
|
||||
S32 kwlen = (S32)strlen(kw);
|
||||
if (kwlen == len && memcmp(kw, word, len) == 0)
|
||||
return lua_keywords[i].type;
|
||||
}
|
||||
return TOK_IDENTIFIER;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Character helpers
|
||||
|
||||
static B32 lua_is_ident_start(char c) {
|
||||
return isalpha((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 lua_is_ident_char(char c) {
|
||||
return isalnum((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 lua_is_hex(char c) {
|
||||
return isdigit((unsigned char)c) ||
|
||||
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Long bracket level detection
|
||||
// Returns the level (number of '=' signs) if we're at a long bracket opening
|
||||
// like [[ or [==[ etc. Returns -1 if not a long bracket.
|
||||
|
||||
static S32 lua_long_bracket_level(const char *p, const char *max_p) {
|
||||
if (p >= max_p || *p != '[') return -1;
|
||||
p++;
|
||||
S32 level = 0;
|
||||
while (p < max_p && *p == '=') { level++; p++; }
|
||||
if (p < max_p && *p == '[') return level;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Scan past the closing long bracket of the given level.
|
||||
// Cursor should be right after the opening [=*[.
|
||||
static void lua_scan_long_bracket_close(Tokenizer *tok, S32 level) {
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == ']') {
|
||||
const char *p = tok->t + 1;
|
||||
S32 count = 0;
|
||||
while (p < tok->max_t && *p == '=' && count < level) { count++; p++; }
|
||||
if (count == level && p < tok->max_t && *p == ']') {
|
||||
tok->t = p + 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Individual token parsers
|
||||
|
||||
static Token lua_parse_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
const char *begin = tok->t;
|
||||
while (tok->t < tok->max_t && lua_is_ident_char(*tok->t))
|
||||
tok->t++;
|
||||
S32 len = (S32)(tok->t - begin);
|
||||
token.type = lua_lookup_keyword(begin, len);
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token lua_parse_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (c == '0' && tok->t < tok->max_t && (*tok->t == 'x' || *tok->t == 'X')) {
|
||||
// Hex literal
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && lua_is_hex(*tok->t)) tok->t++;
|
||||
// Hex float
|
||||
if (tok->t < tok->max_t && *tok->t == '.') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && lua_is_hex(*tok->t)) tok->t++;
|
||||
}
|
||||
// Hex exponent (p/P)
|
||||
if (tok->t < tok->max_t && (*tok->t == 'p' || *tok->t == 'P')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
} else {
|
||||
// Decimal
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
// Float
|
||||
if (tok->t < tok->max_t && *tok->t == '.') {
|
||||
if ((tok->t + 1) < tok->max_t && isdigit((unsigned char)*(tok->t + 1))) {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
} else if ((tok->t + 1) >= tok->max_t || !lua_is_ident_start(*(tok->t + 1))) {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token lua_parse_dot_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
tok->t++; // skip '.'
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token lua_parse_string(Tokenizer *tok, char quote) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
B32 escape = 0;
|
||||
tok->t++; // skip opening quote
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') {
|
||||
if (*tok->t == quote && !escape) { tok->t++; break; }
|
||||
escape = !escape && (*tok->t == '\\');
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token lua_parse_long_string(Tokenizer *tok, S32 level) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
// Skip past opening [=*[
|
||||
tok->t++; // first [
|
||||
tok->t += level; // = signs
|
||||
tok->t++; // second [
|
||||
|
||||
lua_scan_long_bracket_close(tok, level);
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token lua_parse_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
|
||||
tok->t += 2; // skip '--'
|
||||
|
||||
// Check for long comment --[=*[
|
||||
S32 level = lua_long_bracket_level(tok->t, tok->max_t);
|
||||
if (level >= 0) {
|
||||
token.type = TOK_MULTILINE_COMMENT;
|
||||
tok->t++; // first [
|
||||
tok->t += level; // = signs
|
||||
tok->t++; // second [
|
||||
lua_scan_long_bracket_close(tok, level);
|
||||
} else {
|
||||
// Single-line comment
|
||||
token.type = TOK_COMMENT;
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token lua_parse_operator(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_OPERATION;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (tok->t < tok->max_t) {
|
||||
char n = *tok->t;
|
||||
// Two-character operators
|
||||
if ((c == '=' && n == '=') || (c == '~' && n == '=') ||
|
||||
(c == '<' && n == '=') || (c == '>' && n == '=') ||
|
||||
(c == '.' && n == '.') || (c == '<' && n == '<') ||
|
||||
(c == '>' && n == '>') || (c == '/' && n == '/')) {
|
||||
tok->t++;
|
||||
// Three-character: .. can be followed by . to make ...
|
||||
if (c == '.' && n == '.' && tok->t < tok->max_t && *tok->t == '.') {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Main get_next_token
|
||||
|
||||
static Token lua_get_next_token(Tokenizer *tok) {
|
||||
tokenizer_eat_whitespace(tok);
|
||||
|
||||
Token token = {0};
|
||||
token.start = (S32)(tok->t - tok->buf);
|
||||
token.type = TOK_DEFAULT;
|
||||
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.len = 0;
|
||||
return token;
|
||||
}
|
||||
|
||||
tok->start_t = tok->t;
|
||||
char c = *tok->t;
|
||||
|
||||
if (lua_is_ident_start(c)) {
|
||||
return lua_parse_identifier(tok);
|
||||
}
|
||||
if (isdigit((unsigned char)c)) {
|
||||
return lua_parse_number(tok);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '\'':
|
||||
case '"':
|
||||
return lua_parse_string(tok, c);
|
||||
|
||||
case '[': {
|
||||
// Check for long string [=*[
|
||||
S32 level = lua_long_bracket_level(tok->t, tok->max_t);
|
||||
if (level >= 0) {
|
||||
return lua_parse_long_string(tok, level);
|
||||
}
|
||||
// Plain bracket punctuation
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
|
||||
case '-':
|
||||
// Check for comment --
|
||||
if ((tok->t + 1) < tok->max_t && *(tok->t + 1) == '-') {
|
||||
return lua_parse_comment(tok);
|
||||
}
|
||||
// Minus operator
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
case '.':
|
||||
// .digits = float literal
|
||||
if ((tok->t + 1) < tok->max_t && isdigit((unsigned char)*(tok->t + 1))) {
|
||||
return lua_parse_dot_number(tok);
|
||||
}
|
||||
// .. or ... or just .
|
||||
return lua_parse_operator(tok);
|
||||
|
||||
// Punctuation
|
||||
case ';': case ',': case ':':
|
||||
case '{': case '}': case '(': case ')': case ']':
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Directive: shebang or labels (::label::)
|
||||
case '#':
|
||||
// Shebang line or length operator
|
||||
if (tok->t == tok->buf && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '!') {
|
||||
token.type = TOK_DIRECTIVE;
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Operators
|
||||
case '=': case '~': case '<': case '>':
|
||||
case '+': case '*': case '/': case '%':
|
||||
case '^': case '&': case '|':
|
||||
return lua_parse_operator(tok);
|
||||
|
||||
default:
|
||||
token.type = TOK_INVALID;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// tokenize_lua -- main entry point
|
||||
|
||||
static void tokenize_lua(const char *data, S32 length, U8 *out_tokens) {
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
|
||||
Tokenizer tok;
|
||||
tokenizer_init(&tok, data, length);
|
||||
|
||||
Token prev = {0};
|
||||
prev.type = TOK_DEFAULT;
|
||||
|
||||
while (tok.t < tok.max_t) {
|
||||
Token token = lua_get_next_token(&tok);
|
||||
if (token.len == 0) break;
|
||||
|
||||
paint_token(out_tokens, token.start, token.len, token.type);
|
||||
|
||||
// Retroactively mark identifier before '(' as a function
|
||||
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
||||
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
||||
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
||||
}
|
||||
|
||||
prev = token;
|
||||
}
|
||||
}
|
||||
645
c/lexer/lexer_sql.c
Normal file
645
c/lexer/lexer_sql.c
Normal file
@@ -0,0 +1,645 @@
|
||||
// lexer_sql.c -- SQL language tokenizer
|
||||
//
|
||||
// Same pattern as lexer_go.c: a get_next_token() loop that advances a cursor
|
||||
// through the source, producing Token structs. After each token we paint
|
||||
// the per-byte token-type array. A second pass marks identifiers before '('
|
||||
// as functions.
|
||||
//
|
||||
// SQL keywords are case-insensitive, so comparisons use a case-insensitive
|
||||
// match. Covers standard SQL plus common extensions (MySQL, PostgreSQL, etc.).
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
////////////////////////////////
|
||||
// SQL keyword tables
|
||||
|
||||
typedef struct SQLKeywordEntry {
|
||||
const char *word;
|
||||
Token_Type type;
|
||||
} SQLKeywordEntry;
|
||||
|
||||
static const SQLKeywordEntry sql_keywords[] = {
|
||||
// DML / DQL keywords
|
||||
{"SELECT", TOK_KEYWORD},
|
||||
{"FROM", TOK_KEYWORD},
|
||||
{"WHERE", TOK_KEYWORD},
|
||||
{"INSERT", TOK_KEYWORD},
|
||||
{"INTO", TOK_KEYWORD},
|
||||
{"UPDATE", TOK_KEYWORD},
|
||||
{"DELETE", TOK_KEYWORD},
|
||||
{"SET", TOK_KEYWORD},
|
||||
{"VALUES", TOK_KEYWORD},
|
||||
{"AS", TOK_KEYWORD},
|
||||
{"ON", TOK_KEYWORD},
|
||||
{"JOIN", TOK_KEYWORD},
|
||||
{"INNER", TOK_KEYWORD},
|
||||
{"LEFT", TOK_KEYWORD},
|
||||
{"RIGHT", TOK_KEYWORD},
|
||||
{"OUTER", TOK_KEYWORD},
|
||||
{"FULL", TOK_KEYWORD},
|
||||
{"CROSS", TOK_KEYWORD},
|
||||
{"NATURAL", TOK_KEYWORD},
|
||||
{"USING", TOK_KEYWORD},
|
||||
{"ORDER", TOK_KEYWORD},
|
||||
{"BY", TOK_KEYWORD},
|
||||
{"GROUP", TOK_KEYWORD},
|
||||
{"HAVING", TOK_KEYWORD},
|
||||
{"LIMIT", TOK_KEYWORD},
|
||||
{"OFFSET", TOK_KEYWORD},
|
||||
{"UNION", TOK_KEYWORD},
|
||||
{"ALL", TOK_KEYWORD},
|
||||
{"DISTINCT", TOK_KEYWORD},
|
||||
{"TOP", TOK_KEYWORD},
|
||||
{"FETCH", TOK_KEYWORD},
|
||||
{"NEXT", TOK_KEYWORD},
|
||||
{"ROWS", TOK_KEYWORD},
|
||||
{"ONLY", TOK_KEYWORD},
|
||||
{"FIRST", TOK_KEYWORD},
|
||||
{"LAST", TOK_KEYWORD},
|
||||
|
||||
// DDL keywords
|
||||
{"CREATE", TOK_KEYWORD},
|
||||
{"ALTER", TOK_KEYWORD},
|
||||
{"DROP", TOK_KEYWORD},
|
||||
{"TABLE", TOK_KEYWORD},
|
||||
{"VIEW", TOK_KEYWORD},
|
||||
{"INDEX", TOK_KEYWORD},
|
||||
{"DATABASE", TOK_KEYWORD},
|
||||
{"SCHEMA", TOK_KEYWORD},
|
||||
{"COLUMN", TOK_KEYWORD},
|
||||
{"ADD", TOK_KEYWORD},
|
||||
{"RENAME", TOK_KEYWORD},
|
||||
{"TRUNCATE", TOK_KEYWORD},
|
||||
{"REPLACE", TOK_KEYWORD},
|
||||
{"TEMPORARY", TOK_KEYWORD},
|
||||
{"TEMP", TOK_KEYWORD},
|
||||
{"IF", TOK_KEYWORD},
|
||||
{"EXISTS", TOK_KEYWORD},
|
||||
{"CASCADE", TOK_KEYWORD},
|
||||
{"RESTRICT", TOK_KEYWORD},
|
||||
|
||||
// Constraints & keys
|
||||
{"PRIMARY", TOK_KEYWORD},
|
||||
{"KEY", TOK_KEYWORD},
|
||||
{"FOREIGN", TOK_KEYWORD},
|
||||
{"REFERENCES", TOK_KEYWORD},
|
||||
{"UNIQUE", TOK_KEYWORD},
|
||||
{"CHECK", TOK_KEYWORD},
|
||||
{"CONSTRAINT", TOK_KEYWORD},
|
||||
{"DEFAULT", TOK_KEYWORD},
|
||||
{"AUTOINCREMENT", TOK_KEYWORD},
|
||||
{"AUTO_INCREMENT", TOK_KEYWORD},
|
||||
{"IDENTITY", TOK_KEYWORD},
|
||||
|
||||
// Control flow / procedural
|
||||
{"BEGIN", TOK_KEYWORD},
|
||||
{"END", TOK_KEYWORD},
|
||||
{"COMMIT", TOK_KEYWORD},
|
||||
{"ROLLBACK", TOK_KEYWORD},
|
||||
{"SAVEPOINT", TOK_KEYWORD},
|
||||
{"TRANSACTION", TOK_KEYWORD},
|
||||
{"RETURN", TOK_KEYWORD},
|
||||
{"RETURNS", TOK_KEYWORD},
|
||||
{"DECLARE", TOK_KEYWORD},
|
||||
{"CURSOR", TOK_KEYWORD},
|
||||
{"OPEN", TOK_KEYWORD},
|
||||
{"CLOSE", TOK_KEYWORD},
|
||||
{"DEALLOCATE", TOK_KEYWORD},
|
||||
{"EXEC", TOK_KEYWORD},
|
||||
{"EXECUTE", TOK_KEYWORD},
|
||||
{"CALL", TOK_KEYWORD},
|
||||
{"PROCEDURE", TOK_KEYWORD},
|
||||
{"FUNCTION", TOK_KEYWORD},
|
||||
{"TRIGGER", TOK_KEYWORD},
|
||||
{"CASE", TOK_KEYWORD},
|
||||
{"WHEN", TOK_KEYWORD},
|
||||
{"THEN", TOK_KEYWORD},
|
||||
{"ELSE", TOK_KEYWORD},
|
||||
{"WHILE", TOK_KEYWORD},
|
||||
{"LOOP", TOK_KEYWORD},
|
||||
{"FOR", TOK_KEYWORD},
|
||||
{"EACH", TOK_KEYWORD},
|
||||
{"ROW", TOK_KEYWORD},
|
||||
{"AFTER", TOK_KEYWORD},
|
||||
{"BEFORE", TOK_KEYWORD},
|
||||
{"INSTEAD", TOK_KEYWORD},
|
||||
{"OF", TOK_KEYWORD},
|
||||
|
||||
// Logical operators (keywords)
|
||||
{"AND", TOK_KEYWORD},
|
||||
{"OR", TOK_KEYWORD},
|
||||
{"NOT", TOK_KEYWORD},
|
||||
{"IN", TOK_KEYWORD},
|
||||
{"BETWEEN", TOK_KEYWORD},
|
||||
{"LIKE", TOK_KEYWORD},
|
||||
{"ILIKE", TOK_KEYWORD},
|
||||
{"IS", TOK_KEYWORD},
|
||||
{"ANY", TOK_KEYWORD},
|
||||
{"SOME", TOK_KEYWORD},
|
||||
{"EXCEPT", TOK_KEYWORD},
|
||||
{"INTERSECT", TOK_KEYWORD},
|
||||
{"WITH", TOK_KEYWORD},
|
||||
{"RECURSIVE", TOK_KEYWORD},
|
||||
{"OVER", TOK_KEYWORD},
|
||||
{"PARTITION", TOK_KEYWORD},
|
||||
{"WINDOW", TOK_KEYWORD},
|
||||
{"LATERAL", TOK_KEYWORD},
|
||||
|
||||
// Misc
|
||||
{"GRANT", TOK_KEYWORD},
|
||||
{"REVOKE", TOK_KEYWORD},
|
||||
{"EXPLAIN", TOK_KEYWORD},
|
||||
{"ANALYZE", TOK_KEYWORD},
|
||||
{"VACUUM", TOK_KEYWORD},
|
||||
{"PRAGMA", TOK_KEYWORD},
|
||||
{"DESCRIBE", TOK_KEYWORD},
|
||||
{"SHOW", TOK_KEYWORD},
|
||||
{"USE", TOK_KEYWORD},
|
||||
{"COPY", TOK_KEYWORD},
|
||||
{"PERFORM", TOK_KEYWORD},
|
||||
{"RAISE", TOK_KEYWORD},
|
||||
|
||||
// Modifiers
|
||||
{"ASC", TOK_MODIFIER},
|
||||
{"DESC", TOK_MODIFIER},
|
||||
{"NULLS", TOK_MODIFIER},
|
||||
{"NOT", TOK_MODIFIER},
|
||||
|
||||
// Data types
|
||||
{"INT", TOK_TYPE},
|
||||
{"INTEGER", TOK_TYPE},
|
||||
{"SMALLINT", TOK_TYPE},
|
||||
{"BIGINT", TOK_TYPE},
|
||||
{"TINYINT", TOK_TYPE},
|
||||
{"MEDIUMINT", TOK_TYPE},
|
||||
{"SERIAL", TOK_TYPE},
|
||||
{"BIGSERIAL", TOK_TYPE},
|
||||
{"FLOAT", TOK_TYPE},
|
||||
{"REAL", TOK_TYPE},
|
||||
{"DOUBLE", TOK_TYPE},
|
||||
{"DECIMAL", TOK_TYPE},
|
||||
{"NUMERIC", TOK_TYPE},
|
||||
{"PRECISION", TOK_TYPE},
|
||||
{"CHAR", TOK_TYPE},
|
||||
{"VARCHAR", TOK_TYPE},
|
||||
{"TEXT", TOK_TYPE},
|
||||
{"NCHAR", TOK_TYPE},
|
||||
{"NVARCHAR", TOK_TYPE},
|
||||
{"NTEXT", TOK_TYPE},
|
||||
{"BLOB", TOK_TYPE},
|
||||
{"CLOB", TOK_TYPE},
|
||||
{"BYTEA", TOK_TYPE},
|
||||
{"BOOLEAN", TOK_TYPE},
|
||||
{"BOOL", TOK_TYPE},
|
||||
{"DATE", TOK_TYPE},
|
||||
{"TIME", TOK_TYPE},
|
||||
{"DATETIME", TOK_TYPE},
|
||||
{"TIMESTAMP", TOK_TYPE},
|
||||
{"TIMESTAMPTZ", TOK_TYPE},
|
||||
{"INTERVAL", TOK_TYPE},
|
||||
{"UUID", TOK_TYPE},
|
||||
{"JSON", TOK_TYPE},
|
||||
{"JSONB", TOK_TYPE},
|
||||
{"XML", TOK_TYPE},
|
||||
{"ARRAY", TOK_TYPE},
|
||||
{"ENUM", TOK_TYPE},
|
||||
{"MONEY", TOK_TYPE},
|
||||
{"BIT", TOK_TYPE},
|
||||
{"VARBIT", TOK_TYPE},
|
||||
{"INET", TOK_TYPE},
|
||||
{"CIDR", TOK_TYPE},
|
||||
{"MACADDR", TOK_TYPE},
|
||||
{"POINT", TOK_TYPE},
|
||||
{"LINE", TOK_TYPE},
|
||||
{"POLYGON", TOK_TYPE},
|
||||
{"GEOMETRY", TOK_TYPE},
|
||||
{"GEOGRAPHY", TOK_TYPE},
|
||||
|
||||
// Values
|
||||
{"NULL", TOK_VALUE},
|
||||
{"TRUE", TOK_VALUE},
|
||||
{"FALSE", TOK_VALUE},
|
||||
{"CURRENT_DATE", TOK_VALUE},
|
||||
{"CURRENT_TIME", TOK_VALUE},
|
||||
{"CURRENT_TIMESTAMP", TOK_VALUE},
|
||||
{"CURRENT_USER", TOK_VALUE},
|
||||
|
||||
// Built-in aggregate / window / common functions (TOK_MODIFIER)
|
||||
{"COUNT", TOK_MODIFIER},
|
||||
{"SUM", TOK_MODIFIER},
|
||||
{"AVG", TOK_MODIFIER},
|
||||
{"MIN", TOK_MODIFIER},
|
||||
{"MAX", TOK_MODIFIER},
|
||||
{"COALESCE", TOK_MODIFIER},
|
||||
{"NULLIF", TOK_MODIFIER},
|
||||
{"CAST", TOK_MODIFIER},
|
||||
{"CONVERT", TOK_MODIFIER},
|
||||
{"IFNULL", TOK_MODIFIER},
|
||||
{"ISNULL", TOK_MODIFIER},
|
||||
{"NVL", TOK_MODIFIER},
|
||||
{"ROW_NUMBER", TOK_MODIFIER},
|
||||
{"RANK", TOK_MODIFIER},
|
||||
{"DENSE_RANK", TOK_MODIFIER},
|
||||
{"NTILE", TOK_MODIFIER},
|
||||
{"LAG", TOK_MODIFIER},
|
||||
{"LEAD", TOK_MODIFIER},
|
||||
{"FIRST_VALUE", TOK_MODIFIER},
|
||||
{"LAST_VALUE", TOK_MODIFIER},
|
||||
{"SUBSTR", TOK_MODIFIER},
|
||||
{"SUBSTRING", TOK_MODIFIER},
|
||||
{"TRIM", TOK_MODIFIER},
|
||||
{"UPPER", TOK_MODIFIER},
|
||||
{"LOWER", TOK_MODIFIER},
|
||||
{"LENGTH", TOK_MODIFIER},
|
||||
{"CONCAT", TOK_MODIFIER},
|
||||
{"REPLACE", TOK_MODIFIER},
|
||||
{"ABS", TOK_MODIFIER},
|
||||
{"ROUND", TOK_MODIFIER},
|
||||
{"CEIL", TOK_MODIFIER},
|
||||
{"FLOOR", TOK_MODIFIER},
|
||||
{"NOW", TOK_MODIFIER},
|
||||
{"EXTRACT", TOK_MODIFIER},
|
||||
{"STRING_AGG", TOK_MODIFIER},
|
||||
{"GROUP_CONCAT", TOK_MODIFIER},
|
||||
{"ARRAY_AGG", TOK_MODIFIER},
|
||||
{"GREATEST", TOK_MODIFIER},
|
||||
{"LEAST", TOK_MODIFIER},
|
||||
};
|
||||
|
||||
#define SQL_KEYWORD_COUNT (S32)(sizeof(sql_keywords) / sizeof(sql_keywords[0]))
|
||||
|
||||
// Case-insensitive keyword lookup
|
||||
static Token_Type sql_lookup_keyword(const char *word, S32 len) {
|
||||
for (S32 i = 0; i < SQL_KEYWORD_COUNT; i++) {
|
||||
const char *kw = sql_keywords[i].word;
|
||||
S32 kwlen = (S32)strlen(kw);
|
||||
if (kwlen != len) continue;
|
||||
B32 match = 1;
|
||||
for (S32 j = 0; j < len; j++) {
|
||||
if (toupper((unsigned char)word[j]) != (unsigned char)kw[j]) {
|
||||
match = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) return sql_keywords[i].type;
|
||||
}
|
||||
return TOK_IDENTIFIER;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Character helpers
|
||||
|
||||
static B32 sql_is_ident_start(char c) {
|
||||
return isalpha((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 sql_is_ident_char(char c) {
|
||||
return isalnum((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Individual token parsers
|
||||
|
||||
static Token sql_parse_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
const char *begin = tok->t;
|
||||
while (tok->t < tok->max_t && sql_is_ident_char(*tok->t))
|
||||
tok->t++;
|
||||
S32 len = (S32)(tok->t - begin);
|
||||
token.type = sql_lookup_keyword(begin, len);
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token sql_parse_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t))
|
||||
tok->t++;
|
||||
|
||||
// Decimal part
|
||||
if (tok->t < tok->max_t && *tok->t == '.') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t))
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token sql_parse_dot_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
tok->t++; // skip '.'
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Single-quoted string: 'text', with '' as escape for embedded quote
|
||||
static Token sql_parse_string(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
tok->t++; // skip opening '
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '\'') {
|
||||
tok->t++;
|
||||
// '' is an escaped quote inside a string
|
||||
if (tok->t < tok->max_t && *tok->t == '\'') {
|
||||
tok->t++;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Double-quoted identifier: "column_name"
|
||||
static Token sql_parse_quoted_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_IDENTIFIER;
|
||||
|
||||
tok->t++; // skip opening "
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '"') {
|
||||
tok->t++;
|
||||
// "" is an escaped quote inside a quoted identifier
|
||||
if (tok->t < tok->max_t && *tok->t == '"') {
|
||||
tok->t++;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Backtick-quoted identifier (MySQL): `column_name`
|
||||
static Token sql_parse_backtick_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_IDENTIFIER;
|
||||
|
||||
tok->t++; // skip opening `
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '`') { tok->t++; break; }
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Line comment: -- ...
|
||||
static Token sql_parse_line_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_COMMENT;
|
||||
|
||||
tok->t += 2; // skip '--'
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Block comment: /* ... */
|
||||
static Token sql_parse_block_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_MULTILINE_COMMENT;
|
||||
|
||||
tok->t += 2; // skip '/*'
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '*' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '/') {
|
||||
tok->t += 2;
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// MySQL # comment
|
||||
static Token sql_parse_hash_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_COMMENT;
|
||||
|
||||
tok->t++; // skip '#'
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Variable / parameter: @var, @@var, :param, $1
|
||||
static Token sql_parse_variable(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_DIRECTIVE;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (c == '@') {
|
||||
// @@ for system variables
|
||||
if (tok->t < tok->max_t && *tok->t == '@') tok->t++;
|
||||
while (tok->t < tok->max_t && sql_is_ident_char(*tok->t)) tok->t++;
|
||||
} else if (c == ':') {
|
||||
while (tok->t < tok->max_t && sql_is_ident_char(*tok->t)) tok->t++;
|
||||
} else if (c == '$') {
|
||||
// $1, $2, ... (PostgreSQL positional params)
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
// Also handle $tag$ dollar-quoted strings if no digits follow
|
||||
if (tok->t == tok->start_t + 1) {
|
||||
// Just a lone $, treat as punctuation
|
||||
token.type = TOK_PUNCTUATION;
|
||||
}
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token sql_parse_operator(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_OPERATION;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (tok->t < tok->max_t) {
|
||||
char n = *tok->t;
|
||||
// Two-character operators
|
||||
if ((c == '<' && n == '=') || (c == '>' && n == '=') ||
|
||||
(c == '<' && n == '>') || (c == '!' && n == '=') ||
|
||||
(c == '|' && n == '|') || (c == ':' && n == ':')) {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Main get_next_token
|
||||
|
||||
static Token sql_get_next_token(Tokenizer *tok) {
|
||||
tokenizer_eat_whitespace(tok);
|
||||
|
||||
Token token = {0};
|
||||
token.start = (S32)(tok->t - tok->buf);
|
||||
token.type = TOK_DEFAULT;
|
||||
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.len = 0;
|
||||
return token;
|
||||
}
|
||||
|
||||
tok->start_t = tok->t;
|
||||
char c = *tok->t;
|
||||
|
||||
if (sql_is_ident_start(c)) {
|
||||
return sql_parse_identifier(tok);
|
||||
}
|
||||
if (isdigit((unsigned char)c)) {
|
||||
return sql_parse_number(tok);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '\'':
|
||||
return sql_parse_string(tok);
|
||||
|
||||
case '"':
|
||||
return sql_parse_quoted_identifier(tok);
|
||||
|
||||
case '`':
|
||||
return sql_parse_backtick_identifier(tok);
|
||||
|
||||
case '-':
|
||||
if ((tok->t + 1) < tok->max_t && *(tok->t + 1) == '-') {
|
||||
return sql_parse_line_comment(tok);
|
||||
}
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
case '/':
|
||||
if ((tok->t + 1) < tok->max_t && *(tok->t + 1) == '*') {
|
||||
return sql_parse_block_comment(tok);
|
||||
}
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
case '#':
|
||||
return sql_parse_hash_comment(tok);
|
||||
|
||||
case '.':
|
||||
if ((tok->t + 1) < tok->max_t && isdigit((unsigned char)*(tok->t + 1))) {
|
||||
return sql_parse_dot_number(tok);
|
||||
}
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
case '@': case '$':
|
||||
return sql_parse_variable(tok);
|
||||
|
||||
case ':':
|
||||
// :param or :: cast operator
|
||||
if ((tok->t + 1) < tok->max_t) {
|
||||
char n = *(tok->t + 1);
|
||||
if (n == ':') return sql_parse_operator(tok);
|
||||
if (sql_is_ident_start(n)) return sql_parse_variable(tok);
|
||||
}
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Punctuation
|
||||
case ';': case ',':
|
||||
case '{': case '}': case '(': case ')': case '[': case ']':
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Operators
|
||||
case '=': case '<': case '>': case '!':
|
||||
case '+': case '*': case '%': case '&':
|
||||
case '|': case '^': case '~':
|
||||
return sql_parse_operator(tok);
|
||||
|
||||
default:
|
||||
token.type = TOK_INVALID;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// tokenize_sql -- main entry point
|
||||
|
||||
static void tokenize_sql(const char *data, S32 length, U8 *out_tokens) {
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
|
||||
Tokenizer tok;
|
||||
tokenizer_init(&tok, data, length);
|
||||
|
||||
Token prev = {0};
|
||||
prev.type = TOK_DEFAULT;
|
||||
|
||||
while (tok.t < tok.max_t) {
|
||||
Token token = sql_get_next_token(&tok);
|
||||
if (token.len == 0) break;
|
||||
|
||||
paint_token(out_tokens, token.start, token.len, token.type);
|
||||
|
||||
// Retroactively mark identifier before '(' as a function
|
||||
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
||||
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
||||
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
||||
}
|
||||
|
||||
prev = token;
|
||||
}
|
||||
}
|
||||
502
c/lexer/lexer_theme.c
Normal file
502
c/lexer/lexer_theme.c
Normal file
@@ -0,0 +1,502 @@
|
||||
// lexer_theme.c -- Built-in color themes
|
||||
//
|
||||
// Colors ported from the Focus editor theme files.
|
||||
// Hex values are RGB (alpha channel dropped, always opaque in terminal).
|
||||
|
||||
#include "lexer/lexer_theme.h"
|
||||
|
||||
S32 g_active_theme_idx = 0;
|
||||
|
||||
Theme g_themes[THEME_COUNT] = {
|
||||
|
||||
////////////////////////////////
|
||||
// Default
|
||||
// Uses the terminal's own 256-color palette. No background override.
|
||||
{
|
||||
.name = "Default",
|
||||
.use_truecolor = 0,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xBDBDBD),
|
||||
/* COMMENT */ RGB_HEX(0x6A6A6A),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x6A6A6A),
|
||||
/* STRING_LITERAL */ RGB_HEX(0x6AAF50),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0x6AAF50),
|
||||
/* NUMBER */ RGB_HEX(0xC678DD),
|
||||
/* IDENTIFIER */ RGB_HEX(0xBDBDBD),
|
||||
/* FUNCTION */ RGB_HEX(0x56B6C2),
|
||||
/* KEYWORD */ RGB_HEX(0xD19A66),
|
||||
/* TYPE */ RGB_HEX(0x61AFEF),
|
||||
/* VALUE */ RGB_HEX(0xC678DD),
|
||||
/* MODIFIER */ RGB_HEX(0xE06C75),
|
||||
/* DIRECTIVE */ RGB_HEX(0xE06C75),
|
||||
/* PUNCTUATION */ RGB_HEX(0xBDBDBD),
|
||||
/* OPERATION */ RGB_HEX(0xBDBDBD),
|
||||
/* INVALID */ RGB_HEX(0xFF5555),
|
||||
},
|
||||
.ansi_colors = {
|
||||
/* DEFAULT */ 7,
|
||||
/* COMMENT */ 242,
|
||||
/* MULTILINE_COMMENT */ 242,
|
||||
/* STRING_LITERAL */ 2,
|
||||
/* CHAR_LITERAL */ 2,
|
||||
/* NUMBER */ 5,
|
||||
/* IDENTIFIER */ 7,
|
||||
/* FUNCTION */ 6,
|
||||
/* KEYWORD */ 3,
|
||||
/* TYPE */ 4,
|
||||
/* VALUE */ 5,
|
||||
/* MODIFIER */ 1,
|
||||
/* DIRECTIVE */ 1,
|
||||
/* PUNCTUATION */ 7,
|
||||
/* OPERATION */ 7,
|
||||
/* INVALID */ 9,
|
||||
},
|
||||
.background = RGB_HEX(0x1E1E1E),
|
||||
.set_background = 0,
|
||||
.status_bg = RGB_HEX(0x252525),
|
||||
.status_fg = RGB_HEX(0xDDDDDD),
|
||||
.status_fg_dim = RGB_HEX(0x888888),
|
||||
.mb_bg = RGB_HEX(0x3D5A80),
|
||||
.mb_fg = RGB_HEX(0xFFFFFF),
|
||||
.mb_detail = RGB_HEX(0x888888),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Default Light
|
||||
// Uses the terminal's own 256-color palette with a light background.
|
||||
{
|
||||
.name = "Default Light",
|
||||
.use_truecolor = 0,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0x383A42),
|
||||
/* COMMENT */ RGB_HEX(0xA0A1A7),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0xA0A1A7),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xA62C21),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xA62C21),
|
||||
/* NUMBER */ RGB_HEX(0x986801),
|
||||
/* IDENTIFIER */ RGB_HEX(0x383A42),
|
||||
/* FUNCTION */ RGB_HEX(0x0184BC),
|
||||
/* KEYWORD */ RGB_HEX(0x4078F2),
|
||||
/* TYPE */ RGB_HEX(0x50A14F),
|
||||
/* VALUE */ RGB_HEX(0x986801),
|
||||
/* MODIFIER */ RGB_HEX(0x4078F2),
|
||||
/* DIRECTIVE */ RGB_HEX(0xA62C21),
|
||||
/* PUNCTUATION */ RGB_HEX(0x383A42),
|
||||
/* OPERATION */ RGB_HEX(0x383A42),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {
|
||||
/* DEFAULT */ 0, // black
|
||||
/* COMMENT */ 243, // dark gray
|
||||
/* MULTILINE_COMMENT */ 243,
|
||||
/* STRING_LITERAL */ 1, // red
|
||||
/* CHAR_LITERAL */ 1,
|
||||
/* NUMBER */ 5, // magenta
|
||||
/* IDENTIFIER */ 0, // black
|
||||
/* FUNCTION */ 6, // cyan
|
||||
/* KEYWORD */ 4, // blue
|
||||
/* TYPE */ 2, // green
|
||||
/* VALUE */ 5, // magenta
|
||||
/* MODIFIER */ 4, // blue
|
||||
/* DIRECTIVE */ 1, // red
|
||||
/* PUNCTUATION */ 0, // black
|
||||
/* OPERATION */ 0, // black
|
||||
/* INVALID */ 9, // bright red
|
||||
},
|
||||
.background = RGB_HEX(0xFAFAFA),
|
||||
.set_background = 0,
|
||||
.status_bg = RGB_HEX(0xD0D0D0),
|
||||
.status_fg = RGB_HEX(0x000000),
|
||||
.status_fg_dim = RGB_HEX(0x606060),
|
||||
.mb_bg = RGB_HEX(0x0060C0),
|
||||
.mb_fg = RGB_HEX(0xFFFFFF),
|
||||
.mb_detail = RGB_HEX(0x808080),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Focus
|
||||
// The default Focus editor theme.
|
||||
{
|
||||
.name = "Focus",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xBFC9DB),
|
||||
/* COMMENT */ RGB_HEX(0x87919D),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x87919D),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xD4BC7D),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xD4BC7D),
|
||||
/* NUMBER */ RGB_HEX(0xD699B5),
|
||||
/* IDENTIFIER */ RGB_HEX(0xBFC9DB),
|
||||
/* FUNCTION */ RGB_HEX(0xD0C5A9),
|
||||
/* KEYWORD */ RGB_HEX(0xE67D74),
|
||||
/* TYPE */ RGB_HEX(0x82AAA3),
|
||||
/* VALUE */ RGB_HEX(0xD699B5),
|
||||
/* MODIFIER */ RGB_HEX(0xE67D74),
|
||||
/* DIRECTIVE */ RGB_HEX(0xE67D74),
|
||||
/* PUNCTUATION */ RGB_HEX(0xBFC9DB),
|
||||
/* OPERATION */ RGB_HEX(0xE0AD82),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x15212A),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x0C1620),
|
||||
.status_fg = RGB_HEX(0xBFC9DB),
|
||||
.status_fg_dim = RGB_HEX(0x607080),
|
||||
.mb_bg = RGB_HEX(0x1C3040),
|
||||
.mb_fg = RGB_HEX(0xBFC9DB),
|
||||
.mb_detail = RGB_HEX(0x607080),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Handmade Hero
|
||||
// Based on Casey Muratori's emacs theme from the Handmade Hero series.
|
||||
{
|
||||
.name = "Handmade Hero",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xCDAA7D),
|
||||
/* COMMENT */ RGB_HEX(0x7F7F7F),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x87919D),
|
||||
/* STRING_LITERAL */ RGB_HEX(0x6B8E23),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0x6B8E23),
|
||||
/* NUMBER */ RGB_HEX(0xD699B5),
|
||||
/* IDENTIFIER */ RGB_HEX(0xBFC9DB),
|
||||
/* FUNCTION */ RGB_HEX(0xCDAA7D),
|
||||
/* KEYWORD */ RGB_HEX(0xB8860B),
|
||||
/* TYPE */ RGB_HEX(0xB8860B),
|
||||
/* VALUE */ RGB_HEX(0x6B8E23),
|
||||
/* MODIFIER */ RGB_HEX(0xE67D74),
|
||||
/* DIRECTIVE */ RGB_HEX(0xE67D74),
|
||||
/* PUNCTUATION */ RGB_HEX(0xCDAA7D),
|
||||
/* OPERATION */ RGB_HEX(0xCDAA7D),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x161616),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x0C0C0C),
|
||||
.status_fg = RGB_HEX(0xCDAA7D),
|
||||
.status_fg_dim = RGB_HEX(0x6B5A3F),
|
||||
.mb_bg = RGB_HEX(0x403020),
|
||||
.mb_fg = RGB_HEX(0xCDAA7D),
|
||||
.mb_detail = RGB_HEX(0x6B5A3F),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Witness Classic
|
||||
// Jonathan Blow's classic color scheme.
|
||||
{
|
||||
.name = "Witness Classic",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xD3B58D),
|
||||
/* COMMENT */ RGB_HEX(0xFFFF00),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x87919D),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xBEBEBE),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xBEBEBE),
|
||||
/* NUMBER */ RGB_HEX(0xD699B5),
|
||||
/* IDENTIFIER */ RGB_HEX(0xBFC9DB),
|
||||
/* FUNCTION */ RGB_HEX(0xD3B58D),
|
||||
/* KEYWORD */ RGB_HEX(0xFFFFFF),
|
||||
/* TYPE */ RGB_HEX(0x98FB98),
|
||||
/* VALUE */ RGB_HEX(0x7FFFD4),
|
||||
/* MODIFIER */ RGB_HEX(0xE67D74),
|
||||
/* DIRECTIVE */ RGB_HEX(0xE67D74),
|
||||
/* PUNCTUATION */ RGB_HEX(0xD3B58D),
|
||||
/* OPERATION */ RGB_HEX(0xD3B58D),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x292929),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x222222),
|
||||
.status_fg = RGB_HEX(0xDDDDDD),
|
||||
.status_fg_dim = RGB_HEX(0x888888),
|
||||
.mb_bg = RGB_HEX(0x3A3A50),
|
||||
.mb_fg = RGB_HEX(0xFFFFFF),
|
||||
.mb_detail = RGB_HEX(0x888888),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Witness
|
||||
// Jonathan Blow's alternative dark-teal theme.
|
||||
{
|
||||
.name = "Witness",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xD3B58D),
|
||||
/* COMMENT */ RGB_HEX(0x3DDF23),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x87919D),
|
||||
/* STRING_LITERAL */ RGB_HEX(0x0FDFAF),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0x0FDFAF),
|
||||
/* NUMBER */ RGB_HEX(0xD699B5),
|
||||
/* IDENTIFIER */ RGB_HEX(0xBFC9DB),
|
||||
/* FUNCTION */ RGB_HEX(0xD3B58D),
|
||||
/* KEYWORD */ RGB_HEX(0xFFFFFF),
|
||||
/* TYPE */ RGB_HEX(0x98FB98),
|
||||
/* VALUE */ RGB_HEX(0x7FFFD4),
|
||||
/* MODIFIER */ RGB_HEX(0xE67D74),
|
||||
/* DIRECTIVE */ RGB_HEX(0xE67D74),
|
||||
/* PUNCTUATION */ RGB_HEX(0xD3B58D),
|
||||
/* OPERATION */ RGB_HEX(0xE0AD82),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x072626),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x041818),
|
||||
.status_fg = RGB_HEX(0xD3B58D),
|
||||
.status_fg_dim = RGB_HEX(0x6B7A6B),
|
||||
.mb_bg = RGB_HEX(0x0E3A3A),
|
||||
.mb_fg = RGB_HEX(0xD3B58D),
|
||||
.mb_detail = RGB_HEX(0x6B7A6B),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Visual Studio Classic
|
||||
// Light theme matching Visual Studio 6.0 (1998).
|
||||
{
|
||||
.name = "VS Classic",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0x000000),
|
||||
/* COMMENT */ RGB_HEX(0x008000),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x008000),
|
||||
/* STRING_LITERAL */ RGB_HEX(0x800000),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0x800000),
|
||||
/* NUMBER */ RGB_HEX(0x000000),
|
||||
/* IDENTIFIER */ RGB_HEX(0x000000),
|
||||
/* FUNCTION */ RGB_HEX(0x000000),
|
||||
/* KEYWORD */ RGB_HEX(0x0000FF),
|
||||
/* TYPE */ RGB_HEX(0x0000FF),
|
||||
/* VALUE */ RGB_HEX(0x0000FF),
|
||||
/* MODIFIER */ RGB_HEX(0x0000FF),
|
||||
/* DIRECTIVE */ RGB_HEX(0x0000FF),
|
||||
/* PUNCTUATION */ RGB_HEX(0x000000),
|
||||
/* OPERATION */ RGB_HEX(0x000000),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0xFFFFFF),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0xD4D0C8),
|
||||
.status_fg = RGB_HEX(0x000000),
|
||||
.status_fg_dim = RGB_HEX(0x606060),
|
||||
.mb_bg = RGB_HEX(0x0A246A),
|
||||
.mb_fg = RGB_HEX(0xFFFFFF),
|
||||
.mb_detail = RGB_HEX(0xA0A0A0),
|
||||
.fb_bg = RGB_HEX(0xFFFFFF),
|
||||
.set_fb_bg = 1,
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// RAD Debugger
|
||||
// Default color scheme from the RAD Debugger project.
|
||||
{
|
||||
.name = "RAD Debugger",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xDADADA),
|
||||
/* COMMENT */ RGB_HEX(0x5D7856),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x5D7856),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xFFA070),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xFFA070),
|
||||
/* NUMBER */ RGB_HEX(0xD0D0A0),
|
||||
/* IDENTIFIER */ RGB_HEX(0xDADADA),
|
||||
/* FUNCTION */ RGB_HEX(0xDADADA),
|
||||
/* KEYWORD */ RGB_HEX(0xF0C674),
|
||||
/* TYPE */ RGB_HEX(0x70C0B0),
|
||||
/* VALUE */ RGB_HEX(0xD0D0A0),
|
||||
/* MODIFIER */ RGB_HEX(0xF0C674),
|
||||
/* DIRECTIVE */ RGB_HEX(0xCC7070),
|
||||
/* PUNCTUATION */ RGB_HEX(0xDADADA),
|
||||
/* OPERATION */ RGB_HEX(0xDADADA),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x1C1C1C),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x121212),
|
||||
.status_fg = RGB_HEX(0xDADADA),
|
||||
.status_fg_dim = RGB_HEX(0x707070),
|
||||
.mb_bg = RGB_HEX(0x3A2A1A),
|
||||
.mb_fg = RGB_HEX(0xDADADA),
|
||||
.mb_detail = RGB_HEX(0x707070),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// 4coder
|
||||
// Default theme from Allen Webster's 4coder editor.
|
||||
{
|
||||
.name = "4coder",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0x90B080),
|
||||
/* COMMENT */ RGB_HEX(0x2090F0),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x2090F0),
|
||||
/* STRING_LITERAL */ RGB_HEX(0x50FF30),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0x50FF30),
|
||||
/* NUMBER */ RGB_HEX(0x50FF30),
|
||||
/* IDENTIFIER */ RGB_HEX(0x90B080),
|
||||
/* FUNCTION */ RGB_HEX(0x90B080),
|
||||
/* KEYWORD */ RGB_HEX(0xD08F20),
|
||||
/* TYPE */ RGB_HEX(0xD08F20),
|
||||
/* VALUE */ RGB_HEX(0x50FF30),
|
||||
/* MODIFIER */ RGB_HEX(0xD08F20),
|
||||
/* DIRECTIVE */ RGB_HEX(0xFF5F5F),
|
||||
/* PUNCTUATION */ RGB_HEX(0x90B080),
|
||||
/* OPERATION */ RGB_HEX(0x90B080),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x0C0C0C),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x1A1A1A),
|
||||
.status_fg = RGB_HEX(0x90B080),
|
||||
.status_fg_dim = RGB_HEX(0x506050),
|
||||
.mb_bg = RGB_HEX(0x2A3A20),
|
||||
.mb_fg = RGB_HEX(0x90B080),
|
||||
.mb_detail = RGB_HEX(0x606060),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Ryan Fleury
|
||||
// Ryan Fleury's personal theme, ported from the fleury-theme.el emacs package.
|
||||
{
|
||||
.name = "Ryan Fleury",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xB99468),
|
||||
/* COMMENT */ RGB_HEX(0x666666),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x666666),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xFFAA00),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xFFAA00),
|
||||
/* NUMBER */ RGB_HEX(0xFFAA00),
|
||||
/* IDENTIFIER */ RGB_HEX(0xB99468),
|
||||
/* FUNCTION */ RGB_HEX(0xDE451F),
|
||||
/* KEYWORD */ RGB_HEX(0xF0C674),
|
||||
/* TYPE */ RGB_HEX(0xEDB211),
|
||||
/* VALUE */ RGB_HEX(0xFFAA00),
|
||||
/* MODIFIER */ RGB_HEX(0xF0C674),
|
||||
/* DIRECTIVE */ RGB_HEX(0xDC7575),
|
||||
/* PUNCTUATION */ RGB_HEX(0xB99468),
|
||||
/* OPERATION */ RGB_HEX(0xB99468),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x020202),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x1A120B),
|
||||
.status_fg = RGB_HEX(0xE7AA4D),
|
||||
.status_fg_dim = RGB_HEX(0x63523D),
|
||||
.mb_bg = RGB_HEX(0x3A2510),
|
||||
.mb_fg = RGB_HEX(0xE7AA4D),
|
||||
.mb_detail = RGB_HEX(0x63523D),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Gruber Darker
|
||||
// Gruber Darker theme by Alexey Kutepov (rexim).
|
||||
{
|
||||
.name = "Gruber Darker",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xE4E4EF),
|
||||
/* COMMENT */ RGB_HEX(0x565F73),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x565F73),
|
||||
/* STRING_LITERAL */ RGB_HEX(0x73C936),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0x73C936),
|
||||
/* NUMBER */ RGB_HEX(0xFFDD33),
|
||||
/* IDENTIFIER */ RGB_HEX(0xE4E4EF),
|
||||
/* FUNCTION */ RGB_HEX(0x96A6C8),
|
||||
/* KEYWORD */ RGB_HEX(0xFFDD33),
|
||||
/* TYPE */ RGB_HEX(0x96A6C8),
|
||||
/* VALUE */ RGB_HEX(0xFFDD33),
|
||||
/* MODIFIER */ RGB_HEX(0x9E95C7),
|
||||
/* DIRECTIVE */ RGB_HEX(0x9E95C7),
|
||||
/* PUNCTUATION */ RGB_HEX(0xE4E4EF),
|
||||
/* OPERATION */ RGB_HEX(0xE4E4EF),
|
||||
/* INVALID */ RGB_HEX(0xF43841),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x181818),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x101010),
|
||||
.status_fg = RGB_HEX(0xE4E4EF),
|
||||
.status_fg_dim = RGB_HEX(0x565F73),
|
||||
.mb_bg = RGB_HEX(0x282828),
|
||||
.mb_fg = RGB_HEX(0xE4E4EF),
|
||||
.mb_detail = RGB_HEX(0x95A99F),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// VS Dark
|
||||
// Visual Studio Dark theme.
|
||||
{
|
||||
.name = "VS Dark",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xDCDCAA),
|
||||
/* COMMENT */ RGB_HEX(0x6A9955),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x6A9955),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xCE9178),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xCE9178),
|
||||
/* NUMBER */ RGB_HEX(0xB5CEA8),
|
||||
/* IDENTIFIER */ RGB_HEX(0x9CDCFE),
|
||||
/* FUNCTION */ RGB_HEX(0xDCDCAA),
|
||||
/* KEYWORD */ RGB_HEX(0x569CD6),
|
||||
/* TYPE */ RGB_HEX(0x4EC9B0),
|
||||
/* VALUE */ RGB_HEX(0x569CD6),
|
||||
/* MODIFIER */ RGB_HEX(0x569CD6),
|
||||
/* DIRECTIVE */ RGB_HEX(0xC586C0),
|
||||
/* PUNCTUATION */ RGB_HEX(0xD4D4D4),
|
||||
/* OPERATION */ RGB_HEX(0xD4D4D4),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x1E1E1E),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x252526),
|
||||
.status_fg = RGB_HEX(0xCCCCCC),
|
||||
.status_fg_dim = RGB_HEX(0x808080),
|
||||
.mb_bg = RGB_HEX(0x094771),
|
||||
.mb_fg = RGB_HEX(0xFFFFFF),
|
||||
.mb_detail = RGB_HEX(0x808080),
|
||||
},
|
||||
|
||||
////////////////////////////////
|
||||
// Freshcut Contrast
|
||||
// High-contrast dark theme by Dayle Rees.
|
||||
{
|
||||
.name = "Freshcut Contrast",
|
||||
.use_truecolor = 1,
|
||||
.colors = {
|
||||
/* DEFAULT */ RGB_HEX(0xF8F8F2),
|
||||
/* COMMENT */ RGB_HEX(0x737B84),
|
||||
/* MULTILINE_COMMENT */ RGB_HEX(0x737B84),
|
||||
/* STRING_LITERAL */ RGB_HEX(0xE9EE00),
|
||||
/* CHAR_LITERAL */ RGB_HEX(0xE9EE00),
|
||||
/* NUMBER */ RGB_HEX(0x8FBE00),
|
||||
/* IDENTIFIER */ RGB_HEX(0xF8F8F2),
|
||||
/* FUNCTION */ RGB_HEX(0xAEE239),
|
||||
/* KEYWORD */ RGB_HEX(0xC8D7E8),
|
||||
/* TYPE */ RGB_HEX(0x4ECDC4),
|
||||
/* VALUE */ RGB_HEX(0x8FBE00),
|
||||
/* MODIFIER */ RGB_HEX(0x00A8C6),
|
||||
/* DIRECTIVE */ RGB_HEX(0x00A8C6),
|
||||
/* PUNCTUATION */ RGB_HEX(0xF8F8F2),
|
||||
/* OPERATION */ RGB_HEX(0xF8F8F2),
|
||||
/* INVALID */ RGB_HEX(0xFF0000),
|
||||
},
|
||||
.ansi_colors = {},
|
||||
.background = RGB_HEX(0x000000),
|
||||
.set_background = 1,
|
||||
.status_bg = RGB_HEX(0x3C3C3C),
|
||||
.status_fg = RGB_HEX(0xE0E0E0),
|
||||
.status_fg_dim = RGB_HEX(0x808080),
|
||||
.mb_bg = RGB_HEX(0x505050),
|
||||
.mb_fg = RGB_HEX(0xFFFFFF),
|
||||
.mb_detail = RGB_HEX(0x909090),
|
||||
},
|
||||
};
|
||||
50
c/lexer/lexer_theme.h
Normal file
50
c/lexer/lexer_theme.h
Normal file
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
// lexer_theme.h -- Color themes for syntax highlighting
|
||||
//
|
||||
// Each theme maps Token_Type values to 24-bit RGB colors, plus a background.
|
||||
// Themes use true-color ANSI escapes (supported by Windows Terminal, most
|
||||
// modern terminals).
|
||||
|
||||
#include "base/base_core.h"
|
||||
#include "lexer/lexer.h"
|
||||
|
||||
////////////////////////////////
|
||||
// RGB color
|
||||
|
||||
typedef struct RGB { U8 r, g, b; } RGB;
|
||||
|
||||
// Use for static initializers (no compound literal — MSVC C11 compat)
|
||||
#define RGB_HEX(hex) { ((hex) >> 16) & 0xFF, ((hex) >> 8) & 0xFF, (hex) & 0xFF }
|
||||
|
||||
////////////////////////////////
|
||||
// Theme
|
||||
|
||||
typedef struct Theme {
|
||||
const char *name;
|
||||
B32 use_truecolor; // if false, use ansi_colors[] with term_fg() instead
|
||||
RGB colors[TOK_COUNT]; // foreground color per token type (true color)
|
||||
U8 ansi_colors[TOK_COUNT]; // foreground per token type (256-color fallback)
|
||||
RGB background; // terminal background color
|
||||
B32 set_background; // whether to override terminal background
|
||||
|
||||
// UI chrome colors
|
||||
RGB status_bg; // status bar background
|
||||
RGB status_fg; // status bar foreground (focused)
|
||||
RGB status_fg_dim; // status bar foreground (unfocused)
|
||||
RGB mb_bg; // minibuffer item background (highlighted)
|
||||
RGB mb_fg; // minibuffer item foreground (highlighted)
|
||||
RGB mb_detail; // minibuffer detail text color
|
||||
RGB fb_bg; // file browser background (0,0,0 = use WindowBg)
|
||||
B32 set_fb_bg; // whether to override file browser background
|
||||
} Theme;
|
||||
|
||||
////////////////////////////////
|
||||
// Built-in themes
|
||||
|
||||
#define THEME_COUNT 13
|
||||
|
||||
extern Theme g_themes[THEME_COUNT];
|
||||
extern S32 g_active_theme_idx;
|
||||
|
||||
// Convenience: pointer to the active theme
|
||||
static inline Theme *theme_active(void) { return &g_themes[g_active_theme_idx]; }
|
||||
Reference in New Issue
Block a user