99 lines
2.7 KiB
C
99 lines
2.7 KiB
C
#pragma once
|
|
// lexer.h -- Token types, tokenizer interface, and color scheme
|
|
//
|
|
// Follows the Focus editor pattern: a universal Token_Type enum shared by all
|
|
// languages, a per-byte token-type array on each buffer, and language-specific
|
|
// tokenizer functions that fill that array.
|
|
|
|
#include "base/base_core.h"
|
|
|
|
////////////////////////////////
|
|
// Token types (universal across all languages)
|
|
//
|
|
// The enum value doubles as an index into the color map, so CODE_DEFAULT
|
|
// must be 0. Keep this list ordered -- rendering indexes directly.
|
|
|
|
typedef enum Token_Type {
|
|
TOK_DEFAULT = 0,
|
|
|
|
TOK_COMMENT,
|
|
TOK_MULTILINE_COMMENT,
|
|
|
|
TOK_STRING_LITERAL,
|
|
TOK_CHAR_LITERAL,
|
|
|
|
TOK_NUMBER,
|
|
TOK_IDENTIFIER,
|
|
TOK_FUNCTION,
|
|
|
|
TOK_KEYWORD,
|
|
TOK_TYPE,
|
|
TOK_VALUE, // true, false, NULL, nullptr
|
|
TOK_MODIFIER, // const, static, volatile, extern ...
|
|
|
|
TOK_DIRECTIVE, // #include, #define, ...
|
|
TOK_PUNCTUATION,
|
|
TOK_OPERATION,
|
|
|
|
TOK_INVALID,
|
|
|
|
TOK_COUNT
|
|
} Token_Type;
|
|
|
|
////////////////////////////////
|
|
// Language enum
|
|
|
|
typedef enum Lang {
|
|
LANG_PLAIN_TEXT = 0,
|
|
LANG_C,
|
|
LANG_GO,
|
|
LANG_JS,
|
|
LANG_LUA,
|
|
LANG_SQL,
|
|
LANG_COUNT
|
|
} Lang;
|
|
|
|
// Human-readable language names (indexed by Lang)
|
|
extern const char *g_lang_names[LANG_COUNT];
|
|
|
|
////////////////////////////////
|
|
// Tokenizer state (shared by all language tokenizers)
|
|
|
|
typedef struct Tokenizer {
|
|
const char *buf; // start of buffer data
|
|
const char *t; // current scan cursor
|
|
const char *max_t; // one past end
|
|
const char *start_t; // cursor at start of current token
|
|
} Tokenizer;
|
|
|
|
////////////////////////////////
|
|
// Token (returned by get_next_token functions)
|
|
|
|
typedef struct Token {
|
|
S32 start; // byte offset into buffer
|
|
S32 len; // byte length of token
|
|
Token_Type type;
|
|
} Token;
|
|
|
|
////////////////////////////////
|
|
// Language tokenizer function signature
|
|
//
|
|
// A tokenizer function takes a buffer's data + length, and writes token types
|
|
// into the `out_tokens` array (one byte per source byte, same length as data).
|
|
// This is the Focus "paint the token array" approach.
|
|
|
|
typedef void (*LexerTokenizeFn)(const char *data, S32 length, U8 *out_tokens);
|
|
|
|
// Get the tokenizer function for a language.
|
|
static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang);
|
|
|
|
// Detect language from a file name / extension.
|
|
static Lang lexer_detect_lang(const char *filename);
|
|
|
|
////////////////////////////////
|
|
// Common tokenizer helpers (usable by all language tokenizers)
|
|
|
|
static void tokenizer_init(Tokenizer *tok, const char *data, S32 length);
|
|
static void tokenizer_eat_whitespace(Tokenizer *tok);
|
|
static void tokenizer_eat_until_newline(Tokenizer *tok);
|