#pragma once
// lexer.h -- Token types, tokenizer interface, and color scheme
//
// Follows the Focus editor pattern: a universal Token_Type enum shared by all
// languages, a per-byte token-type array on each buffer, and language-specific
// tokenizer functions that fill that array.

#include "base/base_core.h"

////////////////////////////////
// Token types (universal across all languages)
//
// The enum value doubles as an index into the color map, so CODE_DEFAULT
// must be 0.  Keep this list ordered -- rendering indexes directly.

typedef enum Token_Type {
    TOK_DEFAULT = 0,

    TOK_COMMENT,
    TOK_MULTILINE_COMMENT,

    TOK_STRING_LITERAL,
    TOK_CHAR_LITERAL,

    TOK_NUMBER,
    TOK_IDENTIFIER,
    TOK_FUNCTION,

    TOK_KEYWORD,
    TOK_TYPE,
    TOK_VALUE,          // true, false, NULL, nullptr
    TOK_MODIFIER,       // const, static, volatile, extern ...

    TOK_DIRECTIVE,      // #include, #define, ...
    TOK_PUNCTUATION,
    TOK_OPERATION,

    TOK_INVALID,

    TOK_COUNT
} Token_Type;

////////////////////////////////
// Language enum

typedef enum Lang {
    LANG_PLAIN_TEXT = 0,
    LANG_C,
    LANG_GO,
    LANG_JS,
    LANG_LUA,
    LANG_SQL,
    LANG_COUNT
} Lang;

// Human-readable language names (indexed by Lang)
extern const char *g_lang_names[LANG_COUNT];

////////////////////////////////
// Tokenizer state (shared by all language tokenizers)

typedef struct Tokenizer {
    const char *buf;     // start of buffer data
    const char *t;       // current scan cursor
    const char *max_t;   // one past end
    const char *start_t; // cursor at start of current token
} Tokenizer;

////////////////////////////////
// Token (returned by get_next_token functions)

typedef struct Token {
    S32        start;    // byte offset into buffer
    S32        len;      // byte length of token
    Token_Type type;
} Token;

////////////////////////////////
// Language tokenizer function signature
//
// A tokenizer function takes a buffer's data + length, and writes token types
// into the `out_tokens` array (one byte per source byte, same length as data).
// This is the Focus "paint the token array" approach.

typedef void (*LexerTokenizeFn)(const char *data, S32 length, U8 *out_tokens);

// Get the tokenizer function for a language.
static LexerTokenizeFn lexer_get_tokenize_fn(Lang lang);

// Detect language from a file name / extension.
static Lang lexer_detect_lang(const char *filename);

////////////////////////////////
// Common tokenizer helpers (usable by all language tokenizers)

static void tokenizer_init(Tokenizer *tok, const char *data, S32 length);
static void tokenizer_eat_whitespace(Tokenizer *tok);
static void tokenizer_eat_until_newline(Tokenizer *tok);