424 lines
13 KiB
C
424 lines
13 KiB
C
// lexer_c.c -- C language tokenizer
|
|
//
|
|
// Follows the Focus editor pattern: a get_next_token() loop that advances
|
|
// a cursor through the source, producing Token structs. After each token,
|
|
// we paint the per-byte token-type array with the token's type.
|
|
// A second pass retroactively marks identifiers followed by '(' as functions.
|
|
|
|
#include "lexer/lexer.h"
|
|
#include <ctype.h>
|
|
#include <string.h>
|
|
|
|
////////////////////////////////
|
|
// C keyword tables
|
|
|
|
typedef struct KeywordEntry {
|
|
const char *word;
|
|
Token_Type type;
|
|
} KeywordEntry;
|
|
|
|
static const KeywordEntry c_keywords[] = {
|
|
// Control-flow & language keywords
|
|
{"auto", TOK_KEYWORD},
|
|
{"break", TOK_KEYWORD},
|
|
{"case", TOK_KEYWORD},
|
|
{"continue", TOK_KEYWORD},
|
|
{"default", TOK_KEYWORD},
|
|
{"do", TOK_KEYWORD},
|
|
{"else", TOK_KEYWORD},
|
|
{"enum", TOK_KEYWORD},
|
|
{"for", TOK_KEYWORD},
|
|
{"goto", TOK_KEYWORD},
|
|
{"if", TOK_KEYWORD},
|
|
{"inline", TOK_KEYWORD},
|
|
{"restrict", TOK_KEYWORD},
|
|
{"return", TOK_KEYWORD},
|
|
{"sizeof", TOK_KEYWORD},
|
|
{"struct", TOK_KEYWORD},
|
|
{"switch", TOK_KEYWORD},
|
|
{"typedef", TOK_KEYWORD},
|
|
{"union", TOK_KEYWORD},
|
|
{"while", TOK_KEYWORD},
|
|
{"_Alignas", TOK_KEYWORD},
|
|
{"alignas", TOK_KEYWORD},
|
|
{"_Alignof", TOK_KEYWORD},
|
|
{"alignof", TOK_KEYWORD},
|
|
{"_Atomic", TOK_KEYWORD},
|
|
{"_Generic", TOK_KEYWORD},
|
|
{"_Noreturn", TOK_KEYWORD},
|
|
{"static_assert", TOK_KEYWORD},
|
|
{"_Static_assert", TOK_KEYWORD},
|
|
|
|
// Types
|
|
{"char", TOK_TYPE},
|
|
{"double", TOK_TYPE},
|
|
{"float", TOK_TYPE},
|
|
{"int", TOK_TYPE},
|
|
{"long", TOK_TYPE},
|
|
{"short", TOK_TYPE},
|
|
{"void", TOK_TYPE},
|
|
{"bool", TOK_TYPE},
|
|
{"_Bool", TOK_TYPE},
|
|
{"_Complex", TOK_TYPE},
|
|
{"_Imaginary", TOK_TYPE},
|
|
// stdint
|
|
{"int8_t", TOK_TYPE},
|
|
{"int16_t", TOK_TYPE},
|
|
{"int32_t", TOK_TYPE},
|
|
{"int64_t", TOK_TYPE},
|
|
{"uint8_t", TOK_TYPE},
|
|
{"uint16_t", TOK_TYPE},
|
|
{"uint32_t", TOK_TYPE},
|
|
{"uint64_t", TOK_TYPE},
|
|
{"size_t", TOK_TYPE},
|
|
{"ssize_t", TOK_TYPE},
|
|
{"ptrdiff_t", TOK_TYPE},
|
|
{"intptr_t", TOK_TYPE},
|
|
{"uintptr_t", TOK_TYPE},
|
|
{"nullptr_t", TOK_TYPE},
|
|
{"FILE", TOK_TYPE},
|
|
// Project types
|
|
{"U8", TOK_TYPE},
|
|
{"U16", TOK_TYPE},
|
|
{"U32", TOK_TYPE},
|
|
{"U64", TOK_TYPE},
|
|
{"S8", TOK_TYPE},
|
|
{"S16", TOK_TYPE},
|
|
{"S32", TOK_TYPE},
|
|
{"S64", TOK_TYPE},
|
|
{"B32", TOK_TYPE},
|
|
{"F32", TOK_TYPE},
|
|
{"F64", TOK_TYPE},
|
|
|
|
// Values
|
|
{"false", TOK_VALUE},
|
|
{"true", TOK_VALUE},
|
|
{"NULL", TOK_VALUE},
|
|
{"nullptr", TOK_VALUE},
|
|
|
|
// Modifiers
|
|
{"const", TOK_MODIFIER},
|
|
{"constexpr", TOK_MODIFIER},
|
|
{"extern", TOK_MODIFIER},
|
|
{"register", TOK_MODIFIER},
|
|
{"signed", TOK_MODIFIER},
|
|
{"static", TOK_MODIFIER},
|
|
{"unsigned", TOK_MODIFIER},
|
|
{"volatile", TOK_MODIFIER},
|
|
{"thread_local", TOK_MODIFIER},
|
|
{"_Thread_local", TOK_MODIFIER},
|
|
};
|
|
|
|
#define C_KEYWORD_COUNT (S32)(sizeof(c_keywords) / sizeof(c_keywords[0]))
|
|
|
|
static Token_Type c_lookup_keyword(const char *word, S32 len) {
|
|
for (S32 i = 0; i < C_KEYWORD_COUNT; i++) {
|
|
const char *kw = c_keywords[i].word;
|
|
S32 kwlen = (S32)strlen(kw);
|
|
if (kwlen == len && memcmp(kw, word, len) == 0)
|
|
return c_keywords[i].type;
|
|
}
|
|
return TOK_IDENTIFIER;
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Character helpers
|
|
|
|
static B32 c_is_ident_start(char c) {
|
|
return isalpha((unsigned char)c) || c == '_';
|
|
}
|
|
|
|
static B32 c_is_ident_char(char c) {
|
|
return isalnum((unsigned char)c) || c == '_';
|
|
}
|
|
|
|
static B32 c_is_hex(char c) {
|
|
return isdigit((unsigned char)c) ||
|
|
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Individual token parsers
|
|
|
|
static Token c_parse_identifier(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
const char *begin = tok->t;
|
|
while (tok->t < tok->max_t && c_is_ident_char(*tok->t))
|
|
tok->t++;
|
|
S32 len = (S32)(tok->t - begin);
|
|
token.type = c_lookup_keyword(begin, len);
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
static Token c_parse_number(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
token.type = TOK_NUMBER;
|
|
|
|
char start_char = *tok->t;
|
|
tok->t++;
|
|
if (tok->t >= tok->max_t) goto done;
|
|
|
|
if (start_char == '0' && tok->t < tok->max_t) {
|
|
if (*tok->t == 'x' || *tok->t == 'X') {
|
|
tok->t++;
|
|
while (tok->t < tok->max_t && c_is_hex(*tok->t)) tok->t++;
|
|
goto suffixes;
|
|
}
|
|
if (*tok->t == 'b' || *tok->t == 'B') {
|
|
tok->t++;
|
|
while (tok->t < tok->max_t && (*tok->t == '0' || *tok->t == '1')) tok->t++;
|
|
goto suffixes;
|
|
}
|
|
}
|
|
|
|
// Decimal / float
|
|
{
|
|
B32 seen_dot = 0;
|
|
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '.')) {
|
|
if (*tok->t == '.') {
|
|
if (seen_dot) break;
|
|
seen_dot = 1;
|
|
}
|
|
tok->t++;
|
|
}
|
|
// Exponent
|
|
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
|
tok->t++;
|
|
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
|
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
|
}
|
|
// Float suffix
|
|
if (tok->t < tok->max_t && seen_dot) {
|
|
if (*tok->t == 'f' || *tok->t == 'F' || *tok->t == 'l' || *tok->t == 'L')
|
|
tok->t++;
|
|
goto done;
|
|
}
|
|
}
|
|
|
|
suffixes:
|
|
// Integer suffixes: u, l, ll, ul, ull, lu, llu
|
|
if (tok->t < tok->max_t && (*tok->t == 'u' || *tok->t == 'U')) {
|
|
tok->t++;
|
|
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) {
|
|
tok->t++;
|
|
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) tok->t++;
|
|
}
|
|
} else if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) {
|
|
tok->t++;
|
|
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) tok->t++;
|
|
if (tok->t < tok->max_t && (*tok->t == 'u' || *tok->t == 'U')) tok->t++;
|
|
}
|
|
|
|
done:
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
static Token c_parse_string(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
token.type = TOK_STRING_LITERAL;
|
|
|
|
B32 escape = 0;
|
|
tok->t++; // skip opening "
|
|
while (tok->t < tok->max_t && *tok->t != '\n') {
|
|
if (*tok->t == '"' && !escape) { tok->t++; break; }
|
|
escape = !escape && (*tok->t == '\\');
|
|
tok->t++;
|
|
}
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
static Token c_parse_char_literal(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
token.type = TOK_CHAR_LITERAL;
|
|
|
|
tok->t++; // skip opening '
|
|
if (tok->t < tok->max_t && *tok->t == '\\') {
|
|
tok->t++; // escape char
|
|
if (tok->t < tok->max_t && *tok->t != '\n') tok->t++; // the escaped char
|
|
} else if (tok->t < tok->max_t && *tok->t != '\n' && *tok->t != '\'') {
|
|
tok->t++; // the char
|
|
}
|
|
if (tok->t < tok->max_t && *tok->t == '\'') tok->t++; // closing '
|
|
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
static Token c_parse_slash_or_comment(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
|
|
tok->t++; // skip '/'
|
|
if (tok->t >= tok->max_t) {
|
|
token.type = TOK_OPERATION;
|
|
token.len = 1;
|
|
return token;
|
|
}
|
|
|
|
if (*tok->t == '/') {
|
|
// Line comment
|
|
token.type = TOK_COMMENT;
|
|
tok->t++;
|
|
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
|
} else if (*tok->t == '*') {
|
|
// Block comment
|
|
token.type = TOK_MULTILINE_COMMENT;
|
|
tok->t++;
|
|
while (tok->t < tok->max_t) {
|
|
if (*tok->t == '*' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '/') {
|
|
tok->t += 2;
|
|
break;
|
|
}
|
|
tok->t++;
|
|
}
|
|
} else if (*tok->t == '=') {
|
|
token.type = TOK_OPERATION;
|
|
tok->t++;
|
|
} else {
|
|
token.type = TOK_OPERATION;
|
|
}
|
|
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
static Token c_parse_directive(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
token.type = TOK_DIRECTIVE;
|
|
|
|
tok->t++; // skip '#'
|
|
// Skip whitespace between # and directive name
|
|
while (tok->t < tok->max_t && (*tok->t == ' ' || *tok->t == '\t')) tok->t++;
|
|
// Read directive name
|
|
while (tok->t < tok->max_t && isalpha((unsigned char)*tok->t)) tok->t++;
|
|
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
// Two-char operator check
|
|
static Token c_parse_operator(Tokenizer *tok) {
|
|
Token token;
|
|
token.start = (S32)(tok->start_t - tok->buf);
|
|
token.type = TOK_OPERATION;
|
|
|
|
char c = *tok->t;
|
|
tok->t++;
|
|
|
|
if (tok->t < tok->max_t) {
|
|
char n = *tok->t;
|
|
// Two-character operators
|
|
if ((c == '=' && n == '=') || (c == '!' && n == '=') ||
|
|
(c == '<' && n == '=') || (c == '>' && n == '=') ||
|
|
(c == '+' && n == '=') || (c == '-' && n == '=') ||
|
|
(c == '*' && n == '=') || (c == '%' && n == '=') ||
|
|
(c == '&' && n == '=') || (c == '|' && n == '=') ||
|
|
(c == '^' && n == '=') || (c == '+' && n == '+') ||
|
|
(c == '-' && n == '-') || (c == '&' && n == '&') ||
|
|
(c == '|' && n == '|') || (c == '<' && n == '<') ||
|
|
(c == '>' && n == '>') || (c == '-' && n == '>')) {
|
|
tok->t++;
|
|
}
|
|
}
|
|
|
|
// Handle negative number literal: operator '-' followed by digit
|
|
// (not done here -- handled by caller context if needed)
|
|
|
|
token.len = (S32)(tok->t - tok->start_t);
|
|
return token;
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Main get_next_token
|
|
|
|
static Token c_get_next_token(Tokenizer *tok) {
|
|
tokenizer_eat_whitespace(tok);
|
|
|
|
Token token = {0};
|
|
token.start = (S32)(tok->t - tok->buf);
|
|
token.type = TOK_DEFAULT;
|
|
|
|
if (tok->t >= tok->max_t) {
|
|
token.len = 0;
|
|
return token; // EOF
|
|
}
|
|
|
|
tok->start_t = tok->t;
|
|
char c = *tok->t;
|
|
|
|
if (c_is_ident_start(c)) {
|
|
return c_parse_identifier(tok);
|
|
}
|
|
if (isdigit((unsigned char)c)) {
|
|
return c_parse_number(tok);
|
|
}
|
|
|
|
switch (c) {
|
|
case '"': return c_parse_string(tok);
|
|
case '\'': return c_parse_char_literal(tok);
|
|
case '/': return c_parse_slash_or_comment(tok);
|
|
case '#': return c_parse_directive(tok);
|
|
|
|
// Punctuation
|
|
case ';': case ',': case '.':
|
|
case '{': case '}': case '(': case ')': case '[': case ']':
|
|
case '\\':
|
|
token.type = TOK_PUNCTUATION;
|
|
tok->t++;
|
|
token.len = 1;
|
|
return token;
|
|
|
|
// Operators
|
|
case '=': case '!': case '<': case '>':
|
|
case '+': case '-': case '*': case '%':
|
|
case '&': case '|': case '^': case '~':
|
|
case '?': case ':':
|
|
return c_parse_operator(tok);
|
|
|
|
default:
|
|
token.type = TOK_INVALID;
|
|
tok->t++;
|
|
token.len = 1;
|
|
return token;
|
|
}
|
|
}
|
|
|
|
////////////////////////////////
|
|
// tokenize_c -- main entry point
|
|
//
|
|
// Tokenizes the full buffer and paints out_tokens[] with token types.
|
|
// Second pass: retroactively marks identifiers before '(' as functions.
|
|
|
|
static void tokenize_c(const char *data, S32 length, U8 *out_tokens) {
|
|
memset(out_tokens, TOK_DEFAULT, length);
|
|
|
|
Tokenizer tok;
|
|
tokenizer_init(&tok, data, length);
|
|
|
|
Token prev = {0};
|
|
prev.type = TOK_DEFAULT;
|
|
|
|
while (tok.t < tok.max_t) {
|
|
Token token = c_get_next_token(&tok);
|
|
if (token.len == 0) break;
|
|
|
|
paint_token(out_tokens, token.start, token.len, token.type);
|
|
|
|
// Retroactively mark identifier before '(' as a function
|
|
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
|
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
|
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
|
}
|
|
|
|
prev = token;
|
|
}
|
|
}
|