add c and jai
This commit is contained in:
423
c/lexer/lexer_c.c
Normal file
423
c/lexer/lexer_c.c
Normal file
@@ -0,0 +1,423 @@
|
||||
// lexer_c.c -- C language tokenizer
|
||||
//
|
||||
// Follows the Focus editor pattern: a get_next_token() loop that advances
|
||||
// a cursor through the source, producing Token structs. After each token,
|
||||
// we paint the per-byte token-type array with the token's type.
|
||||
// A second pass retroactively marks identifiers followed by '(' as functions.
|
||||
|
||||
#include "lexer/lexer.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
////////////////////////////////
|
||||
// C keyword tables
|
||||
|
||||
typedef struct KeywordEntry {
|
||||
const char *word;
|
||||
Token_Type type;
|
||||
} KeywordEntry;
|
||||
|
||||
static const KeywordEntry c_keywords[] = {
|
||||
// Control-flow & language keywords
|
||||
{"auto", TOK_KEYWORD},
|
||||
{"break", TOK_KEYWORD},
|
||||
{"case", TOK_KEYWORD},
|
||||
{"continue", TOK_KEYWORD},
|
||||
{"default", TOK_KEYWORD},
|
||||
{"do", TOK_KEYWORD},
|
||||
{"else", TOK_KEYWORD},
|
||||
{"enum", TOK_KEYWORD},
|
||||
{"for", TOK_KEYWORD},
|
||||
{"goto", TOK_KEYWORD},
|
||||
{"if", TOK_KEYWORD},
|
||||
{"inline", TOK_KEYWORD},
|
||||
{"restrict", TOK_KEYWORD},
|
||||
{"return", TOK_KEYWORD},
|
||||
{"sizeof", TOK_KEYWORD},
|
||||
{"struct", TOK_KEYWORD},
|
||||
{"switch", TOK_KEYWORD},
|
||||
{"typedef", TOK_KEYWORD},
|
||||
{"union", TOK_KEYWORD},
|
||||
{"while", TOK_KEYWORD},
|
||||
{"_Alignas", TOK_KEYWORD},
|
||||
{"alignas", TOK_KEYWORD},
|
||||
{"_Alignof", TOK_KEYWORD},
|
||||
{"alignof", TOK_KEYWORD},
|
||||
{"_Atomic", TOK_KEYWORD},
|
||||
{"_Generic", TOK_KEYWORD},
|
||||
{"_Noreturn", TOK_KEYWORD},
|
||||
{"static_assert", TOK_KEYWORD},
|
||||
{"_Static_assert", TOK_KEYWORD},
|
||||
|
||||
// Types
|
||||
{"char", TOK_TYPE},
|
||||
{"double", TOK_TYPE},
|
||||
{"float", TOK_TYPE},
|
||||
{"int", TOK_TYPE},
|
||||
{"long", TOK_TYPE},
|
||||
{"short", TOK_TYPE},
|
||||
{"void", TOK_TYPE},
|
||||
{"bool", TOK_TYPE},
|
||||
{"_Bool", TOK_TYPE},
|
||||
{"_Complex", TOK_TYPE},
|
||||
{"_Imaginary", TOK_TYPE},
|
||||
// stdint
|
||||
{"int8_t", TOK_TYPE},
|
||||
{"int16_t", TOK_TYPE},
|
||||
{"int32_t", TOK_TYPE},
|
||||
{"int64_t", TOK_TYPE},
|
||||
{"uint8_t", TOK_TYPE},
|
||||
{"uint16_t", TOK_TYPE},
|
||||
{"uint32_t", TOK_TYPE},
|
||||
{"uint64_t", TOK_TYPE},
|
||||
{"size_t", TOK_TYPE},
|
||||
{"ssize_t", TOK_TYPE},
|
||||
{"ptrdiff_t", TOK_TYPE},
|
||||
{"intptr_t", TOK_TYPE},
|
||||
{"uintptr_t", TOK_TYPE},
|
||||
{"nullptr_t", TOK_TYPE},
|
||||
{"FILE", TOK_TYPE},
|
||||
// Project types
|
||||
{"U8", TOK_TYPE},
|
||||
{"U16", TOK_TYPE},
|
||||
{"U32", TOK_TYPE},
|
||||
{"U64", TOK_TYPE},
|
||||
{"S8", TOK_TYPE},
|
||||
{"S16", TOK_TYPE},
|
||||
{"S32", TOK_TYPE},
|
||||
{"S64", TOK_TYPE},
|
||||
{"B32", TOK_TYPE},
|
||||
{"F32", TOK_TYPE},
|
||||
{"F64", TOK_TYPE},
|
||||
|
||||
// Values
|
||||
{"false", TOK_VALUE},
|
||||
{"true", TOK_VALUE},
|
||||
{"NULL", TOK_VALUE},
|
||||
{"nullptr", TOK_VALUE},
|
||||
|
||||
// Modifiers
|
||||
{"const", TOK_MODIFIER},
|
||||
{"constexpr", TOK_MODIFIER},
|
||||
{"extern", TOK_MODIFIER},
|
||||
{"register", TOK_MODIFIER},
|
||||
{"signed", TOK_MODIFIER},
|
||||
{"static", TOK_MODIFIER},
|
||||
{"unsigned", TOK_MODIFIER},
|
||||
{"volatile", TOK_MODIFIER},
|
||||
{"thread_local", TOK_MODIFIER},
|
||||
{"_Thread_local", TOK_MODIFIER},
|
||||
};
|
||||
|
||||
#define C_KEYWORD_COUNT (S32)(sizeof(c_keywords) / sizeof(c_keywords[0]))
|
||||
|
||||
static Token_Type c_lookup_keyword(const char *word, S32 len) {
|
||||
for (S32 i = 0; i < C_KEYWORD_COUNT; i++) {
|
||||
const char *kw = c_keywords[i].word;
|
||||
S32 kwlen = (S32)strlen(kw);
|
||||
if (kwlen == len && memcmp(kw, word, len) == 0)
|
||||
return c_keywords[i].type;
|
||||
}
|
||||
return TOK_IDENTIFIER;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Character helpers
|
||||
|
||||
static B32 c_is_ident_start(char c) {
|
||||
return isalpha((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 c_is_ident_char(char c) {
|
||||
return isalnum((unsigned char)c) || c == '_';
|
||||
}
|
||||
|
||||
static B32 c_is_hex(char c) {
|
||||
return isdigit((unsigned char)c) ||
|
||||
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Individual token parsers
|
||||
|
||||
static Token c_parse_identifier(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
const char *begin = tok->t;
|
||||
while (tok->t < tok->max_t && c_is_ident_char(*tok->t))
|
||||
tok->t++;
|
||||
S32 len = (S32)(tok->t - begin);
|
||||
token.type = c_lookup_keyword(begin, len);
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_number(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_NUMBER;
|
||||
|
||||
char start_char = *tok->t;
|
||||
tok->t++;
|
||||
if (tok->t >= tok->max_t) goto done;
|
||||
|
||||
if (start_char == '0' && tok->t < tok->max_t) {
|
||||
if (*tok->t == 'x' || *tok->t == 'X') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && c_is_hex(*tok->t)) tok->t++;
|
||||
goto suffixes;
|
||||
}
|
||||
if (*tok->t == 'b' || *tok->t == 'B') {
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && (*tok->t == '0' || *tok->t == '1')) tok->t++;
|
||||
goto suffixes;
|
||||
}
|
||||
}
|
||||
|
||||
// Decimal / float
|
||||
{
|
||||
B32 seen_dot = 0;
|
||||
while (tok->t < tok->max_t && (isdigit((unsigned char)*tok->t) || *tok->t == '.')) {
|
||||
if (*tok->t == '.') {
|
||||
if (seen_dot) break;
|
||||
seen_dot = 1;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
// Exponent
|
||||
if (tok->t < tok->max_t && (*tok->t == 'e' || *tok->t == 'E')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == '+' || *tok->t == '-')) tok->t++;
|
||||
while (tok->t < tok->max_t && isdigit((unsigned char)*tok->t)) tok->t++;
|
||||
}
|
||||
// Float suffix
|
||||
if (tok->t < tok->max_t && seen_dot) {
|
||||
if (*tok->t == 'f' || *tok->t == 'F' || *tok->t == 'l' || *tok->t == 'L')
|
||||
tok->t++;
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
|
||||
suffixes:
|
||||
// Integer suffixes: u, l, ll, ul, ull, lu, llu
|
||||
if (tok->t < tok->max_t && (*tok->t == 'u' || *tok->t == 'U')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) tok->t++;
|
||||
}
|
||||
} else if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) {
|
||||
tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'l' || *tok->t == 'L')) tok->t++;
|
||||
if (tok->t < tok->max_t && (*tok->t == 'u' || *tok->t == 'U')) tok->t++;
|
||||
}
|
||||
|
||||
done:
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_string(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_STRING_LITERAL;
|
||||
|
||||
B32 escape = 0;
|
||||
tok->t++; // skip opening "
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') {
|
||||
if (*tok->t == '"' && !escape) { tok->t++; break; }
|
||||
escape = !escape && (*tok->t == '\\');
|
||||
tok->t++;
|
||||
}
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_char_literal(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_CHAR_LITERAL;
|
||||
|
||||
tok->t++; // skip opening '
|
||||
if (tok->t < tok->max_t && *tok->t == '\\') {
|
||||
tok->t++; // escape char
|
||||
if (tok->t < tok->max_t && *tok->t != '\n') tok->t++; // the escaped char
|
||||
} else if (tok->t < tok->max_t && *tok->t != '\n' && *tok->t != '\'') {
|
||||
tok->t++; // the char
|
||||
}
|
||||
if (tok->t < tok->max_t && *tok->t == '\'') tok->t++; // closing '
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_slash_or_comment(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
|
||||
tok->t++; // skip '/'
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.type = TOK_OPERATION;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
|
||||
if (*tok->t == '/') {
|
||||
// Line comment
|
||||
token.type = TOK_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t && *tok->t != '\n') tok->t++;
|
||||
} else if (*tok->t == '*') {
|
||||
// Block comment
|
||||
token.type = TOK_MULTILINE_COMMENT;
|
||||
tok->t++;
|
||||
while (tok->t < tok->max_t) {
|
||||
if (*tok->t == '*' && (tok->t + 1) < tok->max_t && *(tok->t + 1) == '/') {
|
||||
tok->t += 2;
|
||||
break;
|
||||
}
|
||||
tok->t++;
|
||||
}
|
||||
} else if (*tok->t == '=') {
|
||||
token.type = TOK_OPERATION;
|
||||
tok->t++;
|
||||
} else {
|
||||
token.type = TOK_OPERATION;
|
||||
}
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
static Token c_parse_directive(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_DIRECTIVE;
|
||||
|
||||
tok->t++; // skip '#'
|
||||
// Skip whitespace between # and directive name
|
||||
while (tok->t < tok->max_t && (*tok->t == ' ' || *tok->t == '\t')) tok->t++;
|
||||
// Read directive name
|
||||
while (tok->t < tok->max_t && isalpha((unsigned char)*tok->t)) tok->t++;
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
// Two-char operator check
|
||||
static Token c_parse_operator(Tokenizer *tok) {
|
||||
Token token;
|
||||
token.start = (S32)(tok->start_t - tok->buf);
|
||||
token.type = TOK_OPERATION;
|
||||
|
||||
char c = *tok->t;
|
||||
tok->t++;
|
||||
|
||||
if (tok->t < tok->max_t) {
|
||||
char n = *tok->t;
|
||||
// Two-character operators
|
||||
if ((c == '=' && n == '=') || (c == '!' && n == '=') ||
|
||||
(c == '<' && n == '=') || (c == '>' && n == '=') ||
|
||||
(c == '+' && n == '=') || (c == '-' && n == '=') ||
|
||||
(c == '*' && n == '=') || (c == '%' && n == '=') ||
|
||||
(c == '&' && n == '=') || (c == '|' && n == '=') ||
|
||||
(c == '^' && n == '=') || (c == '+' && n == '+') ||
|
||||
(c == '-' && n == '-') || (c == '&' && n == '&') ||
|
||||
(c == '|' && n == '|') || (c == '<' && n == '<') ||
|
||||
(c == '>' && n == '>') || (c == '-' && n == '>')) {
|
||||
tok->t++;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle negative number literal: operator '-' followed by digit
|
||||
// (not done here -- handled by caller context if needed)
|
||||
|
||||
token.len = (S32)(tok->t - tok->start_t);
|
||||
return token;
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// Main get_next_token
|
||||
|
||||
static Token c_get_next_token(Tokenizer *tok) {
|
||||
tokenizer_eat_whitespace(tok);
|
||||
|
||||
Token token = {0};
|
||||
token.start = (S32)(tok->t - tok->buf);
|
||||
token.type = TOK_DEFAULT;
|
||||
|
||||
if (tok->t >= tok->max_t) {
|
||||
token.len = 0;
|
||||
return token; // EOF
|
||||
}
|
||||
|
||||
tok->start_t = tok->t;
|
||||
char c = *tok->t;
|
||||
|
||||
if (c_is_ident_start(c)) {
|
||||
return c_parse_identifier(tok);
|
||||
}
|
||||
if (isdigit((unsigned char)c)) {
|
||||
return c_parse_number(tok);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '"': return c_parse_string(tok);
|
||||
case '\'': return c_parse_char_literal(tok);
|
||||
case '/': return c_parse_slash_or_comment(tok);
|
||||
case '#': return c_parse_directive(tok);
|
||||
|
||||
// Punctuation
|
||||
case ';': case ',': case '.':
|
||||
case '{': case '}': case '(': case ')': case '[': case ']':
|
||||
case '\\':
|
||||
token.type = TOK_PUNCTUATION;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
|
||||
// Operators
|
||||
case '=': case '!': case '<': case '>':
|
||||
case '+': case '-': case '*': case '%':
|
||||
case '&': case '|': case '^': case '~':
|
||||
case '?': case ':':
|
||||
return c_parse_operator(tok);
|
||||
|
||||
default:
|
||||
token.type = TOK_INVALID;
|
||||
tok->t++;
|
||||
token.len = 1;
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////
|
||||
// tokenize_c -- main entry point
|
||||
//
|
||||
// Tokenizes the full buffer and paints out_tokens[] with token types.
|
||||
// Second pass: retroactively marks identifiers before '(' as functions.
|
||||
|
||||
static void tokenize_c(const char *data, S32 length, U8 *out_tokens) {
|
||||
memset(out_tokens, TOK_DEFAULT, length);
|
||||
|
||||
Tokenizer tok;
|
||||
tokenizer_init(&tok, data, length);
|
||||
|
||||
Token prev = {0};
|
||||
prev.type = TOK_DEFAULT;
|
||||
|
||||
while (tok.t < tok.max_t) {
|
||||
Token token = c_get_next_token(&tok);
|
||||
if (token.len == 0) break;
|
||||
|
||||
paint_token(out_tokens, token.start, token.len, token.type);
|
||||
|
||||
// Retroactively mark identifier before '(' as a function
|
||||
if (token.type == TOK_PUNCTUATION && token.len == 1 &&
|
||||
data[token.start] == '(' && prev.type == TOK_IDENTIFIER) {
|
||||
paint_token(out_tokens, prev.start, prev.len, TOK_FUNCTION);
|
||||
}
|
||||
|
||||
prev = token;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user