package lexer
import (
"html"
"strings"
)
// C syntax highlighting. Same contract as the rest of the package (see lexer.go).
//
// The classification tables below MIRROR the ones in kjøl's own C layer, c/lexer/lexer_c.c
// — a highlighter that colours the same source inside the editor it was written for. Two
// highlighters for one language is one too many, but they cannot share: that one is C
// compiled into an editor, this one is Go compiled to WebAssembly, and nothing is upstream
// of both. So they are two tables that agree, and this comment is the thing that says they
// have to. If you add a type there, add it here.
//
// That is also why U8/S32/F64/B32 are in the type table. They are not C types — they are
// kjøl's — but this highlighter exists to render kjøl's C, and a page about base_core.h
// that paints its own typedefs as bare identifiers has failed at the one job it has.
// The reserved words. Order of the maps below is also the order they are tested in, and
// the categories do not overlap.
var cKeywords = map[string]bool{
"auto": true, "break": true, "case": true, "continue": true, "default": true,
"do": true, "else": true, "enum": true, "for": true, "goto": true, "if": true,
"inline": true, "restrict": true, "return": true, "sizeof": true, "struct": true,
"switch": true, "typedef": true, "union": true, "while": true,
"_Alignas": true, "alignas": true, "_Alignof": true, "alignof": true,
"_Atomic": true, "_Generic": true, "_Noreturn": true,
"static_assert": true, "_Static_assert": true,
}
// Storage class and qualifiers. They read as keywords and are coloured as keywords — the
// C layer's lexer keeps them as a separate TOK_MODIFIER, but that distinction buys a
// reader of a documentation page nothing, so it is not carried over.
var cModifiers = map[string]bool{
"const": true, "constexpr": true, "extern": true, "register": true, "signed": true,
"static": true, "unsigned": true, "volatile": true,
"thread_local": true, "_Thread_local": true,
// Not C. base_core.h's three names for the three meanings of `static`, and the whole
// point of them is that they are visible — so they are coloured like what they are.
"internal": true, "global": true, "local_persist": true,
}
var cTypes = map[string]bool{
"char": true, "double": true, "float": true, "int": true, "long": true,
"short": true, "void": true, "bool": true, "_Bool": true,
"_Complex": true, "_Imaginary": true,
"int8_t": true, "int16_t": true, "int32_t": true, "int64_t": true,
"uint8_t": true, "uint16_t": true, "uint32_t": true, "uint64_t": true,
"size_t": true, "ssize_t": true, "ptrdiff_t": true,
"intptr_t": true, "uintptr_t": true, "nullptr_t": true, "FILE": true,
// kjøl's base layer (c/base/base_core.h).
"U8": true, "U16": true, "U32": true, "U64": true,
"S8": true, "S16": true, "S32": true, "S64": true,
"B32": true, "F32": true, "F64": true,
}
var cValues = map[string]bool{
"false": true, "true": true, "NULL": true, "nullptr": true,
}
// HighlightC turns C source into HTML with the tokens wrapped in coloured spans.
//
// The result is meant for vdom.Raw inside a : it contains no block elements and
// preserves the source's whitespace exactly, so the does the layout.
func HighlightC(src string) string {
var b strings.Builder
b.Grow(len(src) * 2)
// The types this snippet declares about itself.
//
// C cannot be highlighted correctly in one pass, and this is the reason: `Arena *a` is
// a declaration and `a * b` is a multiplication, and they are the same three tokens.
// Telling them apart needs to know that Arena is a type — which is the oldest problem
// in parsing C, and the reason C compilers feed the symbol table back into the lexer.
//
// The alternative most highlighters take is to guess from the spacing, which is wrong
// as often as the author's style differs from theirs. So this does not guess. It reads
// the declarations first — `typedef struct Arena {…} Arena;` says Arena is a type, in
// so many words — and then highlights knowing what the snippet said. A type the
// snippet never declares stays an identifier, which is honest: nothing in the text the
// reader is looking at claims otherwise.
declared := cDeclaredTypes(src)
// Two pieces of state, and both exist for the preprocessor.
//
// lineStart tracks whether we have seen anything but whitespace since the last
// newline, because a `#` is only a directive at the start of a line — everywhere else
// it is the stringize or paste operator inside a macro body, and painting THAT as a
// directive turns the inside of every macro red.
//
// inInclude is set for the rest of the line after `#include`, and it is what makes
// a header path rather than a less-than, an identifier, a dot and a
// greater-than. It is the one place in C where `<` does not mean what it usually does.
lineStart := true
inInclude := false
// prevWord is the last identifier-shaped token seen. After `struct`, `union` or `enum`
// the next identifier IS a type name — that one is syntax, not a guess, so it is worth
// tracking one word of history to get it right.
prevWord := ""
i := 0
for i < len(src) {
c := src[i]
switch {
case c == '\n':
b.WriteByte('\n')
lineStart = true
inInclude = false
prevWord = ""
i++
continue
case c == ' ' || c == '\t' || c == '\r':
b.WriteByte(c)
i++
continue // whitespace does not end lineStart, and does not clear prevWord
// Line comment. // is C99 and every C anyone writes today uses it.
case c == '/' && i+1 < len(src) && src[i+1] == '/':
end := strings.IndexByte(src[i:], '\n')
if end < 0 {
end = len(src)
} else {
end += i
}
span(&b, CommentClass, src[i:end])
i = end
// Block comment.
case c == '/' && i+1 < len(src) && src[i+1] == '*':
end := strings.Index(src[i+2:], "*/")
if end < 0 {
end = len(src)
} else {
end = i + 2 + end + 2
}
span(&b, CommentClass, src[i:end])
i = end
// A preprocessor directive: `#` and the word after it, and only at the start of a
// line. `#define`, `#include`, `#ifdef`, `#pragma`.
case c == '#' && lineStart:
j := i + 1
for j < len(src) && (src[j] == ' ' || src[j] == '\t') {
j++ // `# define` is legal, and rare, and free to support
}
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
span(&b, DirectiveClass, word)
// The rest of an #include line reads differently. Nothing else does.
if strings.HasSuffix(word, "include") || strings.HasSuffix(word, "import") {
inInclude = true
}
i = j
// — but ONLY on an #include line. Anywhere else this is an operator.
case c == '<' && inInclude:
end := i + 1
for end < len(src) && src[end] != '>' && src[end] != '\n' {
end++
}
if end < len(src) && src[end] == '>' {
end++ // include the closing bracket
}
span(&b, StringClass, src[i:end])
i = end
case c == '"':
i = quoted(&b, src, i, '"', true)
// A character literal. In C this is an INT, not a string — but every editor paints
// it like a string, and a reader looking for '\0' is looking for a literal.
case c == '\'':
i = quoted(&b, src, i, '\'', true)
case isDigit(c):
j := cNumberEnd(src, i)
span(&b, NumberClass, src[i:j])
i = j
case isIdentStart(c):
j := i
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
switch {
case cKeywords[word]:
span(&b, KeywordClass, word)
case cModifiers[word]:
span(&b, KeywordClass, word)
case cTypes[word]:
span(&b, TypeClass, word)
case cValues[word]:
span(&b, NumberClass, word) // NULL and true are constants; colour them as such
case declared[word]:
// The snippet said so itself, in a typedef or a struct tag. Not a guess.
span(&b, TypeClass, word)
case prevWord == "struct" || prevWord == "union" || prevWord == "enum":
// Syntax, not a guess: what follows one of these IS a type name.
span(&b, TypeClass, word)
case isTypeSuffixed(word):
span(&b, TypeClass, word)
case callAhead(src, j):
span(&b, FuncClass, word)
default:
b.WriteString(html.EscapeString(word))
}
prevWord = word
lineStart = false
i = j
continue
default:
b.WriteString(html.EscapeString(string(c)))
i++
}
lineStart = false
prevWord = ""
}
return b.String()
}
// cDeclaredTypes reads a snippet's type declarations and returns the names they introduce.
//
// It is a scan, not a parse, and it recognises exactly two shapes — the two that declare a
// type name in C:
//
// struct Arena / union Foo / enum Lang the word after the tag is a type
// typedef ... Name ; the last word before the semicolon
// typedef ... (*Name)(...) ; unless it is a function pointer, whose
// name hides inside the parens
//
// The typedef rule is why brace depth is tracked. `typedef struct Arena { U8 *base; } Arena;`
// contains semicolons that do not end it, and taking the first one would declare a type
// called `base`.
//
// Comments and string literals are skipped, or the word `struct` inside a comment would
// declare the next word in the prose as a type.
func cDeclaredTypes(src string) map[string]bool {
types := map[string]bool{}
inTypedef := false // inside a typedef statement, up to its semicolon
lastWord := "" // last identifier seen in it — the name, for the common shape
fnPtrName := "" // ...unless it is a function pointer, whose name is in (*Name)
prevWord := "" // one word of history, for `struct Arena`
depth := 0 // brace depth: only a depth-0 semicolon ends the typedef
i := 0
for i < len(src) {
c := src[i]
switch {
case c == ' ' || c == '\t' || c == '\r' || c == '\n':
i++ // whitespace must NOT clear prevWord: `struct Arena` has a space in it
case c == '/' && i+1 < len(src) && src[i+1] == '/':
if end := strings.IndexByte(src[i:], '\n'); end < 0 {
i = len(src)
} else {
i += end
}
case c == '/' && i+1 < len(src) && src[i+1] == '*':
if end := strings.Index(src[i+2:], "*/"); end < 0 {
i = len(src)
} else {
i += 2 + end + 2
}
case c == '"' || c == '\'':
i = skipQuoted(src, i, c)
case c == '{':
depth++
i++
case c == '}':
depth--
i++
// (*Name)(...) — a function-pointer typedef. The name is here and nowhere else.
case c == '(' && i+1 < len(src) && src[i+1] == '*':
j := i + 2
for j < len(src) && (src[j] == ' ' || src[j] == '\t') {
j++
}
k := j
for k < len(src) && isIdentPart(src[k]) {
k++
}
if inTypedef && k > j {
fnPtrName = src[j:k]
}
i = k
prevWord = ""
case c == ';' && depth == 0:
if inTypedef {
name := fnPtrName
if name == "" {
name = lastWord
}
if name != "" {
types[name] = true
}
}
inTypedef, lastWord, fnPtrName, prevWord = false, "", "", ""
i++
case isIdentStart(c):
j := i
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
if word == "typedef" {
inTypedef, lastWord, fnPtrName = true, "", ""
} else {
if prevWord == "struct" || prevWord == "union" || prevWord == "enum" {
types[word] = true
}
if inTypedef {
lastWord = word
}
}
prevWord = word
i = j
default:
prevWord = ""
i++
}
}
return types
}
// skipQuoted returns the index one past a quoted literal, without emitting anything.
func skipQuoted(src string, i int, quote byte) int {
j := i + 1
for j < len(src) {
if src[j] == '\\' && j+1 < len(src) {
j += 2
continue
}
if src[j] == quote {
return j + 1
}
if src[j] == '\n' {
return j // unterminated
}
j++
}
return j
}
// isTypeSuffixed reports whether a name ends in _t.
//
// This is a HEURISTIC, and the only one in this file. C has no way to know what is a type
// without a symbol table, and building one to colour a documentation snippet would be
// absurd — so the choice is between guessing and not guessing. `_t` is the one convention
// universal enough to guess on: it is what the standard library does (size_t, uint32_t),
// and a project that uses it for something that is not a type is doing so to be confusing.
//
// Everything else that is not in the tables above stays an identifier. A highlighter that
// paints too little is quietly unhelpful; one that paints too much is actively misleading.
func isTypeSuffixed(word string) bool {
return len(word) > 2 && strings.HasSuffix(word, "_t")
}
// cNumberEnd returns the index one past the number starting at i.
//
// It is its own function because C numbers are a small zoo: 0x25, 0b1011, 1024, 3.14f,
// 1e9, 0xFFull. Go's number scanner (code.go) is a single character-class loop, which is
// enough for Go and would swallow the `x` of `0xFF` into a hex-ish blur and then choke on
// the `ull`.
func cNumberEnd(src string, i int) int {
j := i
switch {
case src[j] == '0' && j+1 < len(src) && (src[j+1] == 'x' || src[j+1] == 'X'):
j += 2
for j < len(src) && isHexDigit(src[j]) {
j++
}
case src[j] == '0' && j+1 < len(src) && (src[j+1] == 'b' || src[j+1] == 'B'):
j += 2
for j < len(src) && (src[j] == '0' || src[j] == '1') {
j++
}
default:
for j < len(src) && (isDigit(src[j]) || src[j] == '.') {
j++
}
// An exponent, but only if it actually has digits after it — otherwise the `e` of
// `1end` is a number and `nd` is an identifier, which is nonsense.
if j < len(src) && (src[j] == 'e' || src[j] == 'E') {
k := j + 1
if k < len(src) && (src[k] == '+' || src[k] == '-') {
k++
}
if k < len(src) && isDigit(src[k]) {
for k < len(src) && isDigit(src[k]) {
k++
}
j = k
}
}
}
// u, U, l, L, f, F, in any order and any number: 0xFFull, 1.0f, 10UL.
for j < len(src) && isNumSuffix(src[j]) {
j++
}
return j
}
func isHexDigit(c byte) bool {
return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
func isNumSuffix(c byte) bool {
return c == 'u' || c == 'U' || c == 'l' || c == 'L' || c == 'f' || c == 'F'
}