kjol/go/lexer/lexer_c.go

package lexer

import (
	"html"
	"strings"
)

// C syntax highlighting. Same contract as the rest of the package (see lexer.go).
//
// The classification tables below MIRROR the ones in kjøl's own C layer, c/lexer/lexer_c.c
// — a highlighter that colours the same source inside the editor it was written for. Two
// highlighters for one language is one too many, but they cannot share: that one is C
// compiled into an editor, this one is Go compiled to WebAssembly, and nothing is upstream
// of both. So they are two tables that agree, and this comment is the thing that says they
// have to. If you add a type there, add it here.
//
// That is also why U8/S32/F64/B32 are in the type table. They are not C types — they are
// kjøl's — but this highlighter exists to render kjøl's C, and a page about base_core.h
// that paints its own typedefs as bare identifiers has failed at the one job it has.

// The reserved words. Order of the maps below is also the order they are tested in, and
// the categories do not overlap.
var cKeywords = map[string]bool{
	"auto": true, "break": true, "case": true, "continue": true, "default": true,
	"do": true, "else": true, "enum": true, "for": true, "goto": true, "if": true,
	"inline": true, "restrict": true, "return": true, "sizeof": true, "struct": true,
	"switch": true, "typedef": true, "union": true, "while": true,
	"_Alignas": true, "alignas": true, "_Alignof": true, "alignof": true,
	"_Atomic": true, "_Generic": true, "_Noreturn": true,
	"static_assert": true, "_Static_assert": true,
}

// Storage class and qualifiers. They read as keywords and are coloured as keywords — the
// C layer's lexer keeps them as a separate TOK_MODIFIER, but that distinction buys a
// reader of a documentation page nothing, so it is not carried over.
var cModifiers = map[string]bool{
	"const": true, "constexpr": true, "extern": true, "register": true, "signed": true,
	"static": true, "unsigned": true, "volatile": true,
	"thread_local": true, "_Thread_local": true,
	// Not C. base_core.h's three names for the three meanings of `static`, and the whole
	// point of them is that they are visible — so they are coloured like what they are.
	"internal": true, "global": true, "local_persist": true,
}

var cTypes = map[string]bool{
	"char": true, "double": true, "float": true, "int": true, "long": true,
	"short": true, "void": true, "bool": true, "_Bool": true,
	"_Complex": true, "_Imaginary": true,
	"int8_t": true, "int16_t": true, "int32_t": true, "int64_t": true,
	"uint8_t": true, "uint16_t": true, "uint32_t": true, "uint64_t": true,
	"size_t": true, "ssize_t": true, "ptrdiff_t": true,
	"intptr_t": true, "uintptr_t": true, "nullptr_t": true, "FILE": true,
	// kjøl's base layer (c/base/base_core.h).
	"U8": true, "U16": true, "U32": true, "U64": true,
	"S8": true, "S16": true, "S32": true, "S64": true,
	"B32": true, "F32": true, "F64": true,
}

var cValues = map[string]bool{
	"false": true, "true": true, "NULL": true, "nullptr": true,
}

// HighlightC turns C source into HTML with the tokens wrapped in coloured spans.
//
// The result is meant for vdom.Raw inside a <pre>: it contains no block elements and
// preserves the source's whitespace exactly, so the <pre> does the layout.
func HighlightC(src string) string {
	var b strings.Builder
	b.Grow(len(src) * 2)

	// The types this snippet declares about itself.
	//
	// C cannot be highlighted correctly in one pass, and this is the reason: `Arena *a` is
	// a declaration and `a * b` is a multiplication, and they are the same three tokens.
	// Telling them apart needs to know that Arena is a type — which is the oldest problem
	// in parsing C, and the reason C compilers feed the symbol table back into the lexer.
	//
	// The alternative most highlighters take is to guess from the spacing, which is wrong
	// as often as the author's style differs from theirs. So this does not guess. It reads
	// the declarations first — `typedef struct Arena {…} Arena;` says Arena is a type, in
	// so many words — and then highlights knowing what the snippet said. A type the
	// snippet never declares stays an identifier, which is honest: nothing in the text the
	// reader is looking at claims otherwise.
	declared := cDeclaredTypes(src)

	// Two pieces of state, and both exist for the preprocessor.
	//
	// lineStart tracks whether we have seen anything but whitespace since the last
	// newline, because a `#` is only a directive at the start of a line — everywhere else
	// it is the stringize or paste operator inside a macro body, and painting THAT as a
	// directive turns the inside of every macro red.
	//
	// inInclude is set for the rest of the line after `#include`, and it is what makes
	// <stdio.h> a header path rather than a less-than, an identifier, a dot and a
	// greater-than. It is the one place in C where `<` does not mean what it usually does.
	lineStart := true
	inInclude := false

	// prevWord is the last identifier-shaped token seen. After `struct`, `union` or `enum`
	// the next identifier IS a type name — that one is syntax, not a guess, so it is worth
	// tracking one word of history to get it right.
	prevWord := ""

	i := 0
	for i < len(src) {
		c := src[i]

		switch {
		case c == '\n':
			b.WriteByte('\n')
			lineStart = true
			inInclude = false
			prevWord = ""
			i++
			continue

		case c == ' ' || c == '\t' || c == '\r':
			b.WriteByte(c)
			i++
			continue // whitespace does not end lineStart, and does not clear prevWord

		// Line comment. // is C99 and every C anyone writes today uses it.
		case c == '/' && i+1 < len(src) && src[i+1] == '/':
			end := strings.IndexByte(src[i:], '\n')
			if end < 0 {
				end = len(src)
			} else {
				end += i
			}
			span(&b, CommentClass, src[i:end])
			i = end

		// Block comment.
		case c == '/' && i+1 < len(src) && src[i+1] == '*':
			end := strings.Index(src[i+2:], "*/")
			if end < 0 {
				end = len(src)
			} else {
				end = i + 2 + end + 2
			}
			span(&b, CommentClass, src[i:end])
			i = end

		// A preprocessor directive: `#` and the word after it, and only at the start of a
		// line. `#define`, `#include`, `#ifdef`, `#pragma`.
		case c == '#' && lineStart:
			j := i + 1
			for j < len(src) && (src[j] == ' ' || src[j] == '\t') {
				j++ // `#  define` is legal, and rare, and free to support
			}
			for j < len(src) && isIdentPart(src[j]) {
				j++
			}
			word := src[i:j]
			span(&b, DirectiveClass, word)
			// The rest of an #include line reads differently. Nothing else does.
			if strings.HasSuffix(word, "include") || strings.HasSuffix(word, "import") {
				inInclude = true
			}
			i = j

		// <stdio.h> — but ONLY on an #include line. Anywhere else this is an operator.
		case c == '<' && inInclude:
			end := i + 1
			for end < len(src) && src[end] != '>' && src[end] != '\n' {
				end++
			}
			if end < len(src) && src[end] == '>' {
				end++ // include the closing bracket
			}
			span(&b, StringClass, src[i:end])
			i = end

		case c == '"':
			i = quoted(&b, src, i, '"', true)

		// A character literal. In C this is an INT, not a string — but every editor paints
		// it like a string, and a reader looking for '\0' is looking for a literal.
		case c == '\'':
			i = quoted(&b, src, i, '\'', true)

		case isDigit(c):
			j := cNumberEnd(src, i)
			span(&b, NumberClass, src[i:j])
			i = j

		case isIdentStart(c):
			j := i
			for j < len(src) && isIdentPart(src[j]) {
				j++
			}
			word := src[i:j]

			switch {
			case cKeywords[word]:
				span(&b, KeywordClass, word)
			case cModifiers[word]:
				span(&b, KeywordClass, word)
			case cTypes[word]:
				span(&b, TypeClass, word)
			case cValues[word]:
				span(&b, NumberClass, word) // NULL and true are constants; colour them as such
			case declared[word]:
				// The snippet said so itself, in a typedef or a struct tag. Not a guess.
				span(&b, TypeClass, word)
			case prevWord == "struct" || prevWord == "union" || prevWord == "enum":
				// Syntax, not a guess: what follows one of these IS a type name.
				span(&b, TypeClass, word)
			case isTypeSuffixed(word):
				span(&b, TypeClass, word)
			case callAhead(src, j):
				span(&b, FuncClass, word)
			default:
				b.WriteString(html.EscapeString(word))
			}

			prevWord = word
			lineStart = false
			i = j
			continue

		default:
			b.WriteString(html.EscapeString(string(c)))
			i++
		}

		lineStart = false
		prevWord = ""
	}
	return b.String()
}

// cDeclaredTypes reads a snippet's type declarations and returns the names they introduce.
//
// It is a scan, not a parse, and it recognises exactly two shapes — the two that declare a
// type name in C:
//
//	struct Arena / union Foo / enum Lang            the word after the tag is a type
//	typedef ... Name ;                              the last word before the semicolon
//	typedef ... (*Name)(...) ;                      unless it is a function pointer, whose
//	                                                name hides inside the parens
//
// The typedef rule is why brace depth is tracked. `typedef struct Arena { U8 *base; } Arena;`
// contains semicolons that do not end it, and taking the first one would declare a type
// called `base`.
//
// Comments and string literals are skipped, or the word `struct` inside a comment would
// declare the next word in the prose as a type.
func cDeclaredTypes(src string) map[string]bool {
	types := map[string]bool{}

	inTypedef := false // inside a typedef statement, up to its semicolon
	lastWord := ""     // last identifier seen in it — the name, for the common shape
	fnPtrName := ""    // ...unless it is a function pointer, whose name is in (*Name)
	prevWord := ""     // one word of history, for `struct Arena`
	depth := 0         // brace depth: only a depth-0 semicolon ends the typedef

	i := 0
	for i < len(src) {
		c := src[i]
		switch {
		case c == ' ' || c == '\t' || c == '\r' || c == '\n':
			i++ // whitespace must NOT clear prevWord: `struct Arena` has a space in it

		case c == '/' && i+1 < len(src) && src[i+1] == '/':
			if end := strings.IndexByte(src[i:], '\n'); end < 0 {
				i = len(src)
			} else {
				i += end
			}

		case c == '/' && i+1 < len(src) && src[i+1] == '*':
			if end := strings.Index(src[i+2:], "*/"); end < 0 {
				i = len(src)
			} else {
				i += 2 + end + 2
			}

		case c == '"' || c == '\'':
			i = skipQuoted(src, i, c)

		case c == '{':
			depth++
			i++

		case c == '}':
			depth--
			i++

		// (*Name)(...) — a function-pointer typedef. The name is here and nowhere else.
		case c == '(' && i+1 < len(src) && src[i+1] == '*':
			j := i + 2
			for j < len(src) && (src[j] == ' ' || src[j] == '\t') {
				j++
			}
			k := j
			for k < len(src) && isIdentPart(src[k]) {
				k++
			}
			if inTypedef && k > j {
				fnPtrName = src[j:k]
			}
			i = k
			prevWord = ""

		case c == ';' && depth == 0:
			if inTypedef {
				name := fnPtrName
				if name == "" {
					name = lastWord
				}
				if name != "" {
					types[name] = true
				}
			}
			inTypedef, lastWord, fnPtrName, prevWord = false, "", "", ""
			i++

		case isIdentStart(c):
			j := i
			for j < len(src) && isIdentPart(src[j]) {
				j++
			}
			word := src[i:j]

			if word == "typedef" {
				inTypedef, lastWord, fnPtrName = true, "", ""
			} else {
				if prevWord == "struct" || prevWord == "union" || prevWord == "enum" {
					types[word] = true
				}
				if inTypedef {
					lastWord = word
				}
			}
			prevWord = word
			i = j

		default:
			prevWord = ""
			i++
		}
	}
	return types
}

// skipQuoted returns the index one past a quoted literal, without emitting anything.
func skipQuoted(src string, i int, quote byte) int {
	j := i + 1
	for j < len(src) {
		if src[j] == '\\' && j+1 < len(src) {
			j += 2
			continue
		}
		if src[j] == quote {
			return j + 1
		}
		if src[j] == '\n' {
			return j // unterminated
		}
		j++
	}
	return j
}

// isTypeSuffixed reports whether a name ends in _t.
//
// This is a HEURISTIC, and the only one in this file. C has no way to know what is a type
// without a symbol table, and building one to colour a documentation snippet would be
// absurd — so the choice is between guessing and not guessing. `_t` is the one convention
// universal enough to guess on: it is what the standard library does (size_t, uint32_t),
// and a project that uses it for something that is not a type is doing so to be confusing.
//
// Everything else that is not in the tables above stays an identifier. A highlighter that
// paints too little is quietly unhelpful; one that paints too much is actively misleading.
func isTypeSuffixed(word string) bool {
	return len(word) > 2 && strings.HasSuffix(word, "_t")
}

// cNumberEnd returns the index one past the number starting at i.
//
// It is its own function because C numbers are a small zoo: 0x25, 0b1011, 1024, 3.14f,
// 1e9, 0xFFull. Go's number scanner (code.go) is a single character-class loop, which is
// enough for Go and would swallow the `x` of `0xFF` into a hex-ish blur and then choke on
// the `ull`.
func cNumberEnd(src string, i int) int {
	j := i

	switch {
	case src[j] == '0' && j+1 < len(src) && (src[j+1] == 'x' || src[j+1] == 'X'):
		j += 2
		for j < len(src) && isHexDigit(src[j]) {
			j++
		}
	case src[j] == '0' && j+1 < len(src) && (src[j+1] == 'b' || src[j+1] == 'B'):
		j += 2
		for j < len(src) && (src[j] == '0' || src[j] == '1') {
			j++
		}
	default:
		for j < len(src) && (isDigit(src[j]) || src[j] == '.') {
			j++
		}
		// An exponent, but only if it actually has digits after it — otherwise the `e` of
		// `1end` is a number and `nd` is an identifier, which is nonsense.
		if j < len(src) && (src[j] == 'e' || src[j] == 'E') {
			k := j + 1
			if k < len(src) && (src[k] == '+' || src[k] == '-') {
				k++
			}
			if k < len(src) && isDigit(src[k]) {
				for k < len(src) && isDigit(src[k]) {
					k++
				}
				j = k
			}
		}
	}

	// u, U, l, L, f, F, in any order and any number: 0xFFull, 1.0f, 10UL.
	for j < len(src) && isNumSuffix(src[j]) {
		j++
	}
	return j
}

func isHexDigit(c byte) bool {
	return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}

func isNumSuffix(c byte) bool {
	return c == 'u' || c == 'U' || c == 'l' || c == 'L' || c == 'f' || c == 'F'
}