maxamundsen/kjol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update kjol website with C documentation

Browse Source
This commit is contained in:
Max Amundsen
2026-07-14 13:05:12 -04:00
parent 02a6dc6c48
commit 7d7b7354df
66 changed files with 23884 additions and 2551 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
go/lexer/lexer.go Normal file
View File

@@ -0,0 +1,124 @@
// Package lexer turns source code into syntax-highlighted HTML.
//
// It is a LEXER, not a parser: it classifies tokens and gives up gracefully on anything it
// does not understand. A highlighter that can fail to render is worse than one that
// occasionally paints an identifier the wrong colour — the reader can see past a wrong
// colour, and cannot see past a blank page. Unterminated strings and comments run to a
// sensible boundary rather than throwing, and a language nobody has written a lexer for
// comes back escaped and unpainted rather than not coming back.
//
// Output is HTML, and every run of source text passes through html.EscapeString on the way
// out. That is not a nicety: the input is source code, which is full of `<`, `>` and `&` —
// a C file is nothing but shifts and arrows, and one of the snippets kjøl's own docs
// display is literally a block of HTML. Unescaped, a snippet containing `<div>` renders a
// div, and half the line disappears into it.
//
// It lives beside webui rather than inside it because it has nothing to do with the DOM:
// it is a string in and a string out, it imports nothing but the standard library, and it
// is as usable from a static site generator or a terminal as from a component. webui just
// happened to be where it was first needed.
//
// The one thing tying it to a UI is the palette — the class names below are Tailwind's, so
// whatever compiles your CSS has to scan THIS package too, or the colours will not exist.
// (In kjøl's own site that is cmd/kjol-web/build.Tailwind's source globs.)
package lexer
import (
"html"
"strings"
)
// A code block is dark in BOTH themes — a light code block on a light page is a different
// kind of thing, and switching it with the theme means the snippet you were reading changes
// colour under you. So these are fixed on-dark colours, not theme tokens: the surface they
// sit on never changes.
//
// They are shared by every language in the package, so that a Go snippet and a C snippet on
// the same page agree about what a string looks like. A reader should not have to relearn
// the palette per section.
const (
CommentClass = "text-ink-faint"
StringClass = "text-emerald-300"
KeywordClass = "text-sky-300"
NumberClass = "text-amber-300"
FuncClass = "text-violet-300"
// The two C needs and Go does not. C has a preprocessor, and C declarations are mostly
// TYPE — so painting `U64` the same colour as `static` throws away the one distinction
// that makes a header skimmable.
TypeClass = "text-teal-300"
DirectiveClass = "text-rose-300"
)
// Highlight returns HTML for src, painted for the named language.
//
// The name is matched case-insensitively, and an UNKNOWN one is not an error: the source
// comes back escaped and unpainted. That is deliberate, and it is why callers can pass a
// caption's language label straight through. A docs page that showed nothing for a shell
// snippet because nobody has written a shell lexer would be trading a small loss of colour
// for a total loss of content — and painting it with the WRONG lexer (Jai through Go's
// keyword table) would be worse than either.
//
// The result is meant for a raw-HTML node inside a <pre>: it contains no block elements and
// preserves the source's whitespace exactly, so the <pre> does the layout.
func Highlight(lang, src string) string {
switch strings.ToLower(lang) {
case "go":
return HighlightGo(src)
case "c":
return HighlightC(src)
default:
return html.EscapeString(src)
}
}
// ---- the pieces every language in here shares ----------------------------
// quoted consumes a quoted literal starting at i and writes it as a string span.
// escapes reports whether a backslash escapes the next byte (false for Go's raw strings).
func quoted(b *strings.Builder, src string, i int, quote byte, escapes bool) int {
j := i + 1
for j < len(src) {
if escapes && src[j] == '\\' && j+1 < len(src) {
j += 2
continue
}
if src[j] == quote {
j++
break
}
if escapes && src[j] == '\n' {
break // unterminated: stop at the line end rather than eating the file
}
j++
}
span(b, StringClass, src[i:j])
return j
}
// callAhead reports whether the next non-space byte at or after i is an opening paren —
// which is the whole of the "is this identifier a function" test. It is a guess, and a good
// one: an identifier immediately before a `(` is being called, declared, or converted to,
// and all three are worth seeing at a glance.
func callAhead(src string, i int) bool {
for i < len(src) && (src[i] == ' ' || src[i] == '\t') {
i++
}
return i < len(src) && src[i] == '('
}
func span(b *strings.Builder, class, text string) {
b.WriteString(`<span class="`)
b.WriteString(class)
b.WriteString(`">`)
b.WriteString(html.EscapeString(text))
b.WriteString(`</span>`)
}
func isDigit(c byte) bool { return c >= '0' && c <= '9' }
func isHexish(c byte) bool {
return c == '.' || c == 'x' || c == 'X' || c == '_' ||
(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
func isIdentStart(c byte) bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') }
func isIdentPart(c byte) bool { return isIdentStart(c) || isDigit(c) }

433
go/lexer/lexer_c.go Normal file
View File

@@ -0,0 +1,433 @@
package lexer
import (
"html"
"strings"
)
// C syntax highlighting. Same contract as the rest of the package (see lexer.go).
//
// The classification tables below MIRROR the ones in kjøl's own C layer, c/lexer/lexer_c.c
// — a highlighter that colours the same source inside the editor it was written for. Two
// highlighters for one language is one too many, but they cannot share: that one is C
// compiled into an editor, this one is Go compiled to WebAssembly, and nothing is upstream
// of both. So they are two tables that agree, and this comment is the thing that says they
// have to. If you add a type there, add it here.
//
// That is also why U8/S32/F64/B32 are in the type table. They are not C types — they are
// kjøl's — but this highlighter exists to render kjøl's C, and a page about base_core.h
// that paints its own typedefs as bare identifiers has failed at the one job it has.
// The reserved words. Order of the maps below is also the order they are tested in, and
// the categories do not overlap.
var cKeywords = map[string]bool{
"auto": true, "break": true, "case": true, "continue": true, "default": true,
"do": true, "else": true, "enum": true, "for": true, "goto": true, "if": true,
"inline": true, "restrict": true, "return": true, "sizeof": true, "struct": true,
"switch": true, "typedef": true, "union": true, "while": true,
"_Alignas": true, "alignas": true, "_Alignof": true, "alignof": true,
"_Atomic": true, "_Generic": true, "_Noreturn": true,
"static_assert": true, "_Static_assert": true,
}
// Storage class and qualifiers. They read as keywords and are coloured as keywords — the
// C layer's lexer keeps them as a separate TOK_MODIFIER, but that distinction buys a
// reader of a documentation page nothing, so it is not carried over.
var cModifiers = map[string]bool{
"const": true, "constexpr": true, "extern": true, "register": true, "signed": true,
"static": true, "unsigned": true, "volatile": true,
"thread_local": true, "_Thread_local": true,
// Not C. base_core.h's three names for the three meanings of `static`, and the whole
// point of them is that they are visible — so they are coloured like what they are.
"internal": true, "global": true, "local_persist": true,
}
var cTypes = map[string]bool{
"char": true, "double": true, "float": true, "int": true, "long": true,
"short": true, "void": true, "bool": true, "_Bool": true,
"_Complex": true, "_Imaginary": true,
"int8_t": true, "int16_t": true, "int32_t": true, "int64_t": true,
"uint8_t": true, "uint16_t": true, "uint32_t": true, "uint64_t": true,
"size_t": true, "ssize_t": true, "ptrdiff_t": true,
"intptr_t": true, "uintptr_t": true, "nullptr_t": true, "FILE": true,
// kjøl's base layer (c/base/base_core.h).
"U8": true, "U16": true, "U32": true, "U64": true,
"S8": true, "S16": true, "S32": true, "S64": true,
"B32": true, "F32": true, "F64": true,
}
var cValues = map[string]bool{
"false": true, "true": true, "NULL": true, "nullptr": true,
}
// HighlightC turns C source into HTML with the tokens wrapped in coloured spans.
//
// The result is meant for vdom.Raw inside a <pre>: it contains no block elements and
// preserves the source's whitespace exactly, so the <pre> does the layout.
func HighlightC(src string) string {
var b strings.Builder
b.Grow(len(src) * 2)
// The types this snippet declares about itself.
//
// C cannot be highlighted correctly in one pass, and this is the reason: `Arena *a` is
// a declaration and `a * b` is a multiplication, and they are the same three tokens.
// Telling them apart needs to know that Arena is a type — which is the oldest problem
// in parsing C, and the reason C compilers feed the symbol table back into the lexer.
//
// The alternative most highlighters take is to guess from the spacing, which is wrong
// as often as the author's style differs from theirs. So this does not guess. It reads
// the declarations first — `typedef struct Arena {…} Arena;` says Arena is a type, in
// so many words — and then highlights knowing what the snippet said. A type the
// snippet never declares stays an identifier, which is honest: nothing in the text the
// reader is looking at claims otherwise.
declared := cDeclaredTypes(src)
// Two pieces of state, and both exist for the preprocessor.
//
// lineStart tracks whether we have seen anything but whitespace since the last
// newline, because a `#` is only a directive at the start of a line — everywhere else
// it is the stringize or paste operator inside a macro body, and painting THAT as a
// directive turns the inside of every macro red.
//
// inInclude is set for the rest of the line after `#include`, and it is what makes
// <stdio.h> a header path rather than a less-than, an identifier, a dot and a
// greater-than. It is the one place in C where `<` does not mean what it usually does.
lineStart := true
inInclude := false
// prevWord is the last identifier-shaped token seen. After `struct`, `union` or `enum`
// the next identifier IS a type name — that one is syntax, not a guess, so it is worth
// tracking one word of history to get it right.
prevWord := ""
i := 0
for i < len(src) {
c := src[i]
switch {
case c == '\n':
b.WriteByte('\n')
lineStart = true
inInclude = false
prevWord = ""
i++
continue
case c == ' ' || c == '\t' || c == '\r':
b.WriteByte(c)
i++
continue // whitespace does not end lineStart, and does not clear prevWord
// Line comment. // is C99 and every C anyone writes today uses it.
case c == '/' && i+1 < len(src) && src[i+1] == '/':
end := strings.IndexByte(src[i:], '\n')
if end < 0 {
end = len(src)
} else {
end += i
}
span(&b, CommentClass, src[i:end])
i = end
// Block comment.
case c == '/' && i+1 < len(src) && src[i+1] == '*':
end := strings.Index(src[i+2:], "*/")
if end < 0 {
end = len(src)
} else {
end = i + 2 + end + 2
}
span(&b, CommentClass, src[i:end])
i = end
// A preprocessor directive: `#` and the word after it, and only at the start of a
// line. `#define`, `#include`, `#ifdef`, `#pragma`.
case c == '#' && lineStart:
j := i + 1
for j < len(src) && (src[j] == ' ' || src[j] == '\t') {
j++ // `# define` is legal, and rare, and free to support
}
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
span(&b, DirectiveClass, word)
// The rest of an #include line reads differently. Nothing else does.
if strings.HasSuffix(word, "include") || strings.HasSuffix(word, "import") {
inInclude = true
}
i = j
// <stdio.h> — but ONLY on an #include line. Anywhere else this is an operator.
case c == '<' && inInclude:
end := i + 1
for end < len(src) && src[end] != '>' && src[end] != '\n' {
end++
}
if end < len(src) && src[end] == '>' {
end++ // include the closing bracket
}
span(&b, StringClass, src[i:end])
i = end
case c == '"':
i = quoted(&b, src, i, '"', true)
// A character literal. In C this is an INT, not a string — but every editor paints
// it like a string, and a reader looking for '\0' is looking for a literal.
case c == '\'':
i = quoted(&b, src, i, '\'', true)
case isDigit(c):
j := cNumberEnd(src, i)
span(&b, NumberClass, src[i:j])
i = j
case isIdentStart(c):
j := i
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
switch {
case cKeywords[word]:
span(&b, KeywordClass, word)
case cModifiers[word]:
span(&b, KeywordClass, word)
case cTypes[word]:
span(&b, TypeClass, word)
case cValues[word]:
span(&b, NumberClass, word) // NULL and true are constants; colour them as such
case declared[word]:
// The snippet said so itself, in a typedef or a struct tag. Not a guess.
span(&b, TypeClass, word)
case prevWord == "struct" || prevWord == "union" || prevWord == "enum":
// Syntax, not a guess: what follows one of these IS a type name.
span(&b, TypeClass, word)
case isTypeSuffixed(word):
span(&b, TypeClass, word)
case callAhead(src, j):
span(&b, FuncClass, word)
default:
b.WriteString(html.EscapeString(word))
}
prevWord = word
lineStart = false
i = j
continue
default:
b.WriteString(html.EscapeString(string(c)))
i++
}
lineStart = false
prevWord = ""
}
return b.String()
}
// cDeclaredTypes reads a snippet's type declarations and returns the names they introduce.
//
// It is a scan, not a parse, and it recognises exactly two shapes — the two that declare a
// type name in C:
//
// struct Arena / union Foo / enum Lang the word after the tag is a type
// typedef ... Name ; the last word before the semicolon
// typedef ... (*Name)(...) ; unless it is a function pointer, whose
// name hides inside the parens
//
// The typedef rule is why brace depth is tracked. `typedef struct Arena { U8 *base; } Arena;`
// contains semicolons that do not end it, and taking the first one would declare a type
// called `base`.
//
// Comments and string literals are skipped, or the word `struct` inside a comment would
// declare the next word in the prose as a type.
func cDeclaredTypes(src string) map[string]bool {
types := map[string]bool{}
inTypedef := false // inside a typedef statement, up to its semicolon
lastWord := "" // last identifier seen in it — the name, for the common shape
fnPtrName := "" // ...unless it is a function pointer, whose name is in (*Name)
prevWord := "" // one word of history, for `struct Arena`
depth := 0 // brace depth: only a depth-0 semicolon ends the typedef
i := 0
for i < len(src) {
c := src[i]
switch {
case c == ' ' || c == '\t' || c == '\r' || c == '\n':
i++ // whitespace must NOT clear prevWord: `struct Arena` has a space in it
case c == '/' && i+1 < len(src) && src[i+1] == '/':
if end := strings.IndexByte(src[i:], '\n'); end < 0 {
i = len(src)
} else {
i += end
}
case c == '/' && i+1 < len(src) && src[i+1] == '*':
if end := strings.Index(src[i+2:], "*/"); end < 0 {
i = len(src)
} else {
i += 2 + end + 2
}
case c == '"' || c == '\'':
i = skipQuoted(src, i, c)
case c == '{':
depth++
i++
case c == '}':
depth--
i++
// (*Name)(...) — a function-pointer typedef. The name is here and nowhere else.
case c == '(' && i+1 < len(src) && src[i+1] == '*':
j := i + 2
for j < len(src) && (src[j] == ' ' || src[j] == '\t') {
j++
}
k := j
for k < len(src) && isIdentPart(src[k]) {
k++
}
if inTypedef && k > j {
fnPtrName = src[j:k]
}
i = k
prevWord = ""
case c == ';' && depth == 0:
if inTypedef {
name := fnPtrName
if name == "" {
name = lastWord
}
if name != "" {
types[name] = true
}
}
inTypedef, lastWord, fnPtrName, prevWord = false, "", "", ""
i++
case isIdentStart(c):
j := i
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
if word == "typedef" {
inTypedef, lastWord, fnPtrName = true, "", ""
} else {
if prevWord == "struct" || prevWord == "union" || prevWord == "enum" {
types[word] = true
}
if inTypedef {
lastWord = word
}
}
prevWord = word
i = j
default:
prevWord = ""
i++
}
}
return types
}
// skipQuoted returns the index one past a quoted literal, without emitting anything.
func skipQuoted(src string, i int, quote byte) int {
j := i + 1
for j < len(src) {
if src[j] == '\\' && j+1 < len(src) {
j += 2
continue
}
if src[j] == quote {
return j + 1
}
if src[j] == '\n' {
return j // unterminated
}
j++
}
return j
}
// isTypeSuffixed reports whether a name ends in _t.
//
// This is a HEURISTIC, and the only one in this file. C has no way to know what is a type
// without a symbol table, and building one to colour a documentation snippet would be
// absurd — so the choice is between guessing and not guessing. `_t` is the one convention
// universal enough to guess on: it is what the standard library does (size_t, uint32_t),
// and a project that uses it for something that is not a type is doing so to be confusing.
//
// Everything else that is not in the tables above stays an identifier. A highlighter that
// paints too little is quietly unhelpful; one that paints too much is actively misleading.
func isTypeSuffixed(word string) bool {
return len(word) > 2 && strings.HasSuffix(word, "_t")
}
// cNumberEnd returns the index one past the number starting at i.
//
// It is its own function because C numbers are a small zoo: 0x25, 0b1011, 1024, 3.14f,
// 1e9, 0xFFull. Go's number scanner (code.go) is a single character-class loop, which is
// enough for Go and would swallow the `x` of `0xFF` into a hex-ish blur and then choke on
// the `ull`.
func cNumberEnd(src string, i int) int {
j := i
switch {
case src[j] == '0' && j+1 < len(src) && (src[j+1] == 'x' || src[j+1] == 'X'):
j += 2
for j < len(src) && isHexDigit(src[j]) {
j++
}
case src[j] == '0' && j+1 < len(src) && (src[j+1] == 'b' || src[j+1] == 'B'):
j += 2
for j < len(src) && (src[j] == '0' || src[j] == '1') {
j++
}
default:
for j < len(src) && (isDigit(src[j]) || src[j] == '.') {
j++
}
// An exponent, but only if it actually has digits after it — otherwise the `e` of
// `1end` is a number and `nd` is an identifier, which is nonsense.
if j < len(src) && (src[j] == 'e' || src[j] == 'E') {
k := j + 1
if k < len(src) && (src[k] == '+' || src[k] == '-') {
k++
}
if k < len(src) && isDigit(src[k]) {
for k < len(src) && isDigit(src[k]) {
k++
}
j = k
}
}
}
// u, U, l, L, f, F, in any order and any number: 0xFFull, 1.0f, 10UL.
for j < len(src) && isNumSuffix(src[j]) {
j++
}
return j
}
func isHexDigit(c byte) bool {
return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
func isNumSuffix(c byte) bool {
return c == 'u' || c == 'U' || c == 'l' || c == 'L' || c == 'f' || c == 'F'
}

198
go/lexer/lexer_c_test.go Normal file
View File

@@ -0,0 +1,198 @@
package lexer
import (
"strings"
"testing"
)
// The escaping test is the one that matters most, and C is worse than Go for it.
//
// A C header opens with `#include <stdio.h>`. If that reaches the page unescaped, the
// browser sees an unknown tag, swallows it, and the line silently loses its second half.
// The same goes for `<<`, `->` and `&&`, which are on nearly every line of real C.
func TestHighlightCEscapes(t *testing.T) {
got := HighlightC("#include <stdio.h>\nx = a << 2 & b->c;\n")
for _, raw := range []string{"<stdio.h>", "<<", "&&", "->"} {
// The only `<` in the output should be the ones opening our own spans.
if strings.Contains(stripSpans(got), raw) {
t.Errorf("%q survived unescaped into the HTML — the browser will eat it:\n%s", raw, got)
}
}
for _, want := range []string{"&lt;stdio.h&gt;", "&lt;&lt;", "-&gt;"} {
if !strings.Contains(got, want) {
t.Errorf("expected %q in the output, got:\n%s", want, got)
}
}
}
func TestHighlightCClassifies(t *testing.T) {
src := "typedef struct Arena { U8 *base; U64 pos; } Arena;\n" +
"\n" +
"internal U64 arena_push(Arena *a, U64 size) {\n" +
" // bump\n" +
" U64 pos = a->pos + 0x10;\n" +
" return pos;\n" +
"}\n"
got := HighlightC(src)
cases := []struct{ class, text, why string }{
{KeywordClass, "internal", "base_core's name for a file-static reads as a storage class"},
{TypeClass, "U64", "kjøl's own base types are types, not bare identifiers"},
{TypeClass, "Arena", "and so is a struct the snippet declared"},
{FuncClass, "arena_push", "an identifier before ( is being declared or called"},
{NumberClass, "0x10", "hex is a number, not a 0 followed by an identifier"},
{CommentClass, "// bump", "a line comment"},
{KeywordClass, "return", "a keyword"},
}
for _, c := range cases {
want := `<span class="` + c.class + `">` + c.text + `</span>`
if !strings.Contains(got, want) {
t.Errorf("%s: expected %s\ngot:\n%s", c.why, want, got)
}
}
}
// The pre-pass. This is what lets `Arena *a` be a declaration rather than a guess — so the
// shapes it has to recognise are worth pinning down one at a time.
func TestCDeclaredTypes(t *testing.T) {
cases := []struct {
src, want, why string
}{
{"typedef struct Arena { U8 *base; U64 pos; } Arena;", "Arena",
"the inner semicolons must not end the typedef — the first one would declare `base`"},
{"typedef enum Lang { LANG_C, LANG_GO } Lang;", "Lang",
"an enum typedef names its type the same way"},
{"typedef struct Arena Arena;", "Arena",
"a forward declaration still declares the name"},
{"struct Tokenizer { S32 at; };", "Tokenizer",
"a plain struct tag, with no typedef at all"},
{"typedef void (*LexerTokenizeFn)(const char *data, S32 len);", "LexerTokenizeFn",
"a function pointer hides its name in the parens — the last word is a parameter"},
{"typedef U64 Handle;", "Handle",
"the simple case"},
}
for _, c := range cases {
if got := cDeclaredTypes(c.src); !got[c.want] {
t.Errorf("%s\n src: %s\n want: %q declared, got %v", c.why, c.src, c.want, keysOf(got))
}
}
// The word `struct` inside a comment or a string declares nothing.
for _, src := range []string{
"// struct Ghost is not real\n",
`const char *s = "struct Ghost";`,
"/* struct Ghost */",
} {
if got := cDeclaredTypes(src); got["Ghost"] {
t.Errorf("a type was declared from inside a comment or string: %s", src)
}
}
}
func keysOf(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
// `#` is a directive only at the start of a line. Inside a macro body it is the stringize
// operator, and painting THAT as a directive turns the inside of every macro red.
func TestHighlightCDirectivesOnlyAtLineStart(t *testing.T) {
got := HighlightC("#define STR(x) #x\n")
if !strings.Contains(got, `<span class="`+DirectiveClass+`">#define</span>`) {
t.Errorf("#define should be a directive:\n%s", got)
}
// The second # (the stringize operator) must NOT be a directive span.
if strings.Count(got, `<span class="`+DirectiveClass+`">`) != 1 {
t.Errorf("the stringize # was painted as a directive too:\n%s", got)
}
}
// The one place in C where `<` is not an operator.
func TestHighlightCAngleBracketsOnlyOnIncludeLines(t *testing.T) {
got := HighlightC("#include <stdio.h>\nif (a < b && c > d) return;\n")
if !strings.Contains(got, `<span class="`+StringClass+`">&lt;stdio.h&gt;</span>`) {
t.Errorf("the header path should be a string span:\n%s", got)
}
// On the NEXT line, `<` is a comparison. If inInclude leaked past the newline, the
// rest of the file from `< b &&...` would be swallowed into one green string.
if strings.Contains(got, `<span class="`+StringClass+`">&lt; b`) {
t.Errorf("the include state leaked onto the following line:\n%s", got)
}
}
func TestHighlightCNumbers(t *testing.T) {
for _, n := range []string{"0x25", "0xFFull", "0b1011", "1024", "3.14f", "1e9", "10UL"} {
got := HighlightC("x = " + n + ";")
want := `<span class="` + NumberClass + `">` + n + `</span>`
if !strings.Contains(got, want) {
t.Errorf("%q was not lexed as one number:\n%s", n, got)
}
}
}
// A snippet in a documentation page is a fragment. It gets cut off mid-string and
// mid-comment all the time, and it must still render — a highlighter that can panic takes
// the whole page with it.
func TestHighlightCSurvivesMalformedInput(t *testing.T) {
for _, src := range []string{
`char *s = "unterminated`,
"/* unterminated block",
"'",
"#",
"#include <unterminated",
"0x",
"",
} {
func() {
defer func() {
if r := recover(); r != nil {
t.Errorf("HighlightC panicked on %q: %v", src, r)
}
}()
HighlightC(src)
}()
}
}
// The highlighter must not change the code. Strip the spans, unescape, and you should have
// exactly what you started with — byte for byte. A highlighter that drops a character is
// showing the reader a program that does not exist.
func TestHighlightCIsLossless(t *testing.T) {
src := "#include <stdio.h>\n" +
"#define KB(n) (((U64)(n)) << 10)\n" +
"\n" +
"typedef struct Arena { U8 *base; U64 pos; } Arena;\n" +
"\n" +
"static inline B32 str8_match(Str8 a, Str8 b) {\n" +
" if (a.size != b.size) return 0; /* cheap out */\n" +
" return MemoryCompare(a.str, b.str, a.size) == 0; // memcmp\n" +
"}\n"
if plain := stripTags(HighlightC(src)); plain != src {
t.Errorf("the highlighter changed the source.\n got: %q\nwant: %q", plain, src)
}
}
// stripSpans removes only our own span tags, leaving the escaped entities alone — so the
// escaping test can ask "is there a raw < left in here" without the spans' own angle
// brackets answering for it.
func stripSpans(s string) string {
s = strings.ReplaceAll(s, `</span>`, "")
for {
i := strings.Index(s, `<span class="`)
if i < 0 {
return s
}
j := strings.Index(s[i:], `">`)
if j < 0 {
return s
}
s = s[:i] + s[i+j+2:]
}
}

99
go/lexer/lexer_go.go Normal file
View File

@@ -0,0 +1,99 @@
package lexer
import (
"html"
"strings"
)
var goKeywords = map[string]bool{
"break": true, "case": true, "chan": true, "const": true, "continue": true,
"default": true, "defer": true, "else": true, "fallthrough": true, "for": true,
"func": true, "go": true, "goto": true, "if": true, "import": true,
"interface": true, "map": true, "package": true, "range": true, "return": true,
"select": true, "struct": true, "switch": true, "type": true, "var": true,
// Not keywords to the Go spec — predeclared identifiers — but every editor colours
// them, and a reader looking for `nil` is looking for the same kind of thing.
"nil": true, "true": true, "false": true, "iota": true,
"string": true, "int": true, "int64": true, "float64": true, "bool": true,
"byte": true, "rune": true, "any": true, "error": true,
}
// HighlightGo turns Go source into HTML with the tokens wrapped in coloured spans.
func HighlightGo(src string) string {
var b strings.Builder
b.Grow(len(src) * 2)
i := 0
for i < len(src) {
c := src[i]
switch {
// Line comment — including the //gowasm: directives, which are the most important
// line in several of the snippets this was written for.
case c == '/' && i+1 < len(src) && src[i+1] == '/':
end := strings.IndexByte(src[i:], '\n')
if end < 0 {
end = len(src)
} else {
end += i
}
span(&b, CommentClass, src[i:end])
i = end
// Block comment.
case c == '/' && i+1 < len(src) && src[i+1] == '*':
end := strings.Index(src[i+2:], "*/")
if end < 0 {
end = len(src)
} else {
end = i + 2 + end + 2
}
span(&b, CommentClass, src[i:end])
i = end
// Interpreted string. Ends at the closing quote or the line's end — an unterminated
// string is a typo in a snippet, not a reason to paint the rest of the file green.
case c == '"':
i = quoted(&b, src, i, '"', true)
// Raw string: no escapes, and it may span lines.
case c == '`':
i = quoted(&b, src, i, '`', false)
// Rune literal.
case c == '\'':
i = quoted(&b, src, i, '\'', true)
case isDigit(c):
j := i
for j < len(src) && (isDigit(src[j]) || isHexish(src[j])) {
j++
}
span(&b, NumberClass, src[i:j])
i = j
case isIdentStart(c):
j := i
for j < len(src) && isIdentPart(src[j]) {
j++
}
word := src[i:j]
switch {
case goKeywords[word]:
span(&b, KeywordClass, word)
case callAhead(src, j):
// Colouring the callee is what makes the shape of a snippet readable at a
// glance: you see what it DOES before you read what it says.
span(&b, FuncClass, word)
default:
b.WriteString(html.EscapeString(word))
}
i = j
default:
b.WriteString(html.EscapeString(string(c)))
i++
}
}
return b.String()
}

99
go/lexer/lexer_go_test.go Normal file
View File

@@ -0,0 +1,99 @@
package lexer
import (
"strings"
"testing"
)
// The highlighter emits HTML, and its input is Go source — which is full of <, > and &.
// Anything that reaches the page unescaped is markup injection into your own docs page:
// a snippet containing `<div>` would render a div.
func TestHighlightGoEscapes(t *testing.T) {
got := HighlightGo(`s := "<div class=\"x\">" // a & b`)
if strings.Contains(got, "<div") {
t.Errorf("a `<div>` in the SOURCE reached the output as markup:\n%s", got)
}
if !strings.Contains(got, "&lt;div") {
t.Errorf("the angle bracket was not escaped:\n%s", got)
}
if !strings.Contains(got, "&amp;") {
t.Errorf("the ampersand was not escaped:\n%s", got)
}
}
func TestHighlightGoClassifies(t *testing.T) {
got := HighlightGo("func main() { x := 42 // note\n}")
for _, want := range []struct{ what, class, text string }{
{"keyword", KeywordClass, "func"},
{"call", FuncClass, "main"},
{"number", NumberClass, "42"},
{"comment", CommentClass, "// note"},
} {
if !strings.Contains(got, `<span class="`+want.class+`">`+want.text+`</span>`) {
t.Errorf("%s %q was not highlighted:\n%s", want.what, want.text, got)
}
}
}
// The //gowasm: directives are the most important line in half these snippets. They are
// comments, and must survive as such.
func TestHighlightGoKeepsDirectives(t *testing.T) {
got := HighlightGo("//gowasm:page / static layout=public\nfunc HomePage() {}")
if !strings.Contains(got, `<span class="`+CommentClass+`">//gowasm:page / static layout=public</span>`) {
t.Errorf("the directive was not kept whole as a comment:\n%s", got)
}
}
// A lexer that can hang or eat the rest of the file on malformed input would take the
// whole page down with it. Unterminated literals stop; they do not run away.
func TestHighlightGoSurvivesMalformedInput(t *testing.T) {
for _, src := range []string{
`x := "unterminated`,
"y := `unterminated raw",
"/* unterminated block",
`z := '`,
"",
} {
got := HighlightGo(src)
// The text must all still be there — mangling is not an acceptable failure mode
// either. Compare on the visible characters, ignoring the spans.
if plain := stripTags(got); plain != src {
t.Errorf("input %q came out as %q", src, plain)
}
}
}
// Nothing is dropped: every byte of the source is still on the page, in order.
func TestHighlightGoIsLossless(t *testing.T) {
src := "package app\n\nimport \"strings\"\n\nfunc f(n int) string {\n\treturn strings.Repeat(\"x\", n) // pad\n}\n"
if plain := stripTags(HighlightGo(src)); plain != src {
t.Errorf("the highlighter changed the source.\n got: %q\nwant: %q", plain, src)
}
}
// stripTags removes the spans and unescapes, recovering the original source.
func stripTags(s string) string {
var b strings.Builder
for i := 0; i < len(s); {
if s[i] == '<' {
j := strings.IndexByte(s[i:], '>')
if j < 0 {
break
}
i += j + 1
continue
}
b.WriteByte(s[i])
i++
}
out := b.String()
// Reverse html.EscapeString, innermost last.
out = strings.ReplaceAll(out, "&lt;", "<")
out = strings.ReplaceAll(out, "&gt;", ">")
out = strings.ReplaceAll(out, "&#34;", `"`)
out = strings.ReplaceAll(out, "&#39;", "'")
out = strings.ReplaceAll(out, "&amp;", "&")
return out
}