kjol/go/lexer/lexer.go

// Package lexer turns source code into syntax-highlighted HTML.
//
// It is a LEXER, not a parser: it classifies tokens and gives up gracefully on anything it
// does not understand. A highlighter that can fail to render is worse than one that
// occasionally paints an identifier the wrong colour — the reader can see past a wrong
// colour, and cannot see past a blank page. Unterminated strings and comments run to a
// sensible boundary rather than throwing, and a language nobody has written a lexer for
// comes back escaped and unpainted rather than not coming back.
//
// Output is HTML, and every run of source text passes through html.EscapeString on the way
// out. That is not a nicety: the input is source code, which is full of `<`, `>` and `&` —
// a C file is nothing but shifts and arrows, and one of the snippets kjøl's own docs
// display is literally a block of HTML. Unescaped, a snippet containing `<div>` renders a
// div, and half the line disappears into it.
//
// It lives beside webui rather than inside it because it has nothing to do with the DOM:
// it is a string in and a string out, it imports nothing but the standard library, and it
// is as usable from a static site generator or a terminal as from a component. webui just
// happened to be where it was first needed.
//
// The one thing tying it to a UI is the palette — the class names below are Tailwind's, so
// whatever compiles your CSS has to scan THIS package too, or the colours will not exist.
// (In kjøl's own site that is cmd/kjol-website/build.Tailwind's source globs.)
package lexer

import (
	"html"
	"strings"
)

// A code block is dark in BOTH themes — a light code block on a light page is a different
// kind of thing, and switching it with the theme means the snippet you were reading changes
// colour under you. So these are fixed on-dark colours, not theme tokens: the surface they
// sit on never changes.
//
// They are shared by every language in the package, so that a Go snippet and a C snippet on
// the same page agree about what a string looks like. A reader should not have to relearn
// the palette per section.
const (
	CommentClass = "text-ink-faint"
	StringClass  = "text-emerald-300"
	KeywordClass = "text-sky-300"
	NumberClass  = "text-amber-300"
	FuncClass    = "text-violet-300"

	// The two C needs and Go does not. C has a preprocessor, and C declarations are mostly
	// TYPE — so painting `U64` the same colour as `static` throws away the one distinction
	// that makes a header skimmable.
	TypeClass      = "text-teal-300"
	DirectiveClass = "text-rose-300"
)

// Highlight returns HTML for src, painted for the named language.
//
// The name is matched case-insensitively, and an UNKNOWN one is not an error: the source
// comes back escaped and unpainted. That is deliberate, and it is why callers can pass a
// caption's language label straight through. A docs page that showed nothing for a shell
// snippet because nobody has written a shell lexer would be trading a small loss of colour
// for a total loss of content — and painting it with the WRONG lexer (Jai through Go's
// keyword table) would be worse than either.
//
// The result is meant for a raw-HTML node inside a <pre>: it contains no block elements and
// preserves the source's whitespace exactly, so the <pre> does the layout.
func Highlight(lang, src string) string {
	switch strings.ToLower(lang) {
	case "go":
		return HighlightGo(src)
	case "c":
		return HighlightC(src)
	default:
		return html.EscapeString(src)
	}
}

// ---- the pieces every language in here shares ----------------------------

// quoted consumes a quoted literal starting at i and writes it as a string span.
// escapes reports whether a backslash escapes the next byte (false for Go's raw strings).
func quoted(b *strings.Builder, src string, i int, quote byte, escapes bool) int {
	j := i + 1
	for j < len(src) {
		if escapes && src[j] == '\\' && j+1 < len(src) {
			j += 2
			continue
		}
		if src[j] == quote {
			j++
			break
		}
		if escapes && src[j] == '\n' {
			break // unterminated: stop at the line end rather than eating the file
		}
		j++
	}
	span(b, StringClass, src[i:j])
	return j
}

// callAhead reports whether the next non-space byte at or after i is an opening paren —
// which is the whole of the "is this identifier a function" test. It is a guess, and a good
// one: an identifier immediately before a `(` is being called, declared, or converted to,
// and all three are worth seeing at a glance.
func callAhead(src string, i int) bool {
	for i < len(src) && (src[i] == ' ' || src[i] == '\t') {
		i++
	}
	return i < len(src) && src[i] == '('
}

func span(b *strings.Builder, class, text string) {
	b.WriteString(`<span class="`)
	b.WriteString(class)
	b.WriteString(`">`)
	b.WriteString(html.EscapeString(text))
	b.WriteString(`</span>`)
}

func isDigit(c byte) bool { return c >= '0' && c <= '9' }
func isHexish(c byte) bool {
	return c == '.' || c == 'x' || c == 'X' || c == '_' ||
		(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
func isIdentStart(c byte) bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') }
func isIdentPart(c byte) bool  { return isIdentStart(c) || isDigit(c) }