Update kjol website with C documentation

go/lexer/lexer_c_test.go

@@ -0,0 +1,198 @@
+package lexer
+
+import (
+	"strings"
+	"testing"
+)
+
+// The escaping test is the one that matters most, and C is worse than Go for it.
+//
+// A C header opens with `#include <stdio.h>`. If that reaches the page unescaped, the
+// browser sees an unknown tag, swallows it, and the line silently loses its second half.
+// The same goes for `<<`, `->` and `&&`, which are on nearly every line of real C.
+func TestHighlightCEscapes(t *testing.T) {
+	got := HighlightC("#include <stdio.h>\nx = a << 2 & b->c;\n")
+
+	for _, raw := range []string{"<stdio.h>", "<<", "&&", "->"} {
+		// The only `<` in the output should be the ones opening our own spans.
+		if strings.Contains(stripSpans(got), raw) {
+			t.Errorf("%q survived unescaped into the HTML — the browser will eat it:\n%s", raw, got)
+		}
+	}
+	for _, want := range []string{"&lt;stdio.h&gt;", "&lt;&lt;", "-&gt;"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("expected %q in the output, got:\n%s", want, got)
+		}
+	}
+}
+
+func TestHighlightCClassifies(t *testing.T) {
+	src := "typedef struct Arena { U8 *base; U64 pos; } Arena;\n" +
+		"\n" +
+		"internal U64 arena_push(Arena *a, U64 size) {\n" +
+		"    // bump\n" +
+		"    U64 pos = a->pos + 0x10;\n" +
+		"    return pos;\n" +
+		"}\n"
+	got := HighlightC(src)
+
+	cases := []struct{ class, text, why string }{
+		{KeywordClass, "internal", "base_core's name for a file-static reads as a storage class"},
+		{TypeClass, "U64", "kjøl's own base types are types, not bare identifiers"},
+		{TypeClass, "Arena", "and so is a struct the snippet declared"},
+		{FuncClass, "arena_push", "an identifier before ( is being declared or called"},
+		{NumberClass, "0x10", "hex is a number, not a 0 followed by an identifier"},
+		{CommentClass, "// bump", "a line comment"},
+		{KeywordClass, "return", "a keyword"},
+	}
+	for _, c := range cases {
+		want := `<span class="` + c.class + `">` + c.text + `</span>`
+		if !strings.Contains(got, want) {
+			t.Errorf("%s: expected %s\ngot:\n%s", c.why, want, got)
+		}
+	}
+}
+
+// The pre-pass. This is what lets `Arena *a` be a declaration rather than a guess — so the
+// shapes it has to recognise are worth pinning down one at a time.
+func TestCDeclaredTypes(t *testing.T) {
+	cases := []struct {
+		src, want, why string
+	}{
+		{"typedef struct Arena { U8 *base; U64 pos; } Arena;", "Arena",
+			"the inner semicolons must not end the typedef — the first one would declare `base`"},
+		{"typedef enum Lang { LANG_C, LANG_GO } Lang;", "Lang",
+			"an enum typedef names its type the same way"},
+		{"typedef struct Arena Arena;", "Arena",
+			"a forward declaration still declares the name"},
+		{"struct Tokenizer { S32 at; };", "Tokenizer",
+			"a plain struct tag, with no typedef at all"},
+		{"typedef void (*LexerTokenizeFn)(const char *data, S32 len);", "LexerTokenizeFn",
+			"a function pointer hides its name in the parens — the last word is a parameter"},
+		{"typedef U64 Handle;", "Handle",
+			"the simple case"},
+	}
+	for _, c := range cases {
+		if got := cDeclaredTypes(c.src); !got[c.want] {
+			t.Errorf("%s\n  src:  %s\n  want: %q declared, got %v", c.why, c.src, c.want, keysOf(got))
+		}
+	}
+
+	// The word `struct` inside a comment or a string declares nothing.
+	for _, src := range []string{
+		"// struct Ghost is not real\n",
+		`const char *s = "struct Ghost";`,
+		"/* struct Ghost */",
+	} {
+		if got := cDeclaredTypes(src); got["Ghost"] {
+			t.Errorf("a type was declared from inside a comment or string: %s", src)
+		}
+	}
+}
+
+func keysOf(m map[string]bool) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}
+
+// `#` is a directive only at the start of a line. Inside a macro body it is the stringize
+// operator, and painting THAT as a directive turns the inside of every macro red.
+func TestHighlightCDirectivesOnlyAtLineStart(t *testing.T) {
+	got := HighlightC("#define STR(x) #x\n")
+
+	if !strings.Contains(got, `<span class="`+DirectiveClass+`">#define</span>`) {
+		t.Errorf("#define should be a directive:\n%s", got)
+	}
+	// The second # (the stringize operator) must NOT be a directive span.
+	if strings.Count(got, `<span class="`+DirectiveClass+`">`) != 1 {
+		t.Errorf("the stringize # was painted as a directive too:\n%s", got)
+	}
+}
+
+// The one place in C where `<` is not an operator.
+func TestHighlightCAngleBracketsOnlyOnIncludeLines(t *testing.T) {
+	got := HighlightC("#include <stdio.h>\nif (a < b && c > d) return;\n")
+
+	if !strings.Contains(got, `<span class="`+StringClass+`">&lt;stdio.h&gt;</span>`) {
+		t.Errorf("the header path should be a string span:\n%s", got)
+	}
+	// On the NEXT line, `<` is a comparison. If inInclude leaked past the newline, the
+	// rest of the file from `< b &&...` would be swallowed into one green string.
+	if strings.Contains(got, `<span class="`+StringClass+`">&lt; b`) {
+		t.Errorf("the include state leaked onto the following line:\n%s", got)
+	}
+}
+
+func TestHighlightCNumbers(t *testing.T) {
+	for _, n := range []string{"0x25", "0xFFull", "0b1011", "1024", "3.14f", "1e9", "10UL"} {
+		got := HighlightC("x = " + n + ";")
+		want := `<span class="` + NumberClass + `">` + n + `</span>`
+		if !strings.Contains(got, want) {
+			t.Errorf("%q was not lexed as one number:\n%s", n, got)
+		}
+	}
+}
+
+// A snippet in a documentation page is a fragment. It gets cut off mid-string and
+// mid-comment all the time, and it must still render — a highlighter that can panic takes
+// the whole page with it.
+func TestHighlightCSurvivesMalformedInput(t *testing.T) {
+	for _, src := range []string{
+		`char *s = "unterminated`,
+		"/* unterminated block",
+		"'",
+		"#",
+		"#include <unterminated",
+		"0x",
+		"",
+	} {
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					t.Errorf("HighlightC panicked on %q: %v", src, r)
+				}
+			}()
+			HighlightC(src)
+		}()
+	}
+}
+
+// The highlighter must not change the code. Strip the spans, unescape, and you should have
+// exactly what you started with — byte for byte. A highlighter that drops a character is
+// showing the reader a program that does not exist.
+func TestHighlightCIsLossless(t *testing.T) {
+	src := "#include <stdio.h>\n" +
+		"#define KB(n) (((U64)(n)) << 10)\n" +
+		"\n" +
+		"typedef struct Arena { U8 *base; U64 pos; } Arena;\n" +
+		"\n" +
+		"static inline B32 str8_match(Str8 a, Str8 b) {\n" +
+		"    if (a.size != b.size) return 0;   /* cheap out */\n" +
+		"    return MemoryCompare(a.str, b.str, a.size) == 0;  // memcmp\n" +
+		"}\n"
+
+	if plain := stripTags(HighlightC(src)); plain != src {
+		t.Errorf("the highlighter changed the source.\n got: %q\nwant: %q", plain, src)
+	}
+}
+
+// stripSpans removes only our own span tags, leaving the escaped entities alone — so the
+// escaping test can ask "is there a raw < left in here" without the spans' own angle
+// brackets answering for it.
+func stripSpans(s string) string {
+	s = strings.ReplaceAll(s, `</span>`, "")
+	for {
+		i := strings.Index(s, `<span class="`)
+		if i < 0 {
+			return s
+		}
+		j := strings.Index(s[i:], `">`)
+		if j < 0 {
+			return s
+		}
+		s = s[:i] + s[i+j+2:]
+	}
+}