kjol/go/lexer/lexer_c_test.go

package lexer

import (
	"strings"
	"testing"
)

// The escaping test is the one that matters most, and C is worse than Go for it.
//
// A C header opens with `#include <stdio.h>`. If that reaches the page unescaped, the
// browser sees an unknown tag, swallows it, and the line silently loses its second half.
// The same goes for `<<`, `->` and `&&`, which are on nearly every line of real C.
func TestHighlightCEscapes(t *testing.T) {
	got := HighlightC("#include <stdio.h>\nx = a << 2 & b->c;\n")

	for _, raw := range []string{"<stdio.h>", "<<", "&&", "->"} {
		// The only `<` in the output should be the ones opening our own spans.
		if strings.Contains(stripSpans(got), raw) {
			t.Errorf("%q survived unescaped into the HTML — the browser will eat it:\n%s", raw, got)
		}
	}
	for _, want := range []string{"&lt;stdio.h&gt;", "&lt;&lt;", "-&gt;"} {
		if !strings.Contains(got, want) {
			t.Errorf("expected %q in the output, got:\n%s", want, got)
		}
	}
}

func TestHighlightCClassifies(t *testing.T) {
	src := "typedef struct Arena { U8 *base; U64 pos; } Arena;\n" +
		"\n" +
		"internal U64 arena_push(Arena *a, U64 size) {\n" +
		"    // bump\n" +
		"    U64 pos = a->pos + 0x10;\n" +
		"    return pos;\n" +
		"}\n"
	got := HighlightC(src)

	cases := []struct{ class, text, why string }{
		{KeywordClass, "internal", "base_core's name for a file-static reads as a storage class"},
		{TypeClass, "U64", "kjøl's own base types are types, not bare identifiers"},
		{TypeClass, "Arena", "and so is a struct the snippet declared"},
		{FuncClass, "arena_push", "an identifier before ( is being declared or called"},
		{NumberClass, "0x10", "hex is a number, not a 0 followed by an identifier"},
		{CommentClass, "// bump", "a line comment"},
		{KeywordClass, "return", "a keyword"},
	}
	for _, c := range cases {
		want := `<span class="` + c.class + `">` + c.text + `</span>`
		if !strings.Contains(got, want) {
			t.Errorf("%s: expected %s\ngot:\n%s", c.why, want, got)
		}
	}
}

// The pre-pass. This is what lets `Arena *a` be a declaration rather than a guess — so the
// shapes it has to recognise are worth pinning down one at a time.
func TestCDeclaredTypes(t *testing.T) {
	cases := []struct {
		src, want, why string
	}{
		{"typedef struct Arena { U8 *base; U64 pos; } Arena;", "Arena",
			"the inner semicolons must not end the typedef — the first one would declare `base`"},
		{"typedef enum Lang { LANG_C, LANG_GO } Lang;", "Lang",
			"an enum typedef names its type the same way"},
		{"typedef struct Arena Arena;", "Arena",
			"a forward declaration still declares the name"},
		{"struct Tokenizer { S32 at; };", "Tokenizer",
			"a plain struct tag, with no typedef at all"},
		{"typedef void (*LexerTokenizeFn)(const char *data, S32 len);", "LexerTokenizeFn",
			"a function pointer hides its name in the parens — the last word is a parameter"},
		{"typedef U64 Handle;", "Handle",
			"the simple case"},
	}
	for _, c := range cases {
		if got := cDeclaredTypes(c.src); !got[c.want] {
			t.Errorf("%s\n  src:  %s\n  want: %q declared, got %v", c.why, c.src, c.want, keysOf(got))
		}
	}

	// The word `struct` inside a comment or a string declares nothing.
	for _, src := range []string{
		"// struct Ghost is not real\n",
		`const char *s = "struct Ghost";`,
		"/* struct Ghost */",
	} {
		if got := cDeclaredTypes(src); got["Ghost"] {
			t.Errorf("a type was declared from inside a comment or string: %s", src)
		}
	}
}

func keysOf(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	return out
}

// `#` is a directive only at the start of a line. Inside a macro body it is the stringize
// operator, and painting THAT as a directive turns the inside of every macro red.
func TestHighlightCDirectivesOnlyAtLineStart(t *testing.T) {
	got := HighlightC("#define STR(x) #x\n")

	if !strings.Contains(got, `<span class="`+DirectiveClass+`">#define</span>`) {
		t.Errorf("#define should be a directive:\n%s", got)
	}
	// The second # (the stringize operator) must NOT be a directive span.
	if strings.Count(got, `<span class="`+DirectiveClass+`">`) != 1 {
		t.Errorf("the stringize # was painted as a directive too:\n%s", got)
	}
}

// The one place in C where `<` is not an operator.
func TestHighlightCAngleBracketsOnlyOnIncludeLines(t *testing.T) {
	got := HighlightC("#include <stdio.h>\nif (a < b && c > d) return;\n")

	if !strings.Contains(got, `<span class="`+StringClass+`">&lt;stdio.h&gt;</span>`) {
		t.Errorf("the header path should be a string span:\n%s", got)
	}
	// On the NEXT line, `<` is a comparison. If inInclude leaked past the newline, the
	// rest of the file from `< b &&...` would be swallowed into one green string.
	if strings.Contains(got, `<span class="`+StringClass+`">&lt; b`) {
		t.Errorf("the include state leaked onto the following line:\n%s", got)
	}
}

func TestHighlightCNumbers(t *testing.T) {
	for _, n := range []string{"0x25", "0xFFull", "0b1011", "1024", "3.14f", "1e9", "10UL"} {
		got := HighlightC("x = " + n + ";")
		want := `<span class="` + NumberClass + `">` + n + `</span>`
		if !strings.Contains(got, want) {
			t.Errorf("%q was not lexed as one number:\n%s", n, got)
		}
	}
}

// A snippet in a documentation page is a fragment. It gets cut off mid-string and
// mid-comment all the time, and it must still render — a highlighter that can panic takes
// the whole page with it.
func TestHighlightCSurvivesMalformedInput(t *testing.T) {
	for _, src := range []string{
		`char *s = "unterminated`,
		"/* unterminated block",
		"'",
		"#",
		"#include <unterminated",
		"0x",
		"",
	} {
		func() {
			defer func() {
				if r := recover(); r != nil {
					t.Errorf("HighlightC panicked on %q: %v", src, r)
				}
			}()
			HighlightC(src)
		}()
	}
}

// The highlighter must not change the code. Strip the spans, unescape, and you should have
// exactly what you started with — byte for byte. A highlighter that drops a character is
// showing the reader a program that does not exist.
func TestHighlightCIsLossless(t *testing.T) {
	src := "#include <stdio.h>\n" +
		"#define KB(n) (((U64)(n)) << 10)\n" +
		"\n" +
		"typedef struct Arena { U8 *base; U64 pos; } Arena;\n" +
		"\n" +
		"static inline B32 str8_match(Str8 a, Str8 b) {\n" +
		"    if (a.size != b.size) return 0;   /* cheap out */\n" +
		"    return MemoryCompare(a.str, b.str, a.size) == 0;  // memcmp\n" +
		"}\n"

	if plain := stripTags(HighlightC(src)); plain != src {
		t.Errorf("the highlighter changed the source.\n got: %q\nwant: %q", plain, src)
	}
}

// stripSpans removes only our own span tags, leaving the escaped entities alone — so the
// escaping test can ask "is there a raw < left in here" without the spans' own angle
// brackets answering for it.
func stripSpans(s string) string {
	s = strings.ReplaceAll(s, `</span>`, "")
	for {
		i := strings.Index(s, `<span class="`)
		if i < 0 {
			return s
		}
		j := strings.Index(s[i:], `">`)
		if j < 0 {
			return s
		}
		s = s[:i] + s[i+j+2:]
	}
}