vendor tsgo

tools/tsgo/internal/stringutil/_scripts/generate-unicode-data.mts

@@ -0,0 +1,247 @@
+#!/usr/bin/env -S node --experimental-strip-types --no-warnings
+
+import * as fs from "fs";
+import * as path from "path";
+
+// All Unicode data is sourced from a single version-pinned @unicode/unicode-*
+// package so the generated tables are reproducible regardless of the Node.js
+// runtime used to run `go generate`. Bumping this constant (and the matching
+// devDependency in package.json) is the only step needed to move to a newer
+// Unicode version.
+const UNICODE_VERSION = "15.1.0";
+const PACKAGE = `@unicode/unicode-${UNICODE_VERSION}`;
+
+const scriptDir = import.meta.dirname;
+const CASE_OUTPUT_PATH = path.join(scriptDir, "..", "js_case_generated.go");
+const IDENTIFIER_OUTPUT_PATH = path.join(scriptDir, "..", "identifier_parts_generated.go");
+
+// A *unicode.RangeTable is split into 16-bit (BMP) and 32-bit (astral) ranges,
+// each carrying a stride so arithmetic sequences (e.g. the alternating
+// upper/lower letters that fill the casing tables) collapse to a single entry.
+type RangeTable = {
+    r16: Range[];
+    r32: Range[];
+    latinOffset: number;
+};
+
+type Range = {
+    lo: number;
+    hi: number;
+    stride: number;
+};
+
+type SpecialCasingEntry = {
+    codePoint: number;
+    lower: number[];
+    upper: number[];
+    conditionalLower: number[];
+    condition: string;
+};
+
+async function loadCodePoints(property: string): Promise<number[]> {
+    const module = await import(`${PACKAGE}/${property}/code-points.js`);
+    return module.default as number[];
+}
+
+async function loadMapping(property: string): Promise<Map<number, number[]>> {
+    const module = await import(`${PACKAGE}/${property}/code-points.js`);
+    return module.default as Map<number, number[]>;
+}
+
+async function loadSimpleMapping(property: string): Promise<Map<number, number>> {
+    const module = await import(`${PACKAGE}/${property}/code-points.js`);
+    return module.default as Map<number, number>;
+}
+
+// Group a sorted, de-duplicated run of code points into ranges sharing a
+// constant stride. The stride of each range is taken from the gap to the next
+// code point, so it never includes a code point that is not in the set; this
+// matches the invariants unicode.Is relies on.
+function toStrideRanges(sorted: number[]): Range[] {
+    const ranges: Range[] = [];
+    let i = 0;
+    while (i < sorted.length) {
+        const lo = sorted[i];
+        const stride = i + 1 < sorted.length ? sorted[i + 1] - sorted[i] : 1;
+        let hi = lo;
+        let j = i + 1;
+        while (j < sorted.length && sorted[j] === hi + stride) {
+            hi = sorted[j];
+            j++;
+        }
+        ranges.push({ lo, hi, stride });
+        i = j;
+    }
+    return ranges;
+}
+
+// Compress a set of code points into a *unicode.RangeTable. BMP and astral code
+// points are separated first so no range straddles the U+FFFF boundary between
+// the R16 and R32 slices.
+function toRangeTable(codePoints: Iterable<number>): RangeTable {
+    const sorted = [...new Set(codePoints)].sort((a, b) => a - b);
+    const bmp = sorted.filter(cp => cp <= 0xFFFF);
+    const astral = sorted.filter(cp => cp > 0xFFFF);
+    const r16 = toStrideRanges(bmp);
+    const r32 = toStrideRanges(astral);
+    // unicode.Is fast-paths Latin-1 by linearly scanning the leading R16 entries
+    // whose Hi is within Latin-1 (U+00FF); LatinOffset records how many those are.
+    let latinOffset = 0;
+    while (latinOffset < r16.length && r16[latinOffset].hi <= 0xFF) {
+        latinOffset++;
+    }
+    return { r16, r32, latinOffset };
+}
+
+function goRuneLiteral(codePoint: number): string {
+    return `0x${codePoint.toString(16).toUpperCase()}`;
+}
+
+function goStringLiteral(codePoints: number[]): string {
+    let text = '"';
+    for (const codePoint of codePoints) {
+        if (codePoint <= 0xFFFF) {
+            text += `\\u${codePoint.toString(16).toUpperCase().padStart(4, "0")}`;
+        }
+        else {
+            text += `\\U${codePoint.toString(16).toUpperCase().padStart(8, "0")}`;
+        }
+    }
+    text += '"';
+    return text;
+}
+
+function renderRangeTable(name: string, table: RangeTable): string {
+    const r16 = table.r16.map(r => `\t\t{${goRuneLiteral(r.lo)}, ${goRuneLiteral(r.hi)}, ${r.stride}},`).join("\n");
+    const r32 = table.r32.map(r => `\t\t{${goRuneLiteral(r.lo)}, ${goRuneLiteral(r.hi)}, ${r.stride}},`).join("\n");
+    return `var ${name} = &unicode.RangeTable{
+\tR16: []unicode.Range16{
+${r16}
+\t},
+\tR32: []unicode.Range32{
+${r32}
+\t},
+\tLatinOffset: ${table.latinOffset},
+}
+`;
+}
+
+async function buildSpecialCasing(simpleLowercase: Map<number, number>, simpleUppercase: Map<number, number>): Promise<SpecialCasingEntry[]> {
+    // The unconditional, locale-insensitive multi-rune mappings. Each map keys a
+    // code point to its full lower/upper expansion (identity when unchanged).
+    const lowerMappings = await loadMapping("Special_Casing/Lowercase");
+    const upperMappings = await loadMapping("Special_Casing/Uppercase");
+    // toLowerCase emits the word-final sigma only in Final_Sigma context, so this
+    // mapping is tracked separately and applied by the Go caser when in context.
+    const finalSigmaMappings = await loadMapping("Special_Casing/Lowercase--Final_Sigma");
+
+    const entries: SpecialCasingEntry[] = [];
+
+    const codePoints = new Set([...simpleLowercase.keys(), ...simpleUppercase.keys(), ...lowerMappings.keys(), ...upperMappings.keys()]);
+    for (const codePoint of codePoints) {
+        entries.push({
+            codePoint,
+            lower: lowerMappings.get(codePoint) ?? [simpleLowercase.get(codePoint) ?? codePoint],
+            upper: upperMappings.get(codePoint) ?? [simpleUppercase.get(codePoint) ?? codePoint],
+            conditionalLower: [codePoint],
+            condition: "specialCasingConditionNone",
+        });
+    }
+
+    for (const [codePoint, lower] of finalSigmaMappings) {
+        const entry = entries.find(entry => entry.codePoint === codePoint);
+        if (entry === undefined) {
+            entries.push({
+                codePoint,
+                lower: [simpleLowercase.get(codePoint) ?? codePoint],
+                upper: upperMappings.get(codePoint) ?? [simpleUppercase.get(codePoint) ?? codePoint],
+                conditionalLower: lower,
+                condition: "specialCasingConditionFinalSigma",
+            });
+        }
+        else {
+            entry.conditionalLower = lower;
+            entry.condition = "specialCasingConditionFinalSigma";
+        }
+    }
+
+    entries.sort((a, b) => a.codePoint - b.codePoint);
+    return entries;
+}
+
+function renderCaseFile(entries: SpecialCasingEntry[], casedTable: RangeTable, caseIgnorableTable: RangeTable): string {
+    const mappings = entries.map(entry => {
+        const conditionalLower = entry.condition === "specialCasingConditionFinalSigma" ? `, conditionalLower: ${goStringLiteral(entry.conditionalLower)}` : "";
+        return `\t${goRuneLiteral(entry.codePoint)}: {lower: ${goStringLiteral(entry.lower)}, upper: ${goStringLiteral(entry.upper)}${conditionalLower}, condition: ${entry.condition}},`;
+    }).join("\n");
+
+    return `// Code generated by generate-unicode-data.mts. DO NOT EDIT.
+// Derived from the ${PACKAGE} package (Unicode ${UNICODE_VERSION}).
+// Includes only the locale-insensitive multi-rune mappings needed for ECMAScript
+// default casing, plus the Final_Sigma context mapping. String.prototype.toLowerCase
+// applies Final_Sigma, but Go's unicode package does not, so the caser applies it
+// from this data when in context. Simple one-rune mappings are included here too
+// so casing stays pinned to this Unicode version, rather than the Go toolchain's
+// unicode tables.
+
+package stringutil
+
+import "unicode"
+
+type specialCasingCondition uint8
+
+const (
+\tspecialCasingConditionNone specialCasingCondition = iota
+\tspecialCasingConditionFinalSigma
+)
+
+type specialCasingMapping struct {
+\tlower            string
+\tupper            string
+\tconditionalLower string
+\tcondition        specialCasingCondition
+}
+
+var specialCasingMappings = map[rune]specialCasingMapping{
+${mappings}
+}
+
+${renderRangeTable("unicodeCasedRanges", casedTable)}
+${renderRangeTable("unicodeCaseIgnorableRanges", caseIgnorableTable)}
+`;
+}
+
+function renderIdentifierFile(startTable: RangeTable, partTable: RangeTable): string {
+    return `// Code generated by generate-unicode-data.mts. DO NOT EDIT.
+// Derived from the ${PACKAGE} package (Unicode ${UNICODE_VERSION}).
+// Based on http://www.unicode.org/reports/tr31/ and
+// https://www.ecma-international.org/ecma-262/6.0/#sec-names-and-keywords:
+// unicodeESNextIdentifierStart corresponds to the ID_Start and Other_ID_Start property, and
+// unicodeESNextIdentifierPart corresponds to ID_Continue, Other_ID_Continue, plus ID_Start and Other_ID_Start.
+
+package stringutil
+
+import "unicode"
+
+${renderRangeTable("unicodeESNextIdentifierStart", startTable)}
+${renderRangeTable("unicodeESNextIdentifierPart", partTable)}
+`;
+}
+
+async function main() {
+    const simpleLowercase = await loadSimpleMapping("Simple_Case_Mapping/Lowercase");
+    const simpleUppercase = await loadSimpleMapping("Simple_Case_Mapping/Uppercase");
+    const entries = await buildSpecialCasing(simpleLowercase, simpleUppercase);
+    const casedTable = toRangeTable(await loadCodePoints("Binary_Property/Cased"));
+    const caseIgnorableTable = toRangeTable(await loadCodePoints("Binary_Property/Case_Ignorable"));
+    fs.writeFileSync(CASE_OUTPUT_PATH, renderCaseFile(entries, casedTable, caseIgnorableTable));
+
+    const idStart = await loadCodePoints("Binary_Property/ID_Start");
+    const idContinue = await loadCodePoints("Binary_Property/ID_Continue");
+    // Other_ID_Start/Other_ID_Continue are already folded into ID_Start/ID_Continue.
+    const startTable = toRangeTable(idStart);
+    const partTable = toRangeTable([...idContinue, ...idStart]);
+    fs.writeFileSync(IDENTIFIER_OUTPUT_PATH, renderIdentifierFile(startTable, partTable));
+}
+
+await main();

tools/tsgo/internal/stringutil/compare.go

@@ -0,0 +1,128 @@
+package stringutil
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+func EquateStringCaseInsensitive(a, b string) bool {
+	// !!!
+	// return a == b || strings.ToUpper(a) == strings.ToUpper(b)
+	return strings.EqualFold(a, b)
+}
+
+func EquateStringCaseSensitive(a, b string) bool {
+	return a == b
+}
+
+func GetStringEqualityComparer(ignoreCase bool) func(a, b string) bool {
+	if ignoreCase {
+		return EquateStringCaseInsensitive
+	}
+	return EquateStringCaseSensitive
+}
+
+type Comparison = int
+
+const (
+	ComparisonLessThan    Comparison = -1
+	ComparisonEqual       Comparison = 0
+	ComparisonGreaterThan Comparison = 1
+)
+
+func CompareStringsCaseInsensitive(a string, b string) Comparison {
+	if a == b {
+		return ComparisonEqual
+	}
+	for {
+		ca, sa := utf8.DecodeRuneInString(a)
+		cb, sb := utf8.DecodeRuneInString(b)
+		if sa == 0 {
+			if sb == 0 {
+				return ComparisonEqual
+			}
+			return ComparisonLessThan
+		}
+		if sb == 0 {
+			return ComparisonGreaterThan
+		}
+		lca := unicode.ToLower(ca)
+		lcb := unicode.ToLower(cb)
+		if lca != lcb {
+			if lca < lcb {
+				return ComparisonLessThan
+			}
+			return ComparisonGreaterThan
+		}
+		a = a[sa:]
+		b = b[sb:]
+	}
+}
+
+func CompareStringsCaseSensitive(a string, b string) Comparison {
+	return strings.Compare(a, b)
+}
+
+func GetStringComparer(ignoreCase bool) func(a, b string) Comparison {
+	if ignoreCase {
+		return CompareStringsCaseInsensitive
+	}
+	return CompareStringsCaseSensitive
+}
+
+func HasPrefix(s string, prefix string, caseSensitive bool) bool {
+	if caseSensitive {
+		return strings.HasPrefix(s, prefix)
+	}
+	if len(prefix) > len(s) {
+		return false
+	}
+	return strings.EqualFold(s[0:len(prefix)], prefix)
+}
+
+func HasSuffix(s string, suffix string, caseSensitive bool) bool {
+	if caseSensitive {
+		return strings.HasSuffix(s, suffix)
+	}
+	if len(suffix) > len(s) {
+		return false
+	}
+	return strings.EqualFold(s[len(s)-len(suffix):], suffix)
+}
+
+func HasPrefixAndSuffixWithoutOverlap(s string, prefix string, suffix string, caseSensitive bool) bool {
+	if len(prefix)+len(suffix) > len(s) {
+		return false
+	}
+
+	return HasPrefix(s, prefix, caseSensitive) && HasSuffix(s, suffix, caseSensitive)
+}
+
+func CompareStringsCaseInsensitiveThenSensitive(a, b string) Comparison {
+	cmp := CompareStringsCaseInsensitive(a, b)
+	if cmp != ComparisonEqual {
+		return cmp
+	}
+	return CompareStringsCaseSensitive(a, b)
+}
+
+// CompareStringsCaseInsensitiveEslintCompatible performs a case-insensitive comparison
+// using toLowerCase() instead of toUpperCase() for ESLint compatibility.
+//
+// `CompareStringsCaseInsensitive` transforms letters to uppercase for unicode reasons,
+// while eslint's `sort-imports` rule transforms letters to lowercase. Which one you choose
+// affects the relative order of letters and ASCII characters 91-96, of which `_` is a
+// valid character in an identifier. So if we used `CompareStringsCaseInsensitive` for
+// import sorting, TypeScript and eslint would disagree about the correct case-insensitive
+// sort order for `__String` and `Foo`. Since eslint's whole job is to create consistency
+// by enforcing nitpicky details like this, it makes way more sense for us to just adopt
+// their convention so users can have auto-imports without making eslint angry.
+func CompareStringsCaseInsensitiveEslintCompatible(a, b string) Comparison {
+	if a == b {
+		return ComparisonEqual
+	}
+	a = strings.ToLower(a)
+	b = strings.ToLower(b)
+	return strings.Compare(a, b)
+}

tools/tsgo/internal/stringutil/generate.go

@@ -0,0 +1,4 @@
+package stringutil
+
+//go:generate node --experimental-strip-types --no-warnings ./_scripts/generate-unicode-data.mts
+//go:generate npx dprint fmt js_case_generated.go identifier_parts_generated.go

tools/tsgo/internal/stringutil/identifier.go

@@ -0,0 +1,17 @@
+package stringutil
+
+import "unicode"
+
+// IsUnicodeIdentifierStart reports whether ch may begin an ECMAScript
+// identifier, i.e. whether it has the Unicode ID_Start (or Other_ID_Start)
+// property. The range table is generated; see generate-unicode-data.mts.
+func IsUnicodeIdentifierStart(ch rune) bool {
+	return unicode.Is(unicodeESNextIdentifierStart, ch)
+}
+
+// IsUnicodeIdentifierPart reports whether ch may appear after the first
+// character of an ECMAScript identifier, i.e. whether it has the Unicode
+// ID_Continue (or Other_ID_Continue) property, which also includes ID_Start.
+func IsUnicodeIdentifierPart(ch rune) bool {
+	return unicode.Is(unicodeESNextIdentifierPart, ch)
+}

tools/tsgo/internal/stringutil/js_case.go

@@ -0,0 +1,157 @@
+package stringutil
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+func ToLowerJS(str string) string {
+	if ascii, ok := toLowerASCII(str); ok {
+		return ascii
+	}
+
+	var builder strings.Builder
+	builder.Grow(len(str))
+	// casedBefore tracks whether the most recent non-Case_Ignorable code point is
+	// "cased", which is the backward half of the Final_Sigma context. We
+	// accumulate it as we stream so we never have to scan (or decode) backwards.
+	casedBefore := false
+	for i := 0; i < len(str); {
+		r, size := DecodeJSStringRune(str[i:])
+		i += size
+		if IsSurrogate(r) {
+			// A lone surrogate has no case mapping; preserve it verbatim, matching
+			// String.prototype.toLowerCase. EncodeJSStringRune restores the sentinel
+			// bytes because WriteRune would re-encode the surrogate as U+FFFD.
+			builder.WriteString(EncodeJSStringRune(r))
+		} else if mapping, ok := specialCasingMappings[r]; ok {
+			if mapping.condition == specialCasingConditionFinalSigma && isFinalSigmaContext(casedBefore, str, i) {
+				builder.WriteString(mapping.conditionalLower)
+			} else {
+				builder.WriteString(mapping.lower)
+			}
+		} else {
+			builder.WriteRune(r)
+		}
+		if !isUnicodeCaseIgnorable(r) {
+			casedBefore = isSigmaCased(r)
+		}
+	}
+	return builder.String()
+}
+
+func ToUpperJS(str string) string {
+	if ascii, ok := toUpperASCII(str); ok {
+		return ascii
+	}
+
+	var builder strings.Builder
+	builder.Grow(len(str))
+	for i := 0; i < len(str); {
+		r, size := DecodeJSStringRune(str[i:])
+		if IsSurrogate(r) {
+			// A lone surrogate has no case mapping; preserve it verbatim, matching
+			// String.prototype.toUpperCase. Copy the sentinel bytes directly because
+			// WriteRune would re-encode the surrogate as U+FFFD.
+			builder.WriteString(str[i : i+size])
+		} else if mapping, ok := specialCasingMappings[r]; ok {
+			builder.WriteString(mapping.upper)
+		} else {
+			builder.WriteRune(r)
+		}
+		i += size
+	}
+
+	return builder.String()
+}
+
+func toLowerASCII(str string) (string, bool) {
+	needsMapping := false
+	for i := range len(str) {
+		ch := str[i]
+		if ch >= utf8.RuneSelf {
+			return "", false
+		}
+		needsMapping = needsMapping || ('A' <= ch && ch <= 'Z')
+	}
+	if !needsMapping {
+		return str, true
+	}
+
+	buf := []byte(str)
+	for i, ch := range buf {
+		if 'A' <= ch && ch <= 'Z' {
+			buf[i] = ch + ('a' - 'A')
+		}
+	}
+	return string(buf), true
+}
+
+func toUpperASCII(str string) (string, bool) {
+	needsMapping := false
+	for i := range len(str) {
+		ch := str[i]
+		if ch >= utf8.RuneSelf {
+			return "", false
+		}
+		needsMapping = needsMapping || ('a' <= ch && ch <= 'z')
+	}
+	if !needsMapping {
+		return str, true
+	}
+
+	buf := []byte(str)
+	for i, ch := range buf {
+		if 'a' <= ch && ch <= 'z' {
+			buf[i] = ch - ('a' - 'A')
+		}
+	}
+	return string(buf), true
+}
+
+// isFinalSigmaContext reports whether a sigma at the current position is in
+// Final_Sigma context: it is preceded by a cased code point and not followed by
+// one. casedBefore carries the backward half (tracked incrementally by the
+// caller so we never scan backwards); afterOffset is the byte offset just past
+// the sigma, from which we scan forward.
+//
+// ECMAScript points at Unicode Default Case Conversion for toLowerCase, and
+// modern V8 reaches that behavior through Intl::ConvertToLower, which uses
+// ICU root-locale lowercasing for non-Latin1 strings like Greek sigma.
+// We intentionally do not delegate this to golang.org/x/text/cases: x/text
+// is a general Unicode casing library, but its root-locale behavior is not
+// an exact match for the JS semantics exercised by String.prototype
+// .toLowerCase(), especially around Final_Sigma context. TypeScript needs the
+// JS behavior itself here, so we keep the context-sensitive part explicit.
+// SpiderMonkey models Final_Sigma with a more explicit context walk, while
+// Unicode Table 3-17 describes it in terms of Cased and Case_Ignorable.
+// We model the exposed V8/ICU behavior directly here: skip Case_Ignorable code
+// points and then look for a Cased code point, exactly as Unicode Table 3-17
+// defines the Final_Sigma condition. The Cased property already subsumes
+// lowercase, uppercase, and titlecase letters, including the
+// DerivedCoreProperties Lowercase/Uppercase extras such as ª, º, and Roman
+// numerals.
+func isFinalSigmaContext(casedBefore bool, str string, afterOffset int) bool {
+	return casedBefore && !hasSigmaCasedAfter(str, afterOffset)
+}
+
+func hasSigmaCasedAfter(str string, start int) bool {
+	for i := start; i < len(str); {
+		r, size := DecodeJSStringRune(str[i:])
+		i += size
+		if isUnicodeCaseIgnorable(r) {
+			continue
+		}
+		return isSigmaCased(r)
+	}
+	return false
+}
+
+func isSigmaCased(r rune) bool {
+	return unicode.Is(unicodeCasedRanges, r)
+}
+
+func isUnicodeCaseIgnorable(r rune) bool {
+	return unicode.Is(unicodeCaseIgnorableRanges, r)
+}

tools/tsgo/internal/stringutil/js_case_test.go

@@ -0,0 +1,49 @@
+package stringutil
+
+import "testing"
+
+func TestJSCasing(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		got  string
+		want string
+	}{
+		{name: "ascii lowercase", got: ToLowerJS("HELLO"), want: "hello"},
+		{name: "ascii uppercase", got: ToUpperJS("hello"), want: "HELLO"},
+		{name: "lowercase dotted i", got: ToLowerJS("İSPANYOL"), want: "i̇spanyol"},
+		{name: "lowercase lone sigma", got: ToLowerJS("Σ"), want: "σ"},
+		{name: "lowercase final sigma", got: ToLowerJS("ΟΣ"), want: "ος"},
+		{name: "lowercase non-sigma greek", got: ToLowerJS("Ω"), want: "ω"},
+		{name: "uppercase sharp s", got: ToUpperJS("ßfoo"), want: "SSFOO"},
+		{name: "uppercase non-ascii simple mapping", got: ToUpperJS("ω"), want: "Ω"},
+		{name: "uppercase ligature", got: ToUpperJS("fioo"), want: "FIOO"},
+		{name: "capitalize-style uppercase", got: ToUpperJS("ß") + "foo", want: "SSfoo"},
+		{name: "uncapitalize-style lowercase", got: ToLowerJS("İ") + "foo", want: "i̇foo"},
+		{name: "lowercase final sigma after lowercase letter without uppercase mapping", got: ToLowerJS("ʕΣ"), want: "ʕς"},
+		{name: "lowercase sigma after modifier letter", got: ToLowerJS("ʰΣ"), want: "ʰσ"},
+		{name: "lowercase sigma after case ignorable ypogegrammeni", got: ToLowerJS("ͅΣ"), want: "ͅσ"},
+		{name: "lowercase final sigma after feminine ordinal indicator", got: ToLowerJS("ªΣ"), want: "ªς"},
+		{name: "lowercase final sigma after masculine ordinal indicator", got: ToLowerJS("ºΣ"), want: "ºς"},
+		{name: "lowercase final sigma after roman numeral", got: ToLowerJS("ⅠΣ"), want: "ⅰς"},
+		{name: "lowercase sigma after uppercase property added after unicode 15", got: ToLowerJS("\u1C89Σ"), want: "\u1C89σ"},
+		{name: "lowercase sigma after uppercase property skewed from local v8 unicode data", got: ToLowerJS("\uA7CBΣ"), want: "\uA7CBσ"},
+		{name: "lowercase sigma before immediate latin letter", got: ToLowerJS("ΣA"), want: "σa"},
+		{name: "lowercase sigma before immediate roman numeral letter", got: ToLowerJS("ΣⅠ"), want: "σⅰ"},
+		{name: "lowercase sigma before case ignorable then latin letter", got: ToLowerJS("ΣͅA"), want: "σͅa"},
+		{name: "uppercase lone surrogate", got: ToUpperJS(EncodeJSStringRune(0xD800)), want: EncodeJSStringRune(0xD800)},
+		{name: "lowercase lone surrogate", got: ToLowerJS("A" + EncodeJSStringRune(0xD800) + "B"), want: "a" + EncodeJSStringRune(0xD800) + "b"},
+		{name: "uppercase lone low surrogate with text", got: ToUpperJS(EncodeJSStringRune(0xDC00) + "x"), want: EncodeJSStringRune(0xDC00) + "X"},
+		{name: "lowercase lone surrogate before sigma", got: ToLowerJS(EncodeJSStringRune(0xD800) + "Σ"), want: EncodeJSStringRune(0xD800) + "σ"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if tt.got != tt.want {
+				t.Fatalf("got %q, want %q", tt.got, tt.want)
+			}
+		})
+	}
+}

tools/tsgo/internal/stringutil/util.go

@@ -0,0 +1,380 @@
+// Package stringutil Exports common rune utilities for parsing and emitting javascript
+package stringutil
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf16"
+	"unicode/utf8"
+)
+
+func IsWhiteSpaceLike(ch rune) bool {
+	return IsWhiteSpaceSingleLine(ch) || IsLineBreak(ch)
+}
+
+func IsWhiteSpaceSingleLine(ch rune) bool {
+	// Note: nextLine is in the Zs space, and should be considered to be a whitespace.
+	// It is explicitly not a line-break as it isn't in the exact set specified by EcmaScript.
+	switch ch {
+	case
+		' ',    // space
+		'\t',   // tab
+		'\v',   // verticalTab
+		'\f',   // formFeed
+		0x0085, // nextLine
+		0x00A0, // nonBreakingSpace
+		0x1680, // ogham
+		0x2000, // enQuad
+		0x2001, // emQuad
+		0x2002, // enSpace
+		0x2003, // emSpace
+		0x2004, // threePerEmSpace
+		0x2005, // fourPerEmSpace
+		0x2006, // sixPerEmSpace
+		0x2007, // figureSpace
+		0x2008, // punctuationEmSpace
+		0x2009, // thinSpace
+		0x200A, // hairSpace
+		0x200B, // zeroWidthSpace
+		0x202F, // narrowNoBreakSpace
+		0x205F, // mathematicalSpace
+		0x3000, // ideographicSpace
+		0xFEFF: // byteOrderMark
+		return true
+	}
+	return false
+}
+
+func IsLineBreak(ch rune) bool {
+	// ES5 7.3:
+	// The ECMAScript line terminator characters are listed in Table 3.
+	//     Table 3: Line Terminator Characters
+	//     Code Unit Value     Name                    Formal Name
+	//     \u000A              Line Feed               <LF>
+	//     \u000D              Carriage Return         <CR>
+	//     \u2028              Line separator          <LS>
+	//     \u2029              Paragraph separator     <PS>
+	// Only the characters in Table 3 are treated as line terminators. Other new line or line
+	// breaking characters are treated as white space but not as line terminators.
+	switch ch {
+	case
+		'\n',   // lineFeed
+		'\r',   // carriageReturn
+		0x2028, // lineSeparator
+		0x2029: // paragraphSeparator
+		return true
+	}
+	return false
+}
+
+func IsDigit(ch rune) bool {
+	return ch >= '0' && ch <= '9'
+}
+
+func IsOctalDigit(ch rune) bool {
+	return ch >= '0' && ch <= '7'
+}
+
+func IsHexDigit(ch rune) bool {
+	return ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f'
+}
+
+func IsASCIILetter(ch rune) bool {
+	return ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'
+}
+
+func ContainsNonASCII(s string) bool {
+	for i := range len(s) {
+		if s[i] >= utf8.RuneSelf {
+			return true
+		}
+	}
+	return false
+}
+
+func SplitLines(text string) []string {
+	lines := make([]string, 0, strings.Count(text, "\n")+1) // preallocate
+	start := 0
+	pos := 0
+	for pos < len(text) {
+		switch text[pos] {
+		case '\r':
+			if pos+1 < len(text) && text[pos+1] == '\n' {
+				lines = append(lines, text[start:pos])
+				pos += 2
+				start = pos
+				continue
+			}
+			fallthrough
+		case '\n':
+			lines = append(lines, text[start:pos])
+			pos++
+			start = pos
+			continue
+		}
+		pos++
+	}
+	if start < len(text) {
+		lines = append(lines, text[start:])
+	}
+	return lines
+}
+
+func GuessIndentation(lines []string) int {
+	const MAX_SMI_X86 int = 0x3fff_ffff
+	indentation := MAX_SMI_X86
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+		i := 0
+		for i < len(line) && i < indentation {
+			ch, size := utf8.DecodeRuneInString(line[i:])
+			if !IsWhiteSpaceLike(ch) {
+				break
+			}
+			i += size
+		}
+		if i < indentation {
+			indentation = i
+		}
+		if indentation == 0 {
+			return 0
+		}
+	}
+	if indentation == MAX_SMI_X86 {
+		return 0
+	}
+	return indentation
+}
+
+// https://tc39.es/ecma262/multipage/global-object.html#sec-encodeuri-uri
+func EncodeURI(s string) string {
+	var builder strings.Builder
+	for i := range len(s) {
+		b := s[i]
+		if !shouldEscapeForEncodeURI(b) {
+			builder.WriteByte(b)
+			continue
+		}
+
+		for _, escaped := range []byte(s[i : i+1]) {
+			builder.WriteByte('%')
+			builder.WriteByte(upperhex[escaped>>4])
+			builder.WriteByte(upperhex[escaped&0x0f])
+		}
+	}
+	return builder.String()
+}
+
+const upperhex = "0123456789ABCDEF"
+
+func shouldEscapeForEncodeURI(b byte) bool {
+	switch {
+	case b >= 'A' && b <= 'Z':
+		return false
+	case b >= 'a' && b <= 'z':
+		return false
+	case b >= '0' && b <= '9':
+		return false
+	}
+
+	switch b {
+	case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '#', '-', '_', '.', '!', '~', '*', '\'', '(', ')':
+		return false
+	default:
+		return true
+	}
+}
+
+func getByteOrderMarkLength(text string) int {
+	if len(text) >= 1 {
+		ch0 := text[0]
+		if ch0 == 0xfe {
+			if len(text) >= 2 && text[1] == 0xff {
+				return 2 // utf16be
+			}
+			return 0
+		}
+		if ch0 == 0xff {
+			if len(text) >= 2 && text[1] == 0xfe {
+				return 2 // utf16le
+			}
+			return 0
+		}
+		if ch0 == 0xef {
+			if len(text) >= 3 && text[1] == 0xbb && text[2] == 0xbf {
+				return 3 // utf8
+			}
+			return 0
+		}
+	}
+	return 0
+}
+
+func RemoveByteOrderMark(text string) string {
+	length := getByteOrderMarkLength(text)
+	if length > 0 {
+		return text[length:]
+	}
+	return text
+}
+
+func AddUTF8ByteOrderMark(text string) string {
+	if getByteOrderMarkLength(text) == 0 {
+		return "\xEF\xBB\xBF" + text
+	}
+	return text
+}
+
+func StripQuotes(name string) string {
+	if len(name) < 2 {
+		return name
+	}
+	firstChar, _ := utf8.DecodeRuneInString(name)
+	lastChar, _ := utf8.DecodeLastRuneInString(name)
+	if firstChar == lastChar && (firstChar == '\'' || firstChar == '"' || firstChar == '`') {
+		return name[1 : len(name)-1]
+	}
+	return name
+}
+
+var matchSlashSomething = regexp.MustCompile(`\\.`)
+
+func matchSlashReplacer(in string) string {
+	return in[1:]
+}
+
+func UnquoteString(str string) string {
+	// strconv.Unquote is insufficient as that only handles a single character inside single quotes, as those are character literals in go
+	inner := StripQuotes(str)
+	// In strada we do str.replace(/\\./g, s => s.substring(1)) - which is to say, replace all backslash-something with just something
+	// That's replicated here faithfully, but it seems wrong! This should probably be an actual unquote operation?
+	return matchSlashSomething.ReplaceAllStringFunc(inner, matchSlashReplacer)
+}
+
+func LowerFirstChar(str string) string {
+	char, size := utf8.DecodeRuneInString(str)
+	if size > 0 {
+		return string(unicode.ToLower(char)) + str[size:]
+	}
+	return str
+}
+
+func TruncateByRunes(str string, maxLength int) string {
+	if len(str) < maxLength {
+		return str
+	}
+	if maxLength <= 0 {
+		return ""
+	}
+	var runeCount int
+	for i := range str {
+		runeCount++
+		if runeCount > maxLength {
+			return str[:i]
+		}
+	}
+	return str
+}
+
+const (
+	// SurrogateLowStart is the boundary between the high and low halves of the
+	// UTF-16 surrogate range. unicode/utf16 only exposes IsSurrogate for the
+	// whole range, so this split point is defined here to distinguish the two.
+	SurrogateLowStart = 0xDC00
+)
+
+func IsHighSurrogate(ch rune) bool {
+	return utf16.IsSurrogate(ch) && ch < SurrogateLowStart
+}
+
+func IsLowSurrogate(ch rune) bool {
+	return utf16.IsSurrogate(ch) && ch >= SurrogateLowStart
+}
+
+func IsSurrogate(ch rune) bool {
+	return utf16.IsSurrogate(ch)
+}
+
+func SurrogatePairToCodePoint(high rune, low rune) rune {
+	return utf16.DecodeRune(high, low)
+}
+
+func CodePointToSurrogatePair(ch rune) (high rune, low rune) {
+	return utf16.EncodeRune(ch)
+}
+
+const (
+	// A lone surrogate (U+D800–U+DFFF) cannot be represented in valid UTF-8, so
+	// EncodeJSStringRune stores it as the 3-byte CESU-8/WTF-8 sentinel that UTF-8
+	// would use for that code point if surrogates were encodable. unicode/utf8
+	// and unicode/utf16 deliberately refuse to encode or decode surrogates, so
+	// the byte math is spelled out here.
+	//
+	// Byte layout for a code point cp in U+D000–U+DFFF (lead nibble 0xD):
+	//   byte0 = 0xE0 | (cp >> 12)          == 0xED
+	//   byte1 = 0x80 | ((cp >> 6) & 0x3F)
+	//   byte2 = 0x80 | (cp & 0x3F)
+	surrogateUTF8Lead     = 0xED   // byte0, shared by the whole U+D000–U+DFFF block
+	surrogateUTF8LeadBits = 0xD000 // (surrogateUTF8Lead & 0x0F) << 12, byte0's decoded contribution
+	utf8ContMarker        = 0x80   // continuation byte marker / min value (10xxxxxx)
+	utf8ContMax           = 0xBF   // continuation byte max value
+	utf8ContMask          = 0x3F   // data bits carried by a continuation byte
+
+	// byte1 bounds that pin the block down to the surrogate range U+D800–U+DFFF:
+	// 0xD800 -> 0xA0, 0xDFFF -> 0xBF.
+	surrogateUTF8Byte1Min = 0xA0
+	surrogateUTF8Byte1Max = 0xBF
+)
+
+func EncodeJSStringRune(ch rune) string {
+	if IsSurrogate(ch) {
+		return string([]byte{
+			surrogateUTF8Lead,
+			byte(utf8ContMarker | ((ch >> 6) & utf8ContMask)),
+			byte(utf8ContMarker | (ch & utf8ContMask)),
+		})
+	}
+	return string(ch)
+}
+
+func DecodeJSStringRune(s string) (rune, int) {
+	if len(s) >= 3 &&
+		s[0] == surrogateUTF8Lead &&
+		s[1] >= surrogateUTF8Byte1Min && s[1] <= surrogateUTF8Byte1Max &&
+		s[2] >= utf8ContMarker && s[2] <= utf8ContMax {
+		return surrogateUTF8LeadBits | rune(s[1]&utf8ContMask)<<6 | rune(s[2]&utf8ContMask), 3
+	}
+	return utf8.DecodeRuneInString(s)
+}
+
+// CombineSurrogatePairs canonicalizes a JS-string value produced by
+// concatenation, merging any adjacent high+low surrogate sentinel pair (as
+// written by EncodeJSStringRune) into the single supplementary code point they
+// represent. This mirrors how concatenating two UTF-16 code units forms a
+// surrogate pair in a JavaScript string. It must be applied wherever separately
+// scanned string values are joined, since each half is only a lone surrogate
+// until it meets its partner. Strings without a lone-surrogate sentinel (the
+// common case) are returned unchanged.
+func CombineSurrogatePairs(s string) string {
+	if strings.IndexByte(s, surrogateUTF8Lead) < 0 {
+		return s
+	}
+	var b strings.Builder
+	b.Grow(len(s))
+	for i := 0; i < len(s); {
+		r, size := DecodeJSStringRune(s[i:])
+		if IsHighSurrogate(r) {
+			if low, lowSize := DecodeJSStringRune(s[i+size:]); IsLowSurrogate(low) {
+				b.WriteRune(SurrogatePairToCodePoint(r, low))
+				i += size + lowSize
+				continue
+			}
+		}
+		b.WriteString(s[i : i+size])
+		i += size
+	}
+	return b.String()
+}

tools/tsgo/internal/stringutil/util_test.go

@@ -0,0 +1,59 @@
+package stringutil
+
+import "testing"
+
+func TestEncodeURI(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "encodes spaces as percent20",
+			input:    "a b",
+			expected: "a%20b",
+		},
+		{
+			name:     "preserves reserved uri characters",
+			input:    ";/?:@&=+$,#",
+			expected: ";/?:@&=+$,#",
+		},
+		{
+			name:     "encodes brackets and unicode using utf8 bytes",
+			input:    "①Ⅻㄨㄩ U1[abc]",
+			expected: "%E2%91%A0%E2%85%AB%E3%84%A8%E3%84%A9%20U1%5Babc%5D",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := EncodeURI(tt.input); got != tt.expected {
+				t.Fatalf("EncodeURI(%q) = %q, expected %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestContainsNonASCII(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		text string
+		want bool
+	}{
+		{name: "ascii", text: "abc", want: false},
+		{name: "non-ascii", text: "é", want: true},
+		{name: "lone surrogate sentinel", text: EncodeJSStringRune(0xD800), want: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := ContainsNonASCII(tt.text); got != tt.want {
+				t.Fatalf("ContainsNonASCII(%q) = %v, want %v", tt.text, got, tt.want)
+			}
+		})
+	}
+}