package ast_test import ( "os" "strings" "testing" "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/stringutil" ) func TestPositionMapASCII(t *testing.T) { t.Parallel() text := "const x = 1;" pm := ast.ComputePositionMap(text) if !pm.IsAsciiOnly() { t.Fatal("expected ASCII-only") } for i := 0; i <= len(text); i++ { if got := pm.UTF8ToUTF16(i); got != i { t.Errorf("UTF8ToUTF16(%d) = %d, want %d", i, got, i) } if got := pm.UTF16ToUTF8(i); got != i { t.Errorf("UTF16ToUTF8(%d) = %d, want %d", i, got, i) } } } func TestPositionMapTwoByte(t *testing.T) { t.Parallel() // "café" — é (U+00E9) is 2 bytes UTF-8, 1 code unit UTF-16 text := "const café = 1;\nconst x = 2;" pm := ast.ComputePositionMap(text) if pm.IsAsciiOnly() { t.Fatal("expected non-ASCII") } // Everything before é (byte offset 9) should be identity for i := range 10 { if got := pm.UTF8ToUTF16(i); got != i { t.Errorf("before é: UTF8ToUTF16(%d) = %d, want %d", i, got, i) } } // é starts at UTF-8 byte 9, UTF-16 offset 9: same if got := pm.UTF8ToUTF16(9); got != 9 { t.Errorf("at é: UTF8ToUTF16(9) = %d, want 9", got) } // After é (byte 11 in UTF-8 = code unit 10 in UTF-16), delta is 1 // ' ' after café: UTF-8 byte 11, UTF-16 offset 10 if got := pm.UTF8ToUTF16(11); got != 10 { t.Errorf("after é: UTF8ToUTF16(11) = %d, want 10", got) } // 'x' on second line: UTF-8 byte 23, UTF-16 offset 22 xUTF8 := strings.LastIndex(text, "x") if got := pm.UTF8ToUTF16(xUTF8); got != xUTF8-1 { t.Errorf("at x: UTF8ToUTF16(%d) = %d, want %d", xUTF8, got, xUTF8-1) } // Reverse: UTF-16 offset 22 should map to UTF-8 byte 23 xUTF16 := xUTF8 - 1 if got := pm.UTF16ToUTF8(xUTF16); got != xUTF8 { t.Errorf("reverse at x: UTF16ToUTF8(%d) = %d, want %d", xUTF16, got, xUTF8) } } func TestPositionMapFourByte(t *testing.T) { t.Parallel() // 🎉 (U+1F389) is 4 bytes UTF-8, 2 code units UTF-16 text := `const a = "🎉";` + "\nconst b = 2;" pm := ast.ComputePositionMap(text) if pm.IsAsciiOnly() { t.Fatal("expected non-ASCII") } // 🎉 starts at byte 11 (after `const a = "`) // UTF-8: bytes 11-14 (4 bytes), UTF-16: units 11-12 (2 code units) // After 🎉: UTF-8 byte 15, UTF-16 offset 13. Delta = 2. // 'b' on second line bUTF8 := strings.LastIndex(text, "b") bUTF16 := bUTF8 - 2 // delta of 2 from emoji if got := pm.UTF8ToUTF16(bUTF8); got != bUTF16 { t.Errorf("at b: UTF8ToUTF16(%d) = %d, want %d", bUTF8, got, bUTF16) } if got := pm.UTF16ToUTF8(bUTF16); got != bUTF8 { t.Errorf("reverse at b: UTF16ToUTF8(%d) = %d, want %d", bUTF16, got, bUTF8) } } func TestPositionMapMultipleNonASCII(t *testing.T) { t.Parallel() // Mix of 2-byte and 4-byte characters // "à" (U+00E0) = 2 bytes UTF-8, 1 code unit UTF-16 (delta +1) // "🎉" (U+1F389) = 4 bytes UTF-8, 2 code units UTF-16 (delta +2) text := "à🎉x" pm := ast.ComputePositionMap(text) // à: UTF-8 [0,2), UTF-16 [0,1) // 🎉: UTF-8 [2,6), UTF-16 [1,3) // x: UTF-8 [6,7), UTF-16 [3,4) tests := []struct { utf8 int utf16 int }{ {0, 0}, {2, 1}, // start of 🎉 {6, 3}, // x {7, 4}, // end } for _, tt := range tests { if got := pm.UTF8ToUTF16(tt.utf8); got != tt.utf16 { t.Errorf("UTF8ToUTF16(%d) = %d, want %d", tt.utf8, got, tt.utf16) } if got := pm.UTF16ToUTF8(tt.utf16); got != tt.utf8 { t.Errorf("UTF16ToUTF8(%d) = %d, want %d", tt.utf16, got, tt.utf8) } } } func TestPositionMapLoneSurrogateSentinel(t *testing.T) { t.Parallel() text := "a" + stringutil.EncodeJSStringRune(0xD800) + "b" pm := ast.ComputePositionMap(text) if pm.IsAsciiOnly() { t.Fatal("expected non-ASCII") } if got := pm.UTF8ToUTF16(len(text)); got != 3 { t.Errorf("UTF8ToUTF16(%d) = %d, want 3", len(text), got) } if got := pm.UTF16ToUTF8(2); got != len(text)-1 { t.Errorf("UTF16ToUTF8(2) = %d, want %d", got, len(text)-1) } } func TestPositionMapRoundtrip(t *testing.T) { t.Parallel() text := "let café = \"🎉\"; // naïve" pm := ast.ComputePositionMap(text) // Convert every valid UTF-16 position to UTF-8 and back utf16Len := pm.UTF8ToUTF16(len(text)) for i := 0; i <= utf16Len; i++ { utf8Pos := pm.UTF16ToUTF8(i) back := pm.UTF8ToUTF16(utf8Pos) if back != i { t.Errorf("roundtrip UTF16->UTF8->UTF16: %d -> %d -> %d", i, utf8Pos, back) } } } func BenchmarkComputePositionMap_ASCII(b *testing.B) { // ~10KB of ASCII TypeScript-like code line := "const variable = someFunction(argument1, argument2);\n" text := strings.Repeat(line, 200) b.ResetTimer() for range b.N { ast.ComputePositionMap(text) } } func BenchmarkComputePositionMap_NonASCII(b *testing.B) { // Mix of ASCII and non-ASCII (comments with unicode) line := "const café = \"héllo wörld 🎉\";\n" text := strings.Repeat(line, 200) b.ResetTimer() for range b.N { ast.ComputePositionMap(text) } } func BenchmarkUTF8ToUTF16_ASCII(b *testing.B) { line := "const variable = someFunction(argument1, argument2);\n" text := strings.Repeat(line, 200) pm := ast.ComputePositionMap(text) positions := []int{0, 100, 500, 1000, 5000, len(text) - 1} b.ResetTimer() for range b.N { for _, p := range positions { pm.UTF8ToUTF16(p) } } } func BenchmarkUTF8ToUTF16_NonASCII(b *testing.B) { line := "const café = \"héllo wörld 🎉\";\n" text := strings.Repeat(line, 200) pm := ast.ComputePositionMap(text) positions := []int{0, 100, 500, 1000, 5000, len(text) - 1} b.ResetTimer() for range b.N { for _, p := range positions { pm.UTF8ToUTF16(p) } } } func BenchmarkUTF16ToUTF8_NonASCII(b *testing.B) { line := "const café = \"héllo wörld 🎉\";\n" text := strings.Repeat(line, 200) pm := ast.ComputePositionMap(text) utf16Len := pm.UTF8ToUTF16(len(text)) positions := []int{0, 100, 500, 1000, 3000, utf16Len - 1} b.ResetTimer() for range b.N { for _, p := range positions { pm.UTF16ToUTF8(p) } } } func BenchmarkComputePositionMap_CheckerTS(b *testing.B) { data, err := os.ReadFile("../../_submodules/TypeScript/src/compiler/checker.ts") if err != nil { b.Skip("checker.ts not available:", err) } text := string(data) b.ResetTimer() for range b.N { ast.ComputePositionMap(text) } }