package lsconv_test import ( "bytes" "encoding/binary" "fmt" "os/exec" "testing" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/json" "github.com/microsoft/typescript-go/internal/ls/lsconv" "github.com/microsoft/typescript-go/internal/lsp/lsproto" "gotest.tools/v3/assert" ) func TestDocumentURIToFileName(t *testing.T) { t.Parallel() tests := []struct { uri lsproto.DocumentUri fileName string }{ {"file:///path/to/file.ts", "/path/to/file.ts"}, {"file://server/share/file.ts", "//server/share/file.ts"}, {"file:///d%3A/work/tsgo932/lib/utils.ts", "d:/work/tsgo932/lib/utils.ts"}, {"file:///D%3A/work/tsgo932/lib/utils.ts", "d:/work/tsgo932/lib/utils.ts"}, {"file:///d%3A/work/tsgo932/app/%28test%29/comp/comp-test.tsx", "d:/work/tsgo932/app/(test)/comp/comp-test.tsx"}, {"file:///path/to/file.ts#section", "/path/to/file.ts"}, {"file:///c:/test/me", "c:/test/me"}, {"file://shares/files/c%23/p.cs", "//shares/files/c#/p.cs"}, {"file:///c:/Source/Z%C3%BCrich%20or%20Zurich%20(%CB%88zj%CA%8A%C9%99r%C9%AAk,/Code/resources/app/plugins/c%23/plugin.json", "c:/Source/Zürich or Zurich (ˈzjʊərɪk,/Code/resources/app/plugins/c#/plugin.json"}, {"file:///c:/test %25/path", "c:/test %/path"}, // {"file:?q", "/"}, {"file:///_:/path", "/_:/path"}, {"file:///users/me/c%23-projects/", "/users/me/c#-projects/"}, {"file://localhost/c%24/GitDevelopment/express", "//localhost/c$/GitDevelopment/express"}, {"file:///c%3A/test%20with%20%2525/c%23code", "c:/test with %25/c#code"}, {"untitled:Untitled-1", "^/untitled/ts-nul-authority/Untitled-1"}, {"untitled:Untitled-1#fragment", "^/untitled/ts-nul-authority/Untitled-1#fragment"}, {"untitled:c:/Users/jrieken/Code/abc.txt", "^/untitled/ts-nul-authority/c:/Users/jrieken/Code/abc.txt"}, {"untitled:C:/Users/jrieken/Code/abc.txt", "^/untitled/ts-nul-authority/C:/Users/jrieken/Code/abc.txt"}, {"untitled://wsl%2Bubuntu/home/jabaile/work/TypeScript-go/newfile.ts", "^/untitled/wsl%2Bubuntu/home/jabaile/work/TypeScript-go/newfile.ts"}, } for _, test := range tests { t.Run(string(test.uri), func(t *testing.T) { t.Parallel() assert.Equal(t, test.uri.FileName(), test.fileName) }) } } func TestFileNameToDocumentURI(t *testing.T) { t.Parallel() tests := []struct { fileName string uri lsproto.DocumentUri }{ {"/path/to/file.ts", "file:///path/to/file.ts"}, {"//server/share/file.ts", "file://server/share/file.ts"}, {"d:/work/tsgo932/lib/utils.ts", "file:///d%3A/work/tsgo932/lib/utils.ts"}, {"d:/work/tsgo932/lib/utils.ts", "file:///d%3A/work/tsgo932/lib/utils.ts"}, {"d:/work/tsgo932/app/(test)/comp/comp-test.tsx", "file:///d%3A/work/tsgo932/app/%28test%29/comp/comp-test.tsx"}, {"/path/to/file.ts", "file:///path/to/file.ts"}, {"c:/test/me", "file:///c%3A/test/me"}, {"//shares/files/c#/p.cs", "file://shares/files/c%23/p.cs"}, {"c:/Source/Zürich or Zurich (ˈzjʊərɪk,/Code/resources/app/plugins/c#/plugin.json", "file:///c%3A/Source/Z%C3%BCrich%20or%20Zurich%20%28%CB%88zj%CA%8A%C9%99r%C9%AAk%2C/Code/resources/app/plugins/c%23/plugin.json"}, {"c:/test %/path", "file:///c%3A/test%20%25/path"}, {"/", "file:///"}, {"/_:/path", "file:///_%3A/path"}, {"/users/me/c#-projects/", "file:///users/me/c%23-projects/"}, {"//localhost/c$/GitDevelopment/express", "file://localhost/c%24/GitDevelopment/express"}, {"c:/test with %25/c#code", "file:///c%3A/test%20with%20%2525/c%23code"}, {"^/untitled/ts-nul-authority/Untitled-1", "untitled:Untitled-1"}, {"^/untitled/ts-nul-authority/c:/Users/jrieken/Code/abc.txt", "untitled:c:/Users/jrieken/Code/abc.txt"}, {"^/untitled/ts-nul-authority///wsl%2Bubuntu/home/jabaile/work/TypeScript-go/newfile.ts", "untitled://wsl%2Bubuntu/home/jabaile/work/TypeScript-go/newfile.ts"}, } for _, test := range tests { t.Run(test.fileName, func(t *testing.T) { t.Parallel() assert.Equal(t, lsconv.FileNameToDocumentURI(test.fileName), test.uri) }) } } type testScript struct { name string text string } func (s *testScript) FileName() string { return s.name } func (s *testScript) Text() string { return s.text } func newTestConverters(text string) (*lsconv.Converters, *testScript) { script := &testScript{name: "test.ts", text: text} lineMap := lsconv.ComputeLSPLineStarts(text) conv := lsconv.NewConverters(lsproto.PositionEncodingKindUTF16, func(_ string) *lsconv.LSPLineMap { return lineMap }) return conv, script } // TestConvertersInvalidUTF8 verifies behavior on text containing invalid UTF-8 // sequences (e.g. lone continuation bytes). Node's TextDecoder substitutes such // bytes with U+FFFD, so the JS-reference test cannot cover this; we assert the // expected Go-side behavior directly. Each invalid byte advances the byte // position by 1 and the UTF-16 character by 1 (RuneError = 1 code unit). func TestConvertersInvalidUTF8(t *testing.T) { t.Parallel() // Text with invalid UTF-8 byte 0x80 (continuation byte without start byte). // Old code used utf8.RuneLen(RuneError)==3, overshooting the byte offset. text := "a\x80b\ncd" conv, script := newTestConverters(text) // (line, char) → byte position. Each row asserts both directions where the // position lies on a character boundary. mappings := []struct { line, char uint32 bytePos core.TextPos }{ {0, 0, 0}, // 'a' {0, 1, 1}, // invalid byte 0x80 {0, 2, 2}, // 'b' {0, 3, 3}, // newline (line end) {1, 0, 4}, // 'c' {1, 1, 5}, // 'd' {1, 2, 6}, // EOF } for _, m := range mappings { lc := lsproto.Position{Line: m.line, Character: m.char} assert.Equal(t, conv.LineAndCharacterToPosition(script, lc), m.bytePos, fmt.Sprintf("LineAndCharacterToPosition(%d,%d)", m.line, m.char)) assert.Equal(t, conv.PositionToLineAndCharacter(script, m.bytePos), lc, fmt.Sprintf("PositionToLineAndCharacter(%d)", m.bytePos)) } // Byte-by-byte round-trip across the entire text. for bytePos := core.TextPos(0); bytePos <= core.TextPos(len(text)); bytePos++ { lc := conv.PositionToLineAndCharacter(script, bytePos) rt := conv.LineAndCharacterToPosition(script, lc) assert.Equal(t, rt, bytePos, fmt.Sprintf("round-trip byte %d", bytePos)) } } // jsReferenceScript is a Node.js script that, given a list of UTF-8 byte buffers, // computes the authoritative mapping between (line, character in UTF-16 code units) // and UTF-8 byte offsets. // // To avoid any string round-tripping at the protocol boundary, the inputs are sent // as raw bytes: the test writes a length-prefixed binary stream to stdin // ([uint32 little-endian count][uint32 LE len][bytes]...[uint32 LE len][bytes]). // Node reads the buffers and decodes each with TextDecoder('utf-8') — which is // essentially what tsserver / sys.ts does when reading file contents from disk // (read as Buffer, decode as UTF-8 to a JS string with real UTF-16 semantics). // // For each input buffer, Node walks the underlying UTF-8 bytes (NOT the decoded // string) to identify codepoint boundaries: every byte is the start of a codepoint // unless it's a UTF-8 continuation byte (0b10xxxxxx). At each boundary it records // the UTF-8 byte offset and the corresponding UTF-16 code unit offset (in the // decoded JS string) and (line, char) using the LSP line-break rules // (\n, \r, \r\n only). // // Output is JSON on stdout: [ [ { bytePos, line, char }, ... ], ... ] const jsReferenceScript = ` const inChunks = []; process.stdin.on('data', c => inChunks.push(c)); process.stdin.on('end', () => { const buf = Buffer.concat(inChunks); let off = 0; const readU32 = () => { const v = buf.readUInt32LE(off); off += 4; return v; }; const n = readU32(); const buffers = []; for (let i = 0; i < n; i++) { const len = readU32(); buffers.push(buf.subarray(off, off + len)); off += len; } const decoder = new TextDecoder('utf-8', { fatal: true }); const out = buffers.map(bytes => { // Decode the raw UTF-8 bytes to a JS string (this is what sys.ts does with file contents). const text = decoder.decode(bytes); // LSP line starts in the *decoded* JS string: \\n, \\r, \\r\\n only. const lineStartsJs = [0]; for (let i = 0; i < text.length; i++) { const c = text.charCodeAt(i); if (c === 13) { if (i + 1 < text.length && text.charCodeAt(i + 1) === 10) i++; lineStartsJs.push(i + 1); } else if (c === 10) { lineStartsJs.push(i + 1); } } // Walk the original UTF-8 byte buffer to find codepoint boundaries. Inputs are // valid UTF-8, so we advance bytePos by the sequence length of each lead byte // and jsIdx by the corresponding UTF-16 code unit count (1 for BMP, 2 for // surrogate pair) of the codepoint at jsIdx in the decoded string. const boundaries = [{ bytePos: 0, jsIdx: 0 }]; let bytePos = 0, jsIdx = 0; while (bytePos < bytes.length) { const seq = utf8SeqLen(bytes[bytePos]); const cp = text.codePointAt(jsIdx); bytePos += seq; jsIdx += cp > 0xFFFF ? 2 : 1; boundaries.push({ bytePos, jsIdx }); } return boundaries.map(({ bytePos, jsIdx }) => { let lo = 0, hi = lineStartsJs.length - 1; while (lo < hi) { const mid = (lo + hi + 1) >> 1; if (lineStartsJs[mid] <= jsIdx) lo = mid; else hi = mid - 1; } return { bytePos, line: lo, char: jsIdx - lineStartsJs[lo] }; }); }); process.stdout.write(JSON.stringify(out)); }); function utf8SeqLen(b) { if (b < 0x80) return 1; if ((b & 0xE0) === 0xC0) return 2; if ((b & 0xF0) === 0xE0) return 3; if ((b & 0xF8) === 0xF0) return 4; throw new Error('invalid UTF-8 lead byte 0x' + b.toString(16)); } ` type jsTuple struct { BytePos int `json:"bytePos"` Line int `json:"line"` Char int `json:"char"` } func runJSReference(t *testing.T, texts []string) [][]jsTuple { t.Helper() if _, err := exec.LookPath("node"); err != nil { t.Skipf("node not available: %v", err) } // Build a length-prefixed binary stream of the raw UTF-8 bytes: // [uint32 LE count] then for each: [uint32 LE length][bytes]. var in bytes.Buffer var u32 [4]byte binary.LittleEndian.PutUint32(u32[:], uint32(len(texts))) in.Write(u32[:]) for _, s := range texts { binary.LittleEndian.PutUint32(u32[:], uint32(len(s))) in.Write(u32[:]) in.WriteString(s) } cmd := exec.Command("node", "-e", jsReferenceScript) cmd.Stdin = &in var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { t.Fatalf("node failed: %v\nstderr: %s", err, stderr.String()) } var out [][]jsTuple assert.NilError(t, json.Unmarshal(stdout.Bytes(), &out)) return out } // TestConvertersAgainstJSReference cross-checks the Go UTF-16 conversions against // authoritative results computed by Node.js using real UTF-16 string semantics. func TestConvertersAgainstJSReference(t *testing.T) { t.Parallel() cases := []struct { name string text string }{ {"empty", ""}, {"ascii", "hello\nworld"}, {"ascii_crlf", "hello\r\nworld\r\n!"}, {"ascii_cr_only", "a\rb\rc"}, {"trailing_newline", "abc\n"}, {"bmp_em_dash", "ab\u2014cd\nef"}, {"bmp_multi", "α\nβ\nγδε\nzz"}, {"supplementary_emoji", "x\U0001F600y\nz"}, // 😀 is 4 UTF-8 bytes, 2 UTF-16 units {"supplementary_at_lineend", "ab\U0001F600\ncd\U0001F60A"}, {"supplementary_only", "\U0001F600\U0001F601\U0001F602"}, {"mixed", "α — \U0001F600\r\nβ\nγ\r"}, {"long_mixed_ws", " \tαβ\n\t\U0001F600 end\n"}, {"zwj_emoji", "\U0001F468\u200D\U0001F4BB\nnext"}, {"only_newlines", "\n\n\r\n\r"}, } texts := make([]string, len(cases)) for i, c := range cases { texts[i] = c.text } refs := runJSReference(t, texts) assert.Equal(t, len(refs), len(cases)) for i, c := range cases { ref := refs[i] t.Run(c.name, func(t *testing.T) { t.Parallel() conv, script := newTestConverters(c.text) for _, tup := range ref { bytePos := core.TextPos(tup.BytePos) expectedLC := lsproto.Position{Line: uint32(tup.Line), Character: uint32(tup.Char)} gotLC := conv.PositionToLineAndCharacter(script, bytePos) assert.Equal(t, gotLC, expectedLC, fmt.Sprintf("PositionToLineAndCharacter(%d) mismatch in %q", bytePos, c.text)) gotPos := conv.LineAndCharacterToPosition(script, expectedLC) assert.Equal(t, gotPos, bytePos, fmt.Sprintf("LineAndCharacterToPosition(%d,%d) mismatch in %q", tup.Line, tup.Char, c.text)) } }) } }