vendor tsgo

2026-07-09 16:50:43 -04:00
parent c06ea2e5a4
commit 98978e4930
5804 changed files with 1556156 additions and 101 deletions
--- a/tools/tsgo/internal/astnav/testmain_test.go
+++ b/tools/tsgo/internal/astnav/testmain_test.go
@@ -0,0 +1,14 @@
+package astnav_test
+
+import (
+	"testing"
+
+	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/testutil/baseline"
+)
+
+func TestMain(m *testing.M) {
+	core.ApplyDebugStackLimit()
+	defer baseline.Track()()
+	m.Run()
+}
--- a/tools/tsgo/internal/astnav/tokens.go
+++ b/tools/tsgo/internal/astnav/tokens.go
@@ -0,0 +1,783 @@
+package astnav
+
+import (
+	"fmt"
+
+	"github.com/microsoft/typescript-go/internal/ast"
+	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/scanner"
+)
+
+func shouldRescanLessThanLessThanToken(s *scanner.Scanner, containingNode *ast.Node, token ast.Kind) bool {
+	return token == ast.KindLessThanLessThanToken && ast.IsJsxChild(containingNode)
+}
+
+func scanNavigationToken(s *scanner.Scanner, containingNode *ast.Node) ast.Kind {
+	token := s.Token()
+	if shouldRescanLessThanLessThanToken(s, containingNode, token) {
+		return s.ReScanJsxToken(true /*allowMultilineJsxText*/)
+	}
+	return token
+}
+
+func GetTouchingPropertyName(sourceFile *ast.SourceFile, position int) *ast.Node {
+	return getTokenAtPosition(sourceFile, position, false /*allowPositionInLeadingTrivia*/, func(node *ast.Node) bool {
+		return ast.IsPropertyNameLiteral(node) || ast.IsKeywordKind(node.Kind) || ast.IsPrivateIdentifier(node)
+	})
+}
+
+func GetTouchingToken(sourceFile *ast.SourceFile, position int) *ast.Node {
+	return getTokenAtPosition(sourceFile, position, false /*allowPositionInLeadingTrivia*/, nil)
+}
+
+func GetTokenAtPosition(sourceFile *ast.SourceFile, position int) *ast.Node {
+	return getTokenAtPosition(sourceFile, position, true /*allowPositionInLeadingTrivia*/, nil)
+}
+
+func getTokenAtPosition(
+	sourceFile *ast.SourceFile,
+	position int,
+	allowPositionInLeadingTrivia bool,
+	includePrecedingTokenAtEndPosition func(node *ast.Node) bool,
+) *ast.Node {
+	// getTokenAtPosition returns a token at the given position in the source file.
+	// The token can be a real node in the AST, or a synthesized token constructed
+	// with information from the scanner. Synthesized tokens are only created when
+	// needed, and they are stored in the source file's token cache such that multiple
+	// calls to getTokenAtPosition with the same position will return the same object
+	// in memory. If there is no token at the given position (possible when
+	// `allowPositionInLeadingTrivia` is false), the lowest node that encloses the
+	// position is returned.
+
+	// `next` tracks the node whose children will be visited on the next iteration.
+	// `prevSubtree` is a node whose end position is equal to the target position,
+	// only if `includePrecedingTokenAtEndPosition` is provided. Once set, the next
+	// iteration of the loop will test the rightmost token of `prevSubtree` to see
+	// if it should be returned.
+	var next, prevSubtree *ast.Node
+	current := sourceFile.AsNode()
+	// `left` tracks the lower boundary of the node/token that could be returned,
+	// and is eventually the scanner's start position, if the scanner is used.
+	left := 0
+	// `nodeAfterLeft` tracks the first node we visit after visiting the node that advances `left`.
+	// When scanning in between nodes for token, we should only scan up to the start of `nodeAfterLeft`.
+	var nodeAfterLeft *ast.Node
+
+	testNode := func(node *ast.Node) int {
+		if node.Kind != ast.KindEndOfFile && node.End() == position &&
+			includePrecedingTokenAtEndPosition != nil && node.Flags&ast.NodeFlagsReparsed == 0 {
+			prevSubtree = node
+		}
+
+		// A node "contains" the position if position < end, except nodes at the file end
+		// treat end as inclusive (there's nowhere else to look). This applies to the EOF
+		// token itself, and to JSDoc nodes reaching EOF (e.g. unterminated JSDoc comments).
+		if node.End() < position || node.End() == position &&
+			node.Kind != ast.KindEndOfFile &&
+			(!ast.IsJSDocKind(node.Kind) || node.End() != sourceFile.EndOfFileToken.End()) {
+			return -1
+		}
+		nodePos := getPosition(node, sourceFile, allowPositionInLeadingTrivia)
+		if nodePos > position {
+			return 1
+		}
+		return 0
+	}
+
+	// We zero in on the node that contains the target position by visiting each
+	// child and JSDoc comment of the current node. Node children are walked in
+	// order, while node lists are binary searched.
+	visitNode := func(node *ast.Node, _ *ast.NodeVisitor) *ast.Node {
+		// We can't abort visiting children, so once a match is found, we set `next`
+		// and do nothing on subsequent visits.
+		if node == nil || node.Flags&ast.NodeFlagsReparsed != 0 {
+			return nil
+		}
+		if nodeAfterLeft == nil {
+			nodeAfterLeft = node
+		}
+		if next == nil {
+			result := testNode(node)
+			switch result {
+			case -1:
+				if !ast.IsJSDocKind(node.Kind) {
+					// We can't move the left boundary into or beyond JSDoc,
+					// because we may end up returning the token after this JSDoc,
+					// constructing it with the scanner, and we need to include
+					// all its leading trivia in its position.
+					left = node.End()
+				}
+				nodeAfterLeft = nil
+			case 0:
+				next = node
+			}
+		}
+		return node
+	}
+
+	visitNodeList := func(nodeList *ast.NodeList, _ *ast.NodeVisitor) *ast.NodeList {
+		if nodeList == nil || len(nodeList.Nodes) == 0 {
+			return nodeList
+		}
+		if nodeAfterLeft == nil {
+			for _, node := range nodeList.Nodes {
+				if node.Flags&ast.NodeFlagsReparsed == 0 {
+					nodeAfterLeft = node
+					break
+				}
+			}
+		}
+		if next == nil {
+			if nodeList.End() == position && includePrecedingTokenAtEndPosition != nil {
+				left = nodeList.End()
+				nodeAfterLeft = nil
+				for i := len(nodeList.Nodes) - 1; i >= 0; i-- {
+					if nodeList.Nodes[i].Flags&ast.NodeFlagsReparsed == 0 {
+						prevSubtree = nodeList.Nodes[i]
+						break
+					}
+				}
+			} else if nodeList.End() <= position {
+				left = nodeList.End()
+				nodeAfterLeft = nil
+			} else if nodeList.Pos() <= position {
+				nodes := nodeList.Nodes
+				index, match := core.BinarySearchUniqueFunc(nodes, func(middle int, node *ast.Node) int {
+					if node.Flags&ast.NodeFlagsReparsed != 0 {
+						return 0
+					}
+					cmp := testNode(node)
+					if cmp < 0 {
+						left = node.End()
+						nodeAfterLeft = nil
+						for i := middle + 1; i < len(nodes); i++ {
+							if nodes[i].Flags&ast.NodeFlagsReparsed == 0 {
+								nodeAfterLeft = nodes[i]
+								break
+							}
+						}
+					}
+					return cmp
+				})
+				if match && nodes[index].Flags&ast.NodeFlagsReparsed != 0 {
+					// filter and search again
+					nodes = core.Filter(nodes, func(node *ast.Node) bool {
+						return node.Flags&ast.NodeFlagsReparsed == 0
+					})
+					index, match = core.BinarySearchUniqueFunc(nodes, func(middle int, node *ast.Node) int {
+						cmp := testNode(node)
+						if cmp < 0 {
+							left = node.End()
+							if middle+1 < len(nodes) {
+								nodeAfterLeft = nodes[middle+1]
+							} else {
+								nodeAfterLeft = nil
+							}
+						}
+						return cmp
+					})
+				}
+				if match {
+					next = nodes[index]
+				}
+			}
+		}
+		return nodeList
+	}
+
+	for {
+		VisitEachChildAndJSDoc(current, sourceFile, visitNode, visitNodeList)
+		// If prevSubtree was set on the last iteration, it ends at the target position.
+		// Check if the rightmost token of prevSubtree should be returned based on the
+		// `includePrecedingTokenAtEndPosition` callback.
+		if prevSubtree != nil {
+			child := FindPrecedingTokenEx(sourceFile, position, prevSubtree, false /*excludeJSDoc*/)
+			if child != nil && child.End() == position && includePrecedingTokenAtEndPosition(child) {
+				// Optimization: includePrecedingTokenAtEndPosition only ever returns true
+				// for real AST nodes, so we don't run the scanner here.
+				return child
+			}
+			prevSubtree = nil
+		}
+
+		// No node was found that contains the target position, so we've gone as deep as
+		// we can in the AST. We've either found a token, or we need to run the scanner
+		// to construct one that isn't stored in the AST.
+		if next == nil {
+			if ast.IsTokenKind(current.Kind) || shouldSkipChild(current) {
+				return current
+			}
+			scanner := scanner.GetScannerForSourceFile(sourceFile, left)
+			end := current.End()
+			// We should only scan up to the start of the next node in the AST after the node ending at position `left`.
+			// It is necessary to enforce this invariant in cases where `position` occurs in between two node/tokens,
+			// such that we would not find a token in the loop below before we reach the next node.
+			// We can fall into this case when `allowPositionInLeadingTrivia` is false and `position` is in a leading trivia,
+			// or when `position` would be in the leading trivia of a node but this node is inside JSDoc:
+			// ```
+			// /**
+			//  * @type {{
+			//  */*$*/ identifier: boolean;
+			//  * }}
+			//  */
+			// ```
+			// The position of marker '$' falls in between the asterisk token and the identifier token, but is not
+			// part of the leading trivia for `identifier`.
+			if nodeAfterLeft != nil {
+				end = nodeAfterLeft.Pos()
+			}
+			for left < end {
+				token := scanNavigationToken(scanner, current)
+				tokenFullStart := scanner.TokenFullStart()
+				tokenStart := core.IfElse(allowPositionInLeadingTrivia, tokenFullStart, scanner.TokenStart())
+				tokenEnd := scanner.TokenEnd()
+				flags := scanner.TokenFlags()
+				if tokenEnd > end {
+					break
+				}
+				if tokenStart <= position && (position < tokenEnd) {
+					if token == ast.KindIdentifier || !ast.IsTokenKind(token) {
+						if ast.IsJSDocKind(current.Kind) {
+							return current
+						}
+						panic(fmt.Sprintf("did not expect %s to have %s in its trivia", current.Kind.String(), token.String()))
+					}
+					return sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current, flags)
+				}
+				if includePrecedingTokenAtEndPosition != nil && tokenEnd == position {
+					prevToken := sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current, flags)
+					if includePrecedingTokenAtEndPosition(prevToken) {
+						return prevToken
+					}
+				}
+				left = tokenEnd
+				scanner.Scan()
+			}
+			return current
+		}
+		current = next
+		left = current.Pos()
+		nodeAfterLeft = nil
+		next = nil
+	}
+}
+
+func getPosition(node *ast.Node, sourceFile *ast.SourceFile, allowPositionInLeadingTrivia bool) int {
+	if allowPositionInLeadingTrivia {
+		return node.Pos()
+	}
+	return scanner.GetTokenPosOfNode(node, sourceFile, true /*includeJSDoc*/)
+}
+
+func findRightmostNode(node *ast.Node) *ast.Node {
+	var next *ast.Node
+	current := node
+	visitNode := func(node *ast.Node, _ *ast.NodeVisitor) *ast.Node {
+		if node != nil {
+			next = node
+		}
+		return node
+	}
+	visitNodes := func(nodeList *ast.NodeList, visitor *ast.NodeVisitor) *ast.NodeList {
+		if nodeList != nil {
+			if rightmost := ast.FindLastVisibleNode(nodeList.Nodes); rightmost != nil {
+				next = rightmost
+			}
+		}
+		return nodeList
+	}
+	visitor := getNodeVisitor(visitNode, visitNodes)
+
+	for {
+		current.VisitEachChild(visitor)
+		if next == nil {
+			return current
+		}
+		current = next
+		next = nil
+	}
+}
+
+func VisitEachChildAndJSDoc(
+	node *ast.Node,
+	sourceFile *ast.SourceFile,
+	visitNode func(*ast.Node, *ast.NodeVisitor) *ast.Node,
+	visitNodes func(*ast.NodeList, *ast.NodeVisitor) *ast.NodeList,
+) {
+	visitor := getNodeVisitor(visitNode, visitNodes)
+	for _, jsdoc := range node.JSDoc(sourceFile) {
+		if visitor.Hooks.VisitNode != nil {
+			visitor.Hooks.VisitNode(jsdoc, visitor)
+		} else {
+			visitor.VisitNode(jsdoc)
+		}
+	}
+	node.VisitEachChild(visitor)
+}
+
+const (
+	comparisonLessThan    = -1
+	comparisonEqualTo     = 0
+	comparisonGreaterThan = 1
+)
+
+// Finds the leftmost token satisfying `position < token.End()`.
+// If the leftmost token satisfying `position < token.End()` is invalid, or if position
+// is in the trivia of that leftmost token,
+// we will find the rightmost valid token with `token.End() <= position`.
+func FindPrecedingToken(sourceFile *ast.SourceFile, position int) *ast.Node {
+	return FindPrecedingTokenEx(sourceFile, position, nil, false)
+}
+
+func FindPrecedingTokenEx(sourceFile *ast.SourceFile, position int, startNode *ast.Node, excludeJSDoc bool) *ast.Node {
+	var find func(node *ast.Node) *ast.Node
+	find = func(n *ast.Node) *ast.Node {
+		if ast.IsNonWhitespaceToken(n) && n.Kind != ast.KindEndOfFile {
+			return n
+		}
+
+		// `foundChild` is the leftmost node that contains the target position.
+		// `prevChild` is the last visited child of the current node.
+		var foundChild, prevChild *ast.Node
+		visitNode := func(node *ast.Node, _ *ast.NodeVisitor) *ast.Node {
+			// skip synthesized nodes (that will exist now because of jsdoc handling)
+			if node == nil || node.Flags&ast.NodeFlagsReparsed != 0 {
+				return node
+			}
+			if foundChild != nil { // We cannot abort visiting children, so once the desired child is found, we do nothing.
+				return node
+			}
+			if position < node.End() && (prevChild == nil || prevChild.End() <= position) {
+				foundChild = node
+			} else {
+				prevChild = node
+			}
+			return node
+		}
+		visitNodes := func(nodeList *ast.NodeList, _ *ast.NodeVisitor) *ast.NodeList {
+			if foundChild != nil {
+				return nodeList
+			}
+			if nodeList != nil && len(nodeList.Nodes) > 0 {
+				nodes := nodeList.Nodes
+				index, match := core.BinarySearchUniqueFunc(nodes, func(middle int, _ *ast.Node) int {
+					// synthetic jsdoc nodes should have jsdocNode.End() <= n.Pos()
+					if nodes[middle].Flags&ast.NodeFlagsReparsed != 0 {
+						return comparisonLessThan
+					}
+					if position < nodes[middle].End() {
+						if middle == 0 || position >= nodes[middle-1].End() {
+							return comparisonEqualTo
+						}
+						return comparisonGreaterThan
+					}
+					return comparisonLessThan
+				})
+
+				if match {
+					foundChild = nodes[index]
+				}
+
+				validLookupIndex := core.IfElse(match, index-1, len(nodes)-1)
+				for i := validLookupIndex; i >= 0; i-- {
+					if nodes[i].Flags&ast.NodeFlagsReparsed != 0 {
+						continue
+					}
+					if prevChild == nil {
+						prevChild = nodes[i]
+					}
+				}
+			}
+			return nodeList
+		}
+		VisitEachChildAndJSDoc(n, sourceFile, visitNode, visitNodes)
+
+		if foundChild != nil {
+			// Note that the span of a node's tokens is [getStartOfNode(node, ...), node.end).
+			// Given that `position < child.end` and child has constituent tokens, we distinguish these cases:
+			// 1) `position` precedes `child`'s tokens or `child` has no tokens (ie: in a comment or whitespace preceding `child`):
+			// we need to find the last token in a previous child node or child tokens.
+			// 2) `position` is within the same span: we recurse on `child`.
+			start := GetStartOfNode(foundChild, sourceFile, !excludeJSDoc /*includeJSDoc*/)
+			lookInPreviousChild := start >= position || // cursor in the leading trivia or preceding tokens
+				!isValidPrecedingNode(foundChild, sourceFile)
+			if lookInPreviousChild {
+				if position >= foundChild.Pos() {
+					// Find jsdoc preceding the foundChild.
+					var jsDoc *ast.Node
+					nodeJSDoc := n.JSDoc(sourceFile)
+					for i := len(nodeJSDoc) - 1; i >= 0; i-- {
+						if nodeJSDoc[i].Pos() >= foundChild.Pos() {
+							jsDoc = nodeJSDoc[i]
+							break
+						}
+					}
+					if jsDoc != nil {
+						if !excludeJSDoc && position < jsDoc.End() {
+							return find(jsDoc)
+						} else {
+							return findRightmostValidToken(jsDoc.End(), sourceFile, n, position, excludeJSDoc)
+						}
+					}
+					return findRightmostValidToken(foundChild.Pos(), sourceFile, n, -1 /*position*/, excludeJSDoc)
+				} else { // Answer is in tokens between two visited children.
+					return findRightmostValidToken(foundChild.Pos(), sourceFile, n, position, excludeJSDoc)
+				}
+			} else {
+				// position is in [foundChild.getStart(), foundChild.End): recur.
+				return find(foundChild)
+			}
+		}
+
+		// We have two cases here: either the position is at the end of the file,
+		// or the desired token is in the unvisited trailing tokens of the current node.
+		if position >= n.End() {
+			return findRightmostValidToken(n.End(), sourceFile, n, -1 /*position*/, excludeJSDoc)
+		} else {
+			return findRightmostValidToken(n.End(), sourceFile, n, position, excludeJSDoc)
+		}
+	}
+
+	var node *ast.Node
+	if startNode != nil {
+		node = startNode
+	} else {
+		node = sourceFile.AsNode()
+	}
+	result := find(node)
+	if result != nil && ast.IsWhitespaceOnlyJsxText(result) {
+		panic("Expected result to be a non-whitespace token.")
+	}
+	return result
+}
+
+func isValidPrecedingNode(node *ast.Node, sourceFile *ast.SourceFile) bool {
+	if node.Kind == ast.KindEndOfFile {
+		return len(node.JSDoc(sourceFile)) > 0
+	}
+	start := GetStartOfNode(node, sourceFile, false /*includeJSDoc*/)
+	width := node.End() - start
+	return !(ast.IsWhitespaceOnlyJsxText(node) || width == 0)
+}
+
+func GetStartOfNode(node *ast.Node, file *ast.SourceFile, includeJSDoc bool) int {
+	return scanner.GetTokenPosOfNode(node, file, includeJSDoc)
+}
+
+// Looks for rightmost valid token in the range [startPos, endPos).
+// If position is >= 0, looks for rightmost valid token that precedes or touches that position.
+func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingNode *ast.Node, position int, excludeJSDoc bool) *ast.Node {
+	if position == -1 {
+		position = containingNode.End()
+	}
+	var find func(n *ast.Node, endPos int) *ast.Node
+	find = func(n *ast.Node, endPos int) *ast.Node {
+		if n == nil {
+			return nil
+		}
+		if ast.IsNonWhitespaceToken(n) {
+			return n
+		}
+
+		var rightmostValidNode *ast.Node
+		rightmostVisitedNodes := make([]*ast.Node, 0, 1) // Nodes after the last valid node.
+		hasChildren := false
+		shouldVisitNode := func(node *ast.Node) bool {
+			// Node is synthetic or out of the desired range: don't visit it.
+			return !(node.Flags&ast.NodeFlagsReparsed != 0 ||
+				node.End() > endPos || GetStartOfNode(node, sourceFile, !excludeJSDoc /*includeJSDoc*/) >= position)
+		}
+		visitNode := func(node *ast.Node, _ *ast.NodeVisitor) *ast.Node {
+			if node == nil || node.Flags&ast.NodeFlagsReparsed != 0 {
+				return node
+			}
+			hasChildren = true
+			if !shouldVisitNode(node) {
+				return node
+			}
+			rightmostVisitedNodes = append(rightmostVisitedNodes, node)
+			if isValidPrecedingNode(node, sourceFile) {
+				rightmostValidNode = node
+				rightmostVisitedNodes = rightmostVisitedNodes[:0]
+			}
+			return node
+		}
+		visitNodes := func(nodeList *ast.NodeList, _ *ast.NodeVisitor) *ast.NodeList {
+			if nodeList != nil && len(nodeList.Nodes) > 0 {
+				hasChildren = true
+				index, _ := core.BinarySearchUniqueFunc(nodeList.Nodes, func(middle int, node *ast.Node) int {
+					if node.End() > endPos {
+						return comparisonGreaterThan
+					}
+					return comparisonLessThan
+				})
+				validIndex := -1
+				for i := index - 1; i >= 0; i-- {
+					if !shouldVisitNode(nodeList.Nodes[i]) {
+						continue
+					}
+					if isValidPrecedingNode(nodeList.Nodes[i], sourceFile) {
+						validIndex = i
+						rightmostValidNode = nodeList.Nodes[i]
+						break
+					}
+				}
+				for i := validIndex + 1; i < index; i++ {
+					if !shouldVisitNode(nodeList.Nodes[i]) {
+						continue
+					}
+					rightmostVisitedNodes = append(rightmostVisitedNodes, nodeList.Nodes[i])
+				}
+			}
+			return nodeList
+		}
+		VisitEachChildAndJSDoc(n, sourceFile, visitNode, visitNodes)
+
+		// Three cases:
+		// 1. The answer is a token of `rightmostValidNode`.
+		// 2. The answer is one of the unvisited tokens that occur after the rightmost valid node.
+		// 3. The current node is a childless, token-less node. The answer is the current node.
+
+		// Case 2: Look at unvisited trailing tokens that occur in between the rightmost visited nodes.
+		if !shouldSkipChild(n) { // JSDoc nodes don't include trivia tokens as children.
+			var startPos int
+			if rightmostValidNode != nil {
+				startPos = rightmostValidNode.End()
+			} else {
+				startPos = n.Pos()
+			}
+			scanner := scanner.GetScannerForSourceFile(sourceFile, startPos)
+			var tokens []*ast.Node
+			for _, visitedNode := range rightmostVisitedNodes {
+				// Trailing tokens that occur before this node.
+				for startPos < min(visitedNode.Pos(), position) {
+					token := scanNavigationToken(scanner, n)
+					tokenStart := scanner.TokenStart()
+					if tokenStart >= position {
+						break
+					}
+					tokenFullStart := scanner.TokenFullStart()
+					tokenEnd := scanner.TokenEnd()
+					startPos = tokenEnd
+					flags := scanner.TokenFlags()
+					tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags))
+					scanner.Scan()
+				}
+				startPos = visitedNode.End()
+				scanner.ResetPos(startPos)
+				scanner.Scan()
+			}
+			// Trailing tokens after last visited node.
+			for startPos < min(endPos, position) {
+				token := scanNavigationToken(scanner, n)
+				tokenStart := scanner.TokenStart()
+				if tokenStart >= position {
+					break
+				}
+				tokenFullStart := scanner.TokenFullStart()
+				tokenEnd := scanner.TokenEnd()
+				startPos = tokenEnd
+				flags := scanner.TokenFlags()
+				tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags))
+				scanner.Scan()
+			}
+
+			lastToken := len(tokens) - 1
+			// Find preceding valid token.
+			for i := lastToken; i >= 0; i-- {
+				if !ast.IsWhitespaceOnlyJsxText(tokens[i]) {
+					return tokens[i]
+				}
+			}
+		}
+
+		// Case 3: childless node.
+		if !hasChildren {
+			if n != containingNode {
+				return n
+			}
+			return nil
+		}
+		// Case 1: recur on rightmostValidNode.
+		if rightmostValidNode != nil {
+			endPos = rightmostValidNode.End()
+		}
+		return find(rightmostValidNode, endPos)
+	}
+
+	return find(containingNode, endPos)
+}
+
+func FindNextToken(previousToken *ast.Node, parent *ast.Node, file *ast.SourceFile) *ast.Node {
+	var find func(n *ast.Node) *ast.Node
+	find = func(n *ast.Node) *ast.Node {
+		if ast.IsTokenKind(n.Kind) && n.Pos() == previousToken.End() {
+			// this is token that starts at the end of previous token - return it
+			return n
+		}
+		// Node that contains `previousToken` or occurs immediately after it.
+		var foundNode *ast.Node
+		visitNode := func(node *ast.Node, _ *ast.NodeVisitor) *ast.Node {
+			if node != nil && node.Flags&ast.NodeFlagsReparsed == 0 &&
+				node.Pos() <= previousToken.End() && node.End() > previousToken.End() {
+				foundNode = node
+			}
+			return node
+		}
+		visitNodes := func(nodeList *ast.NodeList, _ *ast.NodeVisitor) *ast.NodeList {
+			if nodeList != nil && len(nodeList.Nodes) > 0 && foundNode == nil {
+				nodes := nodeList.Nodes
+				index, match := core.BinarySearchUniqueFunc(nodes, func(_ int, node *ast.Node) int {
+					if node.Flags&ast.NodeFlagsReparsed != 0 {
+						return comparisonLessThan
+					}
+					if node.Pos() > previousToken.End() {
+						return comparisonGreaterThan
+					}
+					if node.End() <= previousToken.Pos() {
+						return comparisonLessThan
+					}
+					return comparisonEqualTo
+				})
+				if match {
+					foundNode = nodes[index]
+				}
+			}
+			return nodeList
+		}
+		VisitEachChildAndJSDoc(n, file, visitNode, visitNodes)
+		// Cases:
+		// 1. no answer exists
+		// 2. answer is an unvisited token
+		// 3. answer is in the visited found node
+
+		// Case 3: look for the next token inside the found node.
+		if foundNode != nil {
+			return find(foundNode)
+		}
+		startPos := previousToken.End()
+		// Case 2: look for the next token directly.
+		if startPos >= n.Pos() && startPos < n.End() {
+			scanner := scanner.GetScannerForSourceFile(file, startPos)
+			token := scanner.Token()
+			tokenFullStart := scanner.TokenFullStart()
+			tokenEnd := scanner.TokenEnd()
+			flags := scanner.TokenFlags()
+			// Use tokenFullStart (which includes leading trivia) to match TS's
+			// findNextToken behavior where `n.pos === previousToken.end` is checked
+			// (TS's pos includes trivia, same as Go's Pos()/tokenFullStart).
+			if tokenFullStart == previousToken.End() {
+				return file.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags)
+			}
+			panic(fmt.Sprintf("Expected to find next token at %d, got token %s at %d", previousToken.End(), token, tokenFullStart))
+		}
+		// Case 3: no answer.
+		return nil
+	}
+	return find(parent)
+}
+
+func getNodeVisitor(
+	visitNode func(*ast.Node, *ast.NodeVisitor) *ast.Node,
+	visitNodes func(*ast.NodeList, *ast.NodeVisitor) *ast.NodeList,
+) *ast.NodeVisitor {
+	var wrappedVisitNode func(*ast.Node, *ast.NodeVisitor) *ast.Node
+	var wrappedVisitNodes func(*ast.NodeList, *ast.NodeVisitor) *ast.NodeList
+	if visitNode != nil {
+		wrappedVisitNode = func(n *ast.Node, v *ast.NodeVisitor) *ast.Node {
+			if ast.IsJSDocSingleCommentNodeComment(n) {
+				return n
+			}
+			return visitNode(n, v)
+		}
+	}
+
+	if visitNodes != nil {
+		wrappedVisitNodes = func(n *ast.NodeList, v *ast.NodeVisitor) *ast.NodeList {
+			if ast.IsJSDocSingleCommentNodeList(n) {
+				return n
+			}
+			return visitNodes(n, v)
+		}
+	}
+
+	return ast.NewNodeVisitor(core.Identity, nil, ast.NodeVisitorHooks{
+		VisitNode:  wrappedVisitNode,
+		VisitToken: wrappedVisitNode,
+		VisitNodes: wrappedVisitNodes,
+		VisitModifiers: func(modifiers *ast.ModifierList, visitor *ast.NodeVisitor) *ast.ModifierList {
+			if modifiers != nil {
+				wrappedVisitNodes(&modifiers.NodeList, visitor)
+			}
+			return modifiers
+		},
+	})
+}
+
+func shouldSkipChild(node *ast.Node) bool {
+	return node.Kind == ast.KindJSDoc ||
+		node.Kind == ast.KindJSDocText ||
+		node.Kind == ast.KindJSDocTypeLiteral ||
+		node.Kind == ast.KindJSDocSignature ||
+		ast.IsJSDocLinkLike(node) ||
+		ast.IsJSDocTag(node)
+}
+
+// FindChildOfKind searches for a child node or token of the specified kind within a containing node.
+// This function scans through both AST nodes and intervening tokens to find the first match.
+func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.SourceFile) *ast.Node {
+	lastNodePos := containingNode.Pos()
+	scan := scanner.GetScannerForSourceFile(sourceFile, lastNodePos)
+
+	var foundChild *ast.Node
+	visitNode := func(node *ast.Node) bool {
+		if node == nil || node.Flags&ast.NodeFlagsReparsed != 0 {
+			return false
+		}
+		// Look for child in preceding tokens.
+		startPos := lastNodePos
+		for startPos < node.Pos() {
+			tokenKind := scan.Token()
+			tokenEnd := scan.TokenEnd()
+			if tokenKind == kind {
+				tokenFullStart := scan.TokenFullStart()
+				flags := scan.TokenFlags()
+				foundChild = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode, flags)
+				return true
+			}
+			startPos = tokenEnd
+			scan.Scan()
+		}
+
+		if node.Kind == kind {
+			foundChild = node
+			return true
+		}
+
+		lastNodePos = node.End()
+		scan.ResetPos(lastNodePos)
+		return false
+	}
+
+	ast.ForEachChildAndJSDoc(containingNode, sourceFile, visitNode)
+
+	if foundChild != nil {
+		return foundChild
+	}
+
+	// Look for child in trailing tokens.
+	startPos := lastNodePos
+	for startPos < containingNode.End() {
+		tokenKind := scan.Token()
+		tokenEnd := scan.TokenEnd()
+		if tokenKind == kind {
+			tokenFullStart := scan.TokenFullStart()
+			flags := scan.TokenFlags()
+			token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode, flags)
+			return token
+		}
+		startPos = tokenEnd
+		scan.Scan()
+	}
+	return nil
+}
--- a/tools/tsgo/internal/astnav/tokens_test.go
+++ b/tools/tsgo/internal/astnav/tokens_test.go
@@ -0,0 +1,626 @@
+package astnav_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/microsoft/typescript-go/internal/ast"
+	"github.com/microsoft/typescript-go/internal/astnav"
+	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/parser"
+	"github.com/microsoft/typescript-go/internal/repo"
+	"github.com/microsoft/typescript-go/internal/testutil/baseline"
+	"github.com/microsoft/typescript-go/internal/testutil/jstest"
+	"gotest.tools/v3/assert"
+)
+
+var testFiles = []string{
+	filepath.Join(repo.TypeScriptSubmodulePath(), "src/services/mapCode.ts"),
+}
+
+func TestGetTokenAtPosition(t *testing.T) {
+	t.Parallel()
+	repo.SkipIfNoTypeScriptSubmodule(t)
+	jstest.SkipIfNoNodeJS(t)
+
+	t.Run("baseline", func(t *testing.T) {
+		t.Parallel()
+		baselineTokens(
+			t,
+			"GetTokenAtPosition",
+			false, /*includeEOF*/
+			func(fileText string, positions []int) []*tokenInfo {
+				return tsGetTokensAtPositions(t, fileText, positions)
+			},
+			func(file *ast.SourceFile, pos int) *tokenInfo {
+				return toTokenInfo(astnav.GetTokenAtPosition(file, pos))
+			},
+		)
+	})
+
+	t.Run("go baseline json", func(t *testing.T) {
+		t.Parallel()
+		baselineGoTokensJSON(t, "GetTokenAtPosition", func(file *ast.SourceFile, pos int) *tokenInfo {
+			return toTokenInfo(astnav.GetTokenAtPosition(file, pos))
+		})
+	})
+
+	t.Run("JSDoc type assertion", func(t *testing.T) {
+		t.Parallel()
+		fileText := `function foo(x) {
+    const s = /**@type {string}*/(x)
+}`
+		file := parser.ParseSourceFile(ast.SourceFileParseOptions{
+			FileName: "/test.js",
+			Path:     "/test.js",
+		}, fileText, core.ScriptKindJS)
+
+		// Position of 'x' inside the parenthesized expression (position 52)
+		position := 52
+
+		// This should not panic - it previously panicked with:
+		// "did not expect KindParenthesizedExpression to have KindIdentifier in its trivia"
+		token := astnav.GetTouchingPropertyName(file, position)
+		if token == nil {
+			t.Fatal("Expected to get a token, got nil")
+		}
+
+		// The function may return either the identifier itself or the containing
+		// parenthesized expression, depending on how the AST is structured
+		if token.Kind != ast.KindIdentifier && token.Kind != ast.KindParenthesizedExpression {
+			t.Errorf("Expected identifier or parenthesized expression, got %s", token.Kind)
+		}
+	})
+
+	t.Run("JSDoc type assertion with comment", func(t *testing.T) {
+		t.Parallel()
+		// Exact code from the issue report
+		fileText := `function foo(x) {
+    const s = /**@type {string}*/(x)  // Go-to-definition on x causes panic
+}`
+		file := parser.ParseSourceFile(ast.SourceFileParseOptions{
+			FileName: "/test.js",
+			Path:     "/test.js",
+		}, fileText, core.ScriptKindJS)
+
+		// Find position of 'x' in the type assertion
+		xPos := 52 // Position of 'x' in (x)
+
+		// This should not panic
+		token := astnav.GetTouchingPropertyName(file, xPos)
+		assert.Assert(t, token != nil, "Expected to get a token")
+	})
+
+	t.Run("pointer equality", func(t *testing.T) {
+		t.Parallel()
+		fileText := `
+			function foo() {
+				return 0;
+			}
+		`
+		file := parser.ParseSourceFile(ast.SourceFileParseOptions{
+			FileName: "/file.ts",
+			Path:     "/file.ts",
+		}, fileText, core.ScriptKindTS)
+		assert.Equal(t, astnav.GetTokenAtPosition(file, 0), astnav.GetTokenAtPosition(file, 0))
+	})
+}
+
+func TestGetTouchingPropertyName(t *testing.T) {
+	t.Parallel()
+	jstest.SkipIfNoNodeJS(t)
+	repo.SkipIfNoTypeScriptSubmodule(t)
+
+	baselineTokens(
+		t,
+		"GetTouchingPropertyName",
+		false, /*includeEOF*/
+		func(fileText string, positions []int) []*tokenInfo {
+			return tsGetTouchingPropertyName(t, fileText, positions)
+		},
+		func(file *ast.SourceFile, pos int) *tokenInfo {
+			return toTokenInfo(astnav.GetTouchingPropertyName(file, pos))
+		},
+	)
+
+	t.Run("go baseline json", func(t *testing.T) {
+		t.Parallel()
+		baselineGoTokensJSON(t, "GetTouchingPropertyName", func(file *ast.SourceFile, pos int) *tokenInfo {
+			return toTokenInfo(astnav.GetTouchingPropertyName(file, pos))
+		})
+	})
+}
+
+func baselineTokens(t *testing.T, testName string, includeEOF bool, getTSTokens func(fileText string, positions []int) []*tokenInfo, getGoToken func(file *ast.SourceFile, pos int) *tokenInfo) {
+	for _, fileName := range testFiles {
+		t.Run(filepath.Base(fileName), func(t *testing.T) {
+			t.Parallel()
+			fileText, err := os.ReadFile(fileName)
+			assert.NilError(t, err)
+
+			positions := make([]int, len(fileText)+core.IfElse(includeEOF, 1, 0))
+			for i := range positions {
+				positions[i] = i
+			}
+			tsTokens := getTSTokens(string(fileText), positions)
+			file := parser.ParseSourceFile(ast.SourceFileParseOptions{
+				FileName: "/file.ts",
+				Path:     "/file.ts",
+			}, string(fileText), core.ScriptKindTS)
+
+			var output strings.Builder
+			currentRange := core.NewTextRange(0, 0)
+			currentDiff := tokenDiff{}
+
+			for pos, tsToken := range tsTokens {
+				goToken := getGoToken(file, pos)
+				diff := tokenDiff{goToken: goToken, tsToken: tsToken}
+
+				if !diffEqual(currentDiff, diff) {
+					if !tokensEqual(currentDiff.goToken, currentDiff.tsToken) {
+						writeRangeDiff(&output, file, currentDiff, currentRange, pos)
+					}
+					currentDiff = diff
+					currentRange = core.NewTextRange(pos, pos)
+				}
+				currentRange = currentRange.WithEnd(pos)
+			}
+
+			if !tokensEqual(currentDiff.goToken, currentDiff.tsToken) {
+				writeRangeDiff(&output, file, currentDiff, currentRange, len(tsTokens)-1)
+			}
+
+			baseline.Run(
+				t,
+				fmt.Sprintf("%s.%s.baseline.txt", testName, filepath.Base(fileName)),
+				core.IfElse(output.Len() > 0, output.String(), baseline.NoContent),
+				baseline.Options{
+					Subfolder: "astnav",
+				},
+			)
+		})
+	}
+}
+
+type tokenRun struct {
+	StartPos int    `json:"startPos"`
+	EndPos   int    `json:"endPos"`
+	Kind     string `json:"kind"`
+	NodePos  int    `json:"nodePos"`
+	NodeEnd  int    `json:"nodeEnd"`
+}
+
+func baselineGoTokensJSON(t *testing.T, testName string, getGoToken func(file *ast.SourceFile, pos int) *tokenInfo) {
+	for _, fileName := range testFiles {
+		t.Run(filepath.Base(fileName), func(t *testing.T) {
+			t.Parallel()
+			fileText, err := os.ReadFile(fileName)
+			assert.NilError(t, err)
+
+			file := parser.ParseSourceFile(ast.SourceFileParseOptions{
+				FileName: "/file.ts",
+				Path:     "/file.ts",
+			}, string(fileText), core.ScriptKindTS)
+
+			maxPos := len(fileText)
+			var runs []tokenRun
+			var current *tokenRun
+
+			for pos := range maxPos {
+				token := getGoToken(file, pos)
+				if current != nil && token != nil && current.Kind == token.Kind && current.NodePos == token.Pos && current.NodeEnd == token.End {
+					current.EndPos = pos
+				} else {
+					if current != nil {
+						runs = append(runs, *current)
+					}
+					if token != nil {
+						current = &tokenRun{
+							StartPos: pos,
+							EndPos:   pos,
+							Kind:     token.Kind,
+							NodePos:  token.Pos,
+							NodeEnd:  token.End,
+						}
+					} else {
+						current = nil
+					}
+				}
+			}
+			if current != nil {
+				runs = append(runs, *current)
+			}
+
+			output := core.Must(core.StringifyJson(runs, "", "  "))
+
+			baseline.Run(
+				t,
+				fmt.Sprintf("%s.%s.baseline.json", testName, filepath.Base(fileName)),
+				output,
+				baseline.Options{
+					Subfolder: "astnav",
+				},
+			)
+		})
+	}
+}
+
+type tokenDiff struct {
+	goToken *tokenInfo
+	tsToken *tokenInfo
+}
+
+type tokenInfo struct {
+	Kind string `json:"kind"`
+	Pos  int    `json:"pos"`
+	End  int    `json:"end"`
+}
+
+func toTokenInfo(node *ast.Node) *tokenInfo {
+	if node == nil {
+		return nil
+	}
+	kind := strings.Replace(node.Kind.String(), "Kind", "", 1)
+	switch kind {
+	case "EndOfFile":
+		kind = "EndOfFileToken"
+	}
+	return &tokenInfo{
+		Kind: kind,
+		Pos:  node.Pos(),
+		End:  node.End(),
+	}
+}
+
+func diffEqual(a, b tokenDiff) bool {
+	return tokensEqual(a.goToken, b.goToken) && tokensEqual(a.tsToken, b.tsToken)
+}
+
+func tokensEqual(t1, t2 *tokenInfo) bool {
+	if t1 == nil || t2 == nil {
+		return t1 == t2
+	}
+	return *t1 == *t2
+}
+
+func tsGetTokensAtPositions(t testing.TB, fileText string, positions []int) []*tokenInfo {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, "file.ts"), []byte(fileText), 0o644)
+	assert.NilError(t, err)
+
+	err = os.WriteFile(filepath.Join(dir, "positions.json"), []byte(core.Must(core.StringifyJson(positions, "", ""))), 0o644)
+	assert.NilError(t, err)
+
+	script := `
+		import fs from "fs";
+		export default (ts) => {
+			const positions = JSON.parse(fs.readFileSync("positions.json", "utf8"));
+			const fileText = fs.readFileSync("file.ts", "utf8");
+			const file = ts.createSourceFile(
+				"file.ts",
+				fileText,
+				{ languageVersion: ts.ScriptTarget.Latest, jsDocParsingMode: ts.JSDocParsingMode.ParseAll },
+				/*setParentNodes*/ true
+			);
+			return positions.map(position => {
+				let token = ts.getTokenAtPosition(file, position);
+				if (token.kind === ts.SyntaxKind.SyntaxList) {
+					token = token.parent;
+				}
+				return {
+					kind: ts.Debug.formatSyntaxKind(token.kind),
+					pos: token.pos,
+					end: token.end,
+				};
+			});
+		};`
+
+	info, err := jstest.EvalNodeScriptWithTS[[]*tokenInfo](t, script, dir, "")
+	assert.NilError(t, err)
+	return info
+}
+
+func tsGetTouchingPropertyName(t testing.TB, fileText string, positions []int) []*tokenInfo {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, "file.ts"), []byte(fileText), 0o644)
+	assert.NilError(t, err)
+
+	err = os.WriteFile(filepath.Join(dir, "positions.json"), []byte(core.Must(core.StringifyJson(positions, "", ""))), 0o644)
+	assert.NilError(t, err)
+
+	script := `
+		import fs from "fs";
+		export default (ts) => {
+			const positions = JSON.parse(fs.readFileSync("positions.json", "utf8"));
+			const fileText = fs.readFileSync("file.ts", "utf8");
+			const file = ts.createSourceFile(
+				"file.ts",
+				fileText,
+				{ languageVersion: ts.ScriptTarget.Latest, jsDocParsingMode: ts.JSDocParsingMode.ParseAll },
+				/*setParentNodes*/ true
+			);
+			return positions.map(position => {
+				let token = ts.getTouchingPropertyName(file, position);
+				if (token.kind === ts.SyntaxKind.SyntaxList) {
+					token = token.parent;
+				}
+				return {
+					kind: ts.Debug.formatSyntaxKind(token.kind),
+					pos: token.pos,
+					end: token.end,
+				};
+			});
+		};`
+
+	info, err := jstest.EvalNodeScriptWithTS[[]*tokenInfo](t, script, dir, "")
+	assert.NilError(t, err)
+	return info
+}
+
+func writeRangeDiff(output *strings.Builder, file *ast.SourceFile, diff tokenDiff, rng core.TextRange, position int) {
+	lines := file.ECMALineMap()
+
+	tsTokenPos := position
+	goTokenPos := position
+	tsTokenEnd := position
+	goTokenEnd := position
+	if diff.tsToken != nil {
+		tsTokenPos = diff.tsToken.Pos
+		tsTokenEnd = diff.tsToken.End
+	}
+	if diff.goToken != nil {
+		goTokenPos = diff.goToken.Pos
+		goTokenEnd = diff.goToken.End
+	}
+	tsStartLine, _ := core.PositionToLineAndByteOffset(tsTokenPos, lines)
+	tsEndLine, _ := core.PositionToLineAndByteOffset(tsTokenEnd, lines)
+	goStartLine, _ := core.PositionToLineAndByteOffset(goTokenPos, lines)
+	goEndLine, _ := core.PositionToLineAndByteOffset(goTokenEnd, lines)
+
+	contextLines := 2
+	startLine := min(tsStartLine, goStartLine)
+	endLine := max(tsEndLine, goEndLine)
+	markerLines := []int{tsStartLine, tsEndLine, goStartLine, goEndLine}
+	slices.Sort(markerLines)
+	contextStart := max(0, startLine-contextLines)
+	contextEnd := min(len(lines)-1, endLine+contextLines)
+	digits := len(strconv.Itoa(contextEnd))
+
+	shouldTruncate := func(line int) (result bool, skipTo int) {
+		index, _ := slices.BinarySearch(markerLines, line)
+		if index == 0 || index == len(markerLines) {
+			return false, 0
+		}
+		low := markerLines[index-1]
+		high := markerLines[index]
+		if line-low > 5 && high-line > 5 {
+			return true, high - 5
+		}
+		return false, 0
+	}
+
+	if output.Len() > 0 {
+		output.WriteString("\n\n")
+	}
+
+	output.WriteString(fmt.Sprintf("〚Positions: [%d, %d]〛\n", rng.Pos(), rng.End()))
+	if diff.tsToken != nil {
+		output.WriteString(fmt.Sprintf("【TS: %s [%d, %d)】\n", diff.tsToken.Kind, tsTokenPos, tsTokenEnd))
+	} else {
+		output.WriteString("【TS: nil】\n")
+	}
+	if diff.goToken != nil {
+		output.WriteString(fmt.Sprintf("《Go: %s [%d, %d)》\n", diff.goToken.Kind, goTokenPos, goTokenEnd))
+	} else {
+		output.WriteString("《Go: nil》\n")
+	}
+	for line := contextStart; line <= contextEnd; line++ {
+		if truncate, skipTo := shouldTruncate(line); truncate {
+			output.WriteString(fmt.Sprintf("%s │........ %d lines omitted ........\n", strings.Repeat(" ", digits), skipTo-line+1))
+			line = skipTo
+		}
+		output.WriteString(fmt.Sprintf("%*d │", digits, line+1))
+		end := len(file.Text()) + 1
+		if line < len(lines)-1 {
+			end = int(lines[line+1])
+		}
+		for pos := int(lines[line]); pos < end; pos++ {
+			if pos == rng.End()+1 {
+				output.WriteString("〛")
+			}
+			if diff.tsToken != nil && pos == tsTokenEnd {
+				output.WriteString("】")
+			}
+			if diff.goToken != nil && pos == goTokenEnd {
+				output.WriteString("》")
+			}
+
+			if diff.goToken != nil && pos == goTokenPos {
+				output.WriteString("《")
+			}
+			if diff.tsToken != nil && pos == tsTokenPos {
+				output.WriteString("【")
+			}
+			if pos == rng.Pos() {
+				output.WriteString("〚")
+			}
+
+			if pos < len(file.Text()) {
+				output.WriteByte(file.Text()[pos])
+			}
+		}
+	}
+}
+
+func TestFindPrecedingToken(t *testing.T) {
+	t.Parallel()
+	repo.SkipIfNoTypeScriptSubmodule(t)
+	jstest.SkipIfNoNodeJS(t)
+
+	t.Run("baseline", func(t *testing.T) {
+		t.Parallel()
+		baselineTokens(
+			t,
+			"FindPrecedingToken",
+			true, /*includeEOF*/
+			func(fileText string, positions []int) []*tokenInfo {
+				return tsFindPrecedingTokens(t, fileText, positions)
+			},
+			func(file *ast.SourceFile, pos int) *tokenInfo {
+				return toTokenInfo(astnav.FindPrecedingToken(file, pos))
+			},
+		)
+	})
+
+	t.Run("go baseline json", func(t *testing.T) {
+		t.Parallel()
+		baselineGoTokensJSON(t, "FindPrecedingToken", func(file *ast.SourceFile, pos int) *tokenInfo {
+			return toTokenInfo(astnav.FindPrecedingToken(file, pos))
+		})
+	})
+}
+
+func TestFindNextToken(t *testing.T) {
+	t.Parallel()
+	repo.SkipIfNoTypeScriptSubmodule(t)
+
+	t.Run("go baseline json", func(t *testing.T) {
+		t.Parallel()
+		baselineGoTokensJSON(t, "FindNextToken", func(file *ast.SourceFile, pos int) (result *tokenInfo) {
+			// FindNextToken panics (like Go's assert) when the scanner finds trivia between
+			// previousToken.End() and the next syntactic token. Catch those to avoid crashing
+			// the baseline generator; those positions will be absent from the baseline.
+			defer func() {
+				if r := recover(); r != nil {
+					result = nil
+				}
+			}()
+			token := astnav.GetTokenAtPosition(file, pos)
+			next := astnav.FindNextToken(token, file.AsNode(), file)
+			return toTokenInfo(next)
+		})
+	})
+}
+
+func TestUnitFindPrecedingToken(t *testing.T) {
+	t.Parallel()
+	testCases := []struct {
+		name         string
+		fileContent  string
+		position     int
+		expectedKind ast.Kind
+	}{
+		{
+			name: "after dot in jsdoc",
+			fileContent: `import {
+    CharacterCodes,
+    compareStringsCaseInsensitive,
+    compareStringsCaseSensitive,
+    compareValues,
+    Comparison,
+    Debug,
+    endsWith,
+    equateStringsCaseInsensitive,
+    equateStringsCaseSensitive,
+    GetCanonicalFileName,
+    getDeclarationFileExtension,
+    getStringComparer,
+    identity,
+    lastOrUndefined,
+    Path,
+    some,
+    startsWith,
+} from "./_namespaces/ts.js";
+
+/**
+ * Internally, we represent paths as strings with '/' as the directory separator.
+ * When we make system calls (eg: LanguageServiceHost.getDirectory()),
+ * we expect the host to correctly handle paths in our specified format.
+ *
+ * @internal
+ */
+export const directorySeparator = "/";
+/** @internal */
+export const altDirectorySeparator = "\\";
+const urlSchemeSeparator = "://";
+const backslashRegExp = /\\/g;
+
+
+backslashRegExp.
+
+//Path Tests
+
+/**
+ * Determines whether a charCode corresponds to '/' or '\'.
+ *
+ * @internal
+ */
+export function isAnyDirectorySeparator(charCode: number): boolean {
+    return charCode === CharacterCodes.slash || charCode === CharacterCodes.backslash;
+}`,
+			position:     839,
+			expectedKind: ast.KindDotToken,
+		},
+		{
+			name:         "after comma in parameter list",
+			fileContent:  `takesCb((n, s, ))`,
+			position:     15,
+			expectedKind: ast.KindCommaToken,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			t.Parallel()
+			file := parser.ParseSourceFile(ast.SourceFileParseOptions{
+				FileName: "/file.ts",
+				Path:     "/file.ts",
+			}, testCase.fileContent, core.ScriptKindTS)
+			token := astnav.FindPrecedingToken(file, testCase.position)
+			assert.Equal(t, token.Kind, testCase.expectedKind)
+		})
+	}
+}
+
+func tsFindPrecedingTokens(t *testing.T, fileText string, positions []int) []*tokenInfo {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, "file.ts"), []byte(fileText), 0o644)
+	assert.NilError(t, err)
+
+	err = os.WriteFile(filepath.Join(dir, "positions.json"), []byte(core.Must(core.StringifyJson(positions, "", ""))), 0o644)
+	assert.NilError(t, err)
+
+	script := `
+		import fs from "fs";
+		export default (ts) => {
+			const positions = JSON.parse(fs.readFileSync("positions.json", "utf8"));
+			const fileText = fs.readFileSync("file.ts", "utf8");
+			const file = ts.createSourceFile(
+				"file.ts",
+				fileText,
+				{ languageVersion: ts.ScriptTarget.Latest, jsDocParsingMode: ts.JSDocParsingMode.ParseAll },
+				/*setParentNodes*/ true
+			);
+			return positions.map(position => {
+				let token = ts.findPrecedingToken(position, file);
+				if (token === undefined) {
+					return undefined;
+				}
+				if (token.kind === ts.SyntaxKind.SyntaxList) {
+					token = token.parent;
+				}
+				return {
+					kind: ts.Debug.formatSyntaxKind(token.kind),
+					pos: token.pos,
+					end: token.end,
+				};
+			});
+		};`
+	info, err := jstest.EvalNodeScriptWithTS[[]*tokenInfo](t, script, dir, "")
+	assert.NilError(t, err)
+	return info
+}