vendor tsgo
This commit is contained in:
844
tools/tsgo/internal/api/encoder/encoder.go
Normal file
844
tools/tsgo/internal/api/encoder/encoder.go
Normal file
@@ -0,0 +1,844 @@
|
||||
package encoder
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
"github.com/microsoft/typescript-go/internal/ast"
|
||||
"github.com/microsoft/typescript-go/internal/core"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if ast.KindLastUnaryOperator > 0x3f {
|
||||
panic(fmt.Sprintf("KindLastUnaryOperator (%d) exceeds the 6-bit commonData capacity (max 63)", ast.KindLastUnaryOperator))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
NodeOffsetKind = iota * 4
|
||||
NodeOffsetPos
|
||||
NodeOffsetEnd
|
||||
NodeOffsetNext
|
||||
NodeOffsetParent
|
||||
NodeOffsetData
|
||||
NodeOffsetFlags
|
||||
// NodeSize is the number of bytes that represents a single node in the encoded format.
|
||||
NodeSize
|
||||
)
|
||||
|
||||
const (
|
||||
NodeDataTypeChildren uint32 = iota << 30
|
||||
NodeDataTypeString
|
||||
NodeDataTypeExtendedData
|
||||
)
|
||||
|
||||
const (
|
||||
NodeDataTypeMask uint32 = 0xc0_00_00_00
|
||||
NodeDataChildMask uint32 = 0x00_00_00_ff
|
||||
NodeDataStringIndexMask uint32 = 0x00_ff_ff_ff
|
||||
)
|
||||
|
||||
const (
|
||||
SyntaxKindNodeList uint32 = 1<<32 - 1
|
||||
)
|
||||
|
||||
const (
|
||||
HeaderOffsetMetadata = iota * 4
|
||||
HeaderOffsetHashLo0
|
||||
HeaderOffsetHashLo1
|
||||
HeaderOffsetHashHi0
|
||||
HeaderOffsetHashHi1
|
||||
HeaderOffsetParseOptions
|
||||
HeaderOffsetStringOffsets
|
||||
HeaderOffsetStringData
|
||||
HeaderOffsetExtendedData
|
||||
HeaderOffsetStructuredData
|
||||
HeaderOffsetNodes
|
||||
HeaderSize
|
||||
)
|
||||
|
||||
const (
|
||||
ProtocolVersion uint8 = 5
|
||||
)
|
||||
|
||||
// Source File Binary Format
|
||||
// =========================
|
||||
//
|
||||
// The following defines a protocol for serializing TypeScript SourceFile objects to a compact binary format. All integer
|
||||
// values are little-endian.
|
||||
//
|
||||
// Overview
|
||||
// --------
|
||||
//
|
||||
// The format comprises seven sections:
|
||||
//
|
||||
// | Section | Length | Description |
|
||||
// | ------------------ | ------------------ | ----------------------------------------------------------------------------------------------- |
|
||||
// | Header | 44 bytes | Contains the content hash, parse options, flags, and byte offsets to the start of each section. |
|
||||
// | String offsets | 8 bytes per string | Pairs of starting byte offsets and ending byte offsets into the **string data** section. |
|
||||
// | String data | variable | UTF-8 encoded string data. |
|
||||
// | Extended node data | variable | Extra data for some kinds of nodes. |
|
||||
// | Structured data | variable | Msgpack-encoded metadata blobs (e.g. file references). |
|
||||
// | Nodes | 28 bytes per node | Defines the AST structure of the file, with references to strings and extended data. |
|
||||
//
|
||||
// Header (44 bytes)
|
||||
// -----------------
|
||||
//
|
||||
// The header contains the following fields:
|
||||
//
|
||||
// | Byte offset | Type | Field |
|
||||
// | ----------- | --------- | ------------------------------------------------- |
|
||||
// | 0 | uint8 | Protocol version |
|
||||
// | 1-3 | | Reserved |
|
||||
// | 4-19 | uint128 | Source file content hash (xxh3, LE) |
|
||||
// | 20-23 | uint32 | Parse options (bitmask; bit 0: JSX, bit 1: Force) |
|
||||
// | 24-27 | uint32 | Byte offset to string offsets section |
|
||||
// | 28-31 | uint32 | Byte offset to string data section |
|
||||
// | 32-35 | uint32 | Byte offset to extended node data section |
|
||||
// | 36-39 | uint32 | Byte offset to structured data section |
|
||||
// | 40-43 | uint32 | Byte offset to nodes section |
|
||||
//
|
||||
// String offsets (8 bytes per string)
|
||||
// -----------------------------------
|
||||
//
|
||||
// Each string offset entry consists of two 4-byte unsigned integers, representing the start and end byte offsets into the
|
||||
// **string data** section.
|
||||
//
|
||||
// String data (variable)
|
||||
// ----------------------
|
||||
//
|
||||
// The string data section contains UTF-8 encoded string data, with WTF-8 used for JS strings containing lone UTF-16
|
||||
// surrogates. In typical cases, the entirety of the string data is the source file text, and individual nodes with
|
||||
// string properties reference their positional slice of the file text. In cases where a node's string property is not
|
||||
// equal to the slice of file text at its position, the unique string is appended to the string data section after the
|
||||
// file text.
|
||||
//
|
||||
// Extended node data (variable)
|
||||
// -----------------------------
|
||||
//
|
||||
// The extended node data section contains additional data for specific node types. The length and meaning of each entry
|
||||
// is defined by the node type.
|
||||
//
|
||||
// Currently, the only node types that use this section are `TemplateHead`, `TemplateMiddle`, `TemplateTail`, and
|
||||
// `SourceFile`. The extended data format for the first three is:
|
||||
//
|
||||
// | Byte offset | Type | Field |
|
||||
// | ----------- | ------ | ------------------------------------------------ |
|
||||
// | 0-4 | uint32 | Index of `text` in the string offsets section |
|
||||
// | 4-8 | uint32 | Index of `rawText` in the string offsets section |
|
||||
// | 8-12 | uint32 | Value of `templateFlags` |
|
||||
//
|
||||
// and for `SourceFile` is:
|
||||
//
|
||||
// | Byte offset | Type | Field |
|
||||
// | ----------- | ------ | -------------------------------------------------------------- |
|
||||
// | 0-4 | uint32 | Index of `text` in the string offsets section |
|
||||
// | 4-8 | uint32 | Index of `fileName` in the string offsets section |
|
||||
// | 8-12 | uint32 | Index of `path` in the string offsets section |
|
||||
// | 12-16 | uint32 | Value of `languageVariant` |
|
||||
// | 16-20 | uint32 | Value of `scriptKind` |
|
||||
// | 20-24 | uint32 | Byte offset of `referencedFiles` in structured data section |
|
||||
// | 24-28 | uint32 | Byte offset of `typeReferenceDirectives` in structured data |
|
||||
// | 28-32 | uint32 | Byte offset of `libReferenceDirectives` in structured data |
|
||||
// | 32-36 | uint32 | Byte offset of `imports` node index array in structured data |
|
||||
// | 36-40 | uint32 | Byte offset of `moduleAugmentations` node index array |
|
||||
// | 40-44 | uint32 | Byte offset of `ambientModuleNames` string array |
|
||||
// | 44-48 | uint32 | Node index of `externalModuleIndicator` (0 = nil) |
|
||||
//
|
||||
// Structured data (variable)
|
||||
// --------------------------
|
||||
//
|
||||
// The structured data section contains msgpack-encoded metadata blobs. Each blob is a self-contained
|
||||
// msgpack value. File reference arrays use the following tuple format:
|
||||
//
|
||||
// [pos: uint, end: uint, fileName: string, resolutionMode: uint, preserve: bool]
|
||||
//
|
||||
// Node index arrays (imports, moduleAugmentations) are msgpack arrays of uint values, where each
|
||||
// value is a node index into the nodes section. String arrays (ambientModuleNames) are msgpack
|
||||
// arrays of string values.
|
||||
//
|
||||
// An offset of 0xFFFFFFFF indicates no data (empty array).
|
||||
//
|
||||
// Nodes (28 bytes per node)
|
||||
// -------------------------
|
||||
//
|
||||
// The nodes section contains the AST structure of the file. Nodes are represented in a flat array in source order,
|
||||
// heavily inspired by https://marvinh.dev/blog/speeding-up-javascript-ecosystem-part-11/. Each node has the following
|
||||
// structure:
|
||||
//
|
||||
// | Byte offset | Type | Field |
|
||||
// | ----------- | ------ | -------------------------- |
|
||||
// | 0-4 | uint32 | Kind |
|
||||
// | 4-8 | uint32 | Pos |
|
||||
// | 8-12 | uint32 | End |
|
||||
// | 12-16 | uint32 | Node index of next sibling |
|
||||
// | 16-20 | uint32 | Node index of parent |
|
||||
// | 20-24 | | Node data |
|
||||
// | 24-28 | uint32 | Node flags |
|
||||
//
|
||||
// The first 28 bytes of the nodes section are zeros representing a nil node, such that nodes without a parent or next
|
||||
// sibling can unambiuously use `0` for those indices.
|
||||
//
|
||||
// NodeLists are represented as normal nodes with the special `kind` value `0xff_ff_ff_ff`. They are considered the parent
|
||||
// of their contents in the encoded format. A client reconstructing an AST similar to TypeScript's internal representation
|
||||
// should instead set the `parent` pointers of a NodeList's children to the NodeList's parent. A NodeList's `data` field
|
||||
// is the uint32 length of the list, and does not use one of the data types described below.
|
||||
//
|
||||
// For node types other than NodeList, the node data field encodes one of the following, determined by the first 2 bits of
|
||||
// the field:
|
||||
//
|
||||
// | Value | Data type | Description |
|
||||
// | ----- | --------- | ------------------------------------------------------------------------------------ |
|
||||
// | 0b00 | Children | Disambiguates which named properties of the node its children should be assigned to. |
|
||||
// | 0b01 | String | The index of the node's string property in the **string offsets** section. |
|
||||
// | 0b10 | Extended | The byte offset of the node's extended data into the **extended node data** section. |
|
||||
// | 0b11 | Reserved | Reserved for future use. |
|
||||
//
|
||||
// In all node data types, the remaining 6 bits of the first byte are used to encode small values specific to the node
|
||||
// type. For most node types, these are individual boolean flags. For unary expressions, all 6 bits encode the operator's
|
||||
// SyntaxKind value (e.g., PlusPlusToken=45, TildeToken=54), which fits because KindLastUnaryOperator (54) <= 0x3f (63).
|
||||
//
|
||||
// | Node type | Bits 0-5 | Notes |
|
||||
// | ---------------------------- | ------------------------------------- | ------------------------------ |
|
||||
// | `ImportSpecifier` | Bit 0: `isTypeOnly` | |
|
||||
// | `ImportClause` | Bit 0: `isTypeOnly`, Bit 1: `isDefer` | |
|
||||
// | `ExportSpecifier` | Bit 0: `isTypeOnly` | |
|
||||
// | `ImportEqualsDeclaration` | Bit 0: `isTypeOnly` | |
|
||||
// | `ExportDeclaration` | Bit 0: `isTypeOnly` | |
|
||||
// | `ImportTypeNode` | Bit 0: `isTypeOf` | |
|
||||
// | `ExportAssignment` | Bit 0: `isExportEquals` | |
|
||||
// | `Block` | Bit 0: `multiline` | |
|
||||
// | `ArrayLiteralExpression` | Bit 0: `multiline` | |
|
||||
// | `ObjectLiteralExpression` | Bit 0: `multiline` | |
|
||||
// | `JsxText` | Bit 0: `containsOnlyTriviaWhiteSpaces`| |
|
||||
// | `JSDocTypeLiteral` | Bit 0: `isArrayType` | |
|
||||
// | `JsDocPropertyTag` | Bit 0: `isBracketed`, Bit 1: `isNameFirst` | |
|
||||
// | `JsDocParameterTag` | Bit 0: `isBracketed`, Bit 1: `isNameFirst` | |
|
||||
// | `VariableDeclarationList` | Bit 0: is `let`, Bit 1: is `const` | |
|
||||
// | `ImportAttributes` | Bit 0: `multiline`, Bit 1: is `assert`| |
|
||||
// | `PrefixUnaryExpression` | Bits 0-5: operator SyntaxKind | e.g., `!`, `~`, `++`, `--` |
|
||||
// | `PostfixUnaryExpression` | Bits 0-5: operator SyntaxKind | e.g., `++`, `--` |
|
||||
//
|
||||
// The remaining 3 bytes of the node data field vary by data type:
|
||||
//
|
||||
// ### Children (0b00)
|
||||
//
|
||||
// If a node has fewer children than its type allows, additional data is needed to determine which properties the children
|
||||
// correspond to. The last byte of the 4-byte data field is a bitmask representing the child properties of the node type,
|
||||
// in visitor order, where `1` indicates that the child at that property is present and `0` indicates that the property is
|
||||
// nil. For example, a `MethodDeclaration` has the following child properties:
|
||||
//
|
||||
// | Property name | Bit position |
|
||||
// | -------------- | ------------ |
|
||||
// | modifiers | 0 |
|
||||
// | asteriskToken | 1 |
|
||||
// | name | 2 |
|
||||
// | postfixToken | 3 |
|
||||
// | typeParameters | 4 |
|
||||
// | parameters | 5 |
|
||||
// | returnType | 6 |
|
||||
// | body | 7 |
|
||||
//
|
||||
// A bitmask with value `0b01100101` would indicate that the next four direct descendants (i.e., node records that have a
|
||||
// `parent` set to the node index of the `MethodDeclaration`) of the node are its `modifiers`, `name`, `parameters`, and
|
||||
// `body` properties, in that order. The remaining properties are nil. (To reconstruct the node with named properties, the
|
||||
// client must consult a static table of each node type's child property names.)
|
||||
//
|
||||
// The bitmask may be zero for node types that can only have a single child, since no disambiguation is needed.
|
||||
// Additionally, the children data type may be used for nodes that can never have children, but do not require other
|
||||
// data types.
|
||||
//
|
||||
// ### String (0b01)
|
||||
//
|
||||
// The string data type is used for nodes with a single string property. (Currently, the name of that property is always
|
||||
// `text`.) The last three bytes of the 4-byte data field form a single 24-bit unsigned integer (i.e.,
|
||||
// `uint32(0x00_ff_ff_ff & node.data)`) _N_ that is an index into the **string offsets** section. The *N*th 32-bit
|
||||
// unsigned integer in the **string offsets** section is the byte offset of the start of the string in the **string data**
|
||||
// section, and the *N+1*th 32-bit unsigned integer is the byte offset of the end of the string in the
|
||||
// **string data** section.
|
||||
//
|
||||
// ### Extended (0b10)
|
||||
//
|
||||
// The extended data type is used for nodes with properties that don't fit into either the children or string data types.
|
||||
// The last three bytes of the 4-byte data field form a single 24-bit unsigned integer (i.e.,
|
||||
// `uint32(0x00_ff_ff_ff & node.data)`) _N_ that is a byte offset into the **extended node data** section. The length and
|
||||
// meaning of the data at that offset is defined by the node type. See the **Extended node data** section for details on
|
||||
// the format of the extended data for specific node types.
|
||||
//
|
||||
// Encoding Arbitrary Nodes
|
||||
// ------------------------
|
||||
//
|
||||
// The same binary format can be used to encode an arbitrary subtree of a SourceFile, not just a whole SourceFile. When
|
||||
// encoding a non-SourceFile node, the format is identical with the following differences:
|
||||
//
|
||||
// - The content hash fields in the header (bytes 4-19) are zero.
|
||||
// - The parse options field in the header (bytes 20-23) is zero.
|
||||
// - The root node in the nodes section uses its actual node kind and data encoding (via getNodeData) rather than the
|
||||
// SourceFile-specific extended data format.
|
||||
//
|
||||
// The string data section contains only the strings referenced by nodes in the subtree, rather than the full source
|
||||
// file text. The EncodeNode function provides this entrypoint.
|
||||
|
||||
// SourceFileHash returns the 128-bit content hash for a source file as a hex string.
|
||||
func SourceFileHash(sourceFile *ast.SourceFile) string {
|
||||
h := sourceFile.Hash
|
||||
return fmt.Sprintf("%016x%016x", h.Hi, h.Lo)
|
||||
}
|
||||
|
||||
// encodeParseOptions encodes the per-file ExternalModuleIndicatorOptions as a uint32 bitmask.
|
||||
func encodeParseOptions(opts ast.ExternalModuleIndicatorOptions) uint32 {
|
||||
var bits uint32
|
||||
if opts.JSX {
|
||||
bits |= 1
|
||||
}
|
||||
if opts.Force {
|
||||
bits |= 2
|
||||
}
|
||||
return bits
|
||||
}
|
||||
|
||||
// NodeIndexTable maps between AST nodes and their encoder indices for O(1) node handle resolution.
|
||||
type NodeIndexTable struct {
|
||||
Nodes []*ast.Node // index → node (for resolution)
|
||||
sortedOnce sync.Once
|
||||
sortedIdx []uint32 // indices into Nodes, sorted by node ID; built lazily
|
||||
}
|
||||
|
||||
var nodeIndexTableKey = ast.NewSourceFileDataKey[*NodeIndexTable]()
|
||||
|
||||
// GetIndex returns the encoder index for the given node.
|
||||
// On the first call the sortedIdx array is built (O(n log n) sort on a flat []uint32),
|
||||
// then subsequent calls use binary search (O(log n)). This turns out to be much faster than
|
||||
// building a map[*ast.Node]uint32 and not significantly slower for lookups.
|
||||
func (t *NodeIndexTable) GetIndex(node *ast.Node) uint32 {
|
||||
t.sortedOnce.Do(func() {
|
||||
idx := make([]uint32, 0, len(t.Nodes))
|
||||
for i, n := range t.Nodes {
|
||||
if n != nil {
|
||||
idx = append(idx, uint32(i))
|
||||
}
|
||||
}
|
||||
nodes := t.Nodes
|
||||
slices.SortFunc(idx, func(a, b uint32) int {
|
||||
return cmp.Compare(ast.GetNodeId(nodes[a]), ast.GetNodeId(nodes[b]))
|
||||
})
|
||||
t.sortedIdx = idx
|
||||
})
|
||||
target := ast.GetNodeId(node)
|
||||
i, found := core.BinarySearchUniqueFunc(t.sortedIdx, func(_ int, el uint32) int {
|
||||
return cmp.Compare(ast.GetNodeId(t.Nodes[el]), target)
|
||||
})
|
||||
if found {
|
||||
return t.sortedIdx[i]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// BuildNodeIndexTable walks the AST in the same order as encodeTree and builds
|
||||
// a NodeIndexTable without performing the full binary encoding. This is used to
|
||||
// eagerly create index tables for files that need node handles before getSourceFile
|
||||
// is called. The indices produced are guaranteed to match those from EncodeSourceFile.
|
||||
func BuildNodeIndexTable(sourceFile *ast.SourceFile) *NodeIndexTable {
|
||||
var nodeCount uint32
|
||||
nodeTable := make([]*ast.Node, 1, sourceFile.NodeCount+1) // index 0 = nil sentinel
|
||||
|
||||
visitor := &ast.NodeVisitor{
|
||||
Hooks: ast.NodeVisitorHooks{
|
||||
VisitNodes: func(nodeList *ast.NodeList, visitor *ast.NodeVisitor) *ast.NodeList {
|
||||
if nodeList == nil {
|
||||
return nodeList
|
||||
}
|
||||
nodeCount++
|
||||
nodeTable = append(nodeTable, nil) // NodeLists are not *ast.Node
|
||||
visitor.VisitSlice(nodeList.Nodes)
|
||||
return nodeList
|
||||
},
|
||||
VisitModifiers: func(modifiers *ast.ModifierList, visitor *ast.NodeVisitor) *ast.ModifierList {
|
||||
if modifiers != nil && len(modifiers.Nodes) > 0 {
|
||||
visitor.Hooks.VisitNodes(&modifiers.NodeList, visitor)
|
||||
}
|
||||
return modifiers
|
||||
},
|
||||
},
|
||||
}
|
||||
visitor.Visit = func(node *ast.Node) *ast.Node {
|
||||
nodeCount++
|
||||
nodeTable = append(nodeTable, node)
|
||||
visitor.VisitEachChild(node)
|
||||
for _, jsdoc := range node.JSDoc(sourceFile) {
|
||||
visitor.Visit(jsdoc)
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
rootNode := sourceFile.AsNode()
|
||||
// Index 1 = root node (matches encodeTree)
|
||||
nodeCount++
|
||||
nodeTable = append(nodeTable, rootNode)
|
||||
|
||||
visitor.VisitEachChild(rootNode)
|
||||
for _, jsdoc := range rootNode.JSDoc(sourceFile) {
|
||||
visitor.Visit(jsdoc)
|
||||
}
|
||||
|
||||
return &NodeIndexTable{Nodes: nodeTable}
|
||||
}
|
||||
|
||||
func GetNodeIndexTable(sourceFile *ast.SourceFile) *NodeIndexTable {
|
||||
return ast.GetOrComputeSourceFileData(sourceFile, nodeIndexTableKey, BuildNodeIndexTable)
|
||||
}
|
||||
|
||||
// EncodeSourceFile encodes an entire source file AST into the binary format.
|
||||
// Returns the encoded bytes and a NodeIndexTable mapping encoder indices to AST nodes.
|
||||
func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, error) {
|
||||
data, nodeTable, err := encodeTree(sourceFile.AsNode(), sourceFile)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
nodeTable = ast.GetOrComputeSourceFileData(sourceFile, nodeIndexTableKey, func(*ast.SourceFile) *NodeIndexTable {
|
||||
return nodeTable
|
||||
})
|
||||
return data, nodeTable, nil
|
||||
}
|
||||
|
||||
// EncodeNode encodes an arbitrary AST node and its descendants into the binary format.
|
||||
// The sourceFile is needed to provide the source text for efficient string encoding.
|
||||
// When encoding a non-SourceFile node, the header hash and parse options fields will be zero.
|
||||
// Returns the encoded bytes and a NodeIndexTable mapping encoder indices to AST nodes.
|
||||
func EncodeNode(node *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, error) {
|
||||
return encodeTree(node, sourceFile)
|
||||
}
|
||||
|
||||
func encodeTree(rootNode *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, error) {
|
||||
var parentIndex, nodeCount, prevIndex uint32
|
||||
var extendedData []byte
|
||||
var structuredData []byte
|
||||
var strs *stringTable
|
||||
var positionMap *ast.PositionMap
|
||||
if rootNode.Kind == ast.KindSourceFile {
|
||||
strs = newStringTable(sourceFile.Text(), sourceFile.TextCount)
|
||||
positionMap = sourceFile.GetPositionMap()
|
||||
} else {
|
||||
strs = newStringTable("", 0)
|
||||
if sourceFile != nil {
|
||||
positionMap = sourceFile.GetPositionMap()
|
||||
}
|
||||
}
|
||||
if positionMap == nil {
|
||||
positionMap = ast.ComputePositionMap("")
|
||||
}
|
||||
utf16 := func(pos int) uint32 {
|
||||
return uint32(positionMap.UTF8ToUTF16(pos))
|
||||
}
|
||||
var initialNodeCount int
|
||||
if sourceFile != nil {
|
||||
initialNodeCount = sourceFile.NodeCount
|
||||
}
|
||||
nodes := make([]byte, 0, (initialNodeCount+1)*NodeSize)
|
||||
|
||||
// Build node index table for O(1) handle resolution.
|
||||
// Index 0 is a nil sentinel; real nodes start at index 1.
|
||||
nodeTable := make([]*ast.Node, 1, initialNodeCount+1) // index 0 = nil sentinel
|
||||
|
||||
// Build a small map of nodes we need to track indices for (imports + moduleAugmentations).
|
||||
// Values start at 0 and are filled in during the walk.
|
||||
var nodeIndexMap map[*ast.Node]uint32
|
||||
var sfExtendedDataOffset int // byte offset in extendedData where SourceFile fields start
|
||||
if rootNode.Kind == ast.KindSourceFile {
|
||||
sf := rootNode.AsSourceFile()
|
||||
total := len(sf.Imports()) + len(sf.ModuleAugmentations)
|
||||
if sf.ExternalModuleIndicator != nil && sf.ExternalModuleIndicator != rootNode {
|
||||
total++
|
||||
}
|
||||
if total > 0 {
|
||||
nodeIndexMap = make(map[*ast.Node]uint32, total)
|
||||
for _, imp := range sf.Imports() {
|
||||
nodeIndexMap[imp.AsNode()] = 0
|
||||
}
|
||||
for _, aug := range sf.ModuleAugmentations {
|
||||
nodeIndexMap[aug.AsNode()] = 0
|
||||
}
|
||||
if sf.ExternalModuleIndicator != nil && sf.ExternalModuleIndicator != rootNode {
|
||||
nodeIndexMap[sf.ExternalModuleIndicator] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
visitor := &ast.NodeVisitor{
|
||||
Hooks: ast.NodeVisitorHooks{
|
||||
VisitNodes: func(nodeList *ast.NodeList, visitor *ast.NodeVisitor) *ast.NodeList {
|
||||
if nodeList == nil {
|
||||
return nodeList
|
||||
}
|
||||
|
||||
nodeCount++
|
||||
nodeTable = append(nodeTable, nil) // NodeLists are not *ast.Node
|
||||
if prevIndex != 0 {
|
||||
// this is the next sibling of `prevNode`
|
||||
b0, b1, b2, b3 := uint8(nodeCount), uint8(nodeCount>>8), uint8(nodeCount>>16), uint8(nodeCount>>24)
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+0] = b0
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+1] = b1
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+2] = b2
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3
|
||||
}
|
||||
|
||||
nodes = appendUint32s(nodes, SyntaxKindNodeList, utf16(nodeList.Pos()), utf16(nodeList.End()), 0, parentIndex, uint32(len(nodeList.Nodes)), 0)
|
||||
|
||||
saveParentIndex := parentIndex
|
||||
|
||||
currentIndex := nodeCount
|
||||
prevIndex = 0
|
||||
parentIndex = currentIndex
|
||||
visitor.VisitSlice(nodeList.Nodes)
|
||||
prevIndex = currentIndex
|
||||
parentIndex = saveParentIndex
|
||||
|
||||
return nodeList
|
||||
},
|
||||
VisitModifiers: func(modifiers *ast.ModifierList, visitor *ast.NodeVisitor) *ast.ModifierList {
|
||||
if modifiers != nil && len(modifiers.Nodes) > 0 {
|
||||
visitor.Hooks.VisitNodes(&modifiers.NodeList, visitor)
|
||||
}
|
||||
return modifiers
|
||||
},
|
||||
},
|
||||
}
|
||||
visitor.Visit = func(node *ast.Node) *ast.Node {
|
||||
nodeCount++
|
||||
nodeTable = append(nodeTable, node)
|
||||
if prevIndex != 0 {
|
||||
// this is the next sibling of `prevNode`
|
||||
b0, b1, b2, b3 := uint8(nodeCount), uint8(nodeCount>>8), uint8(nodeCount>>16), uint8(nodeCount>>24)
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+0] = b0
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+1] = b1
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+2] = b2
|
||||
nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3
|
||||
}
|
||||
|
||||
nodes = appendUint32s(nodes, uint32(node.Kind), utf16(node.Pos()), utf16(node.End()), 0, parentIndex, getNodeData(node, strs, positionMap, &extendedData, &structuredData), uint32(node.Flags))
|
||||
|
||||
if nodeIndexMap != nil {
|
||||
if _, ok := nodeIndexMap[node]; ok {
|
||||
nodeIndexMap[node] = nodeCount
|
||||
}
|
||||
}
|
||||
|
||||
saveParentIndex := parentIndex
|
||||
|
||||
currentIndex := nodeCount
|
||||
prevIndex = 0
|
||||
parentIndex = currentIndex
|
||||
visitor.VisitEachChild(node)
|
||||
if sourceFile != nil {
|
||||
for _, jsdoc := range node.JSDoc(sourceFile) {
|
||||
visitor.Visit(jsdoc)
|
||||
}
|
||||
}
|
||||
prevIndex = currentIndex
|
||||
parentIndex = saveParentIndex
|
||||
return node
|
||||
}
|
||||
|
||||
nodes = appendUint32s(nodes, 0, 0, 0, 0, 0, 0, 0)
|
||||
|
||||
nodeCount++
|
||||
parentIndex++
|
||||
nodeTable = append(nodeTable, rootNode) // index 1 = root node
|
||||
|
||||
sfExtendedDataOffset = len(extendedData)
|
||||
nodes = appendUint32s(nodes, uint32(rootNode.Kind), utf16(rootNode.Pos()), utf16(rootNode.End()), 0, 0, getNodeData(rootNode, strs, positionMap, &extendedData, &structuredData), uint32(rootNode.Flags))
|
||||
|
||||
visitor.VisitEachChild(rootNode)
|
||||
if sourceFile != nil {
|
||||
for _, jsdoc := range rootNode.JSDoc(sourceFile) {
|
||||
visitor.Visit(jsdoc)
|
||||
}
|
||||
}
|
||||
|
||||
var hash xxh3.Uint128
|
||||
var parseOpts uint32
|
||||
if rootNode.Kind == ast.KindSourceFile {
|
||||
hash = sourceFile.Hash
|
||||
parseOpts = encodeParseOptions(sourceFile.ParseOptions().ExternalModuleIndicatorOptions)
|
||||
|
||||
// Encode imports, moduleAugmentations, and ambientModuleNames into structured data,
|
||||
// and patch the placeholder offsets in the SourceFile extended data.
|
||||
sf := rootNode.AsSourceFile()
|
||||
importsOffset := encodeNodeIndexArray(sf.Imports(), nodeIndexMap, &structuredData)
|
||||
moduleAugmentationsOffset := encodeModuleAugmentations(sf.ModuleAugmentations, nodeIndexMap, &structuredData)
|
||||
ambientModuleNamesOffset := encodeStringArray(sf.AmbientModuleNames, &structuredData)
|
||||
// Patch the 3 placeholder uint32s at sfExtendedDataOffset + 32, 36, 40
|
||||
binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+32:], importsOffset)
|
||||
binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+36:], moduleAugmentationsOffset)
|
||||
binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+40:], ambientModuleNamesOffset)
|
||||
// Patch externalModuleIndicator node index at offset 44
|
||||
var externalModuleIndicatorIndex uint32
|
||||
if sf.ExternalModuleIndicator != nil {
|
||||
if sf.ExternalModuleIndicator == rootNode {
|
||||
externalModuleIndicatorIndex = 1 // root node index
|
||||
} else {
|
||||
externalModuleIndicatorIndex = nodeIndexMap[sf.ExternalModuleIndicator]
|
||||
}
|
||||
}
|
||||
binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+44:], externalModuleIndicatorIndex)
|
||||
}
|
||||
|
||||
metadata := uint32(ProtocolVersion) << 24
|
||||
offsetStringTableOffsets := HeaderSize
|
||||
offsetStringTableData := HeaderSize + len(strs.offsets)*4
|
||||
offsetExtendedData := offsetStringTableData + strs.stringLength()
|
||||
offsetStructuredData := offsetExtendedData + len(extendedData)
|
||||
offsetNodes := offsetStructuredData + len(structuredData)
|
||||
|
||||
header := []uint32{
|
||||
metadata,
|
||||
uint32(hash.Lo), uint32(hash.Lo >> 32),
|
||||
uint32(hash.Hi), uint32(hash.Hi >> 32),
|
||||
parseOpts,
|
||||
uint32(offsetStringTableOffsets),
|
||||
uint32(offsetStringTableData),
|
||||
uint32(offsetExtendedData),
|
||||
uint32(offsetStructuredData),
|
||||
uint32(offsetNodes),
|
||||
}
|
||||
|
||||
var headerBytes, strsBytes []byte
|
||||
headerBytes = appendUint32s(nil, header...)
|
||||
strsBytes = strs.encode()
|
||||
|
||||
return slices.Concat(
|
||||
headerBytes,
|
||||
strsBytes,
|
||||
extendedData,
|
||||
structuredData,
|
||||
nodes,
|
||||
), &NodeIndexTable{Nodes: nodeTable}, nil
|
||||
}
|
||||
|
||||
func appendUint32s(buf []byte, values ...uint32) []byte {
|
||||
for _, value := range values {
|
||||
buf = binary.LittleEndian.AppendUint32(buf, value)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func getNodeData(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) uint32 {
|
||||
t := getNodeDataType(node)
|
||||
switch t {
|
||||
case NodeDataTypeChildren:
|
||||
return t | getNodeCommonData(node) | uint32(getChildrenPropertyMask(node))
|
||||
case NodeDataTypeString:
|
||||
return t | getNodeCommonData(node) | recordNodeStrings(node, strs)
|
||||
case NodeDataTypeExtendedData:
|
||||
return t | getNodeCommonData(node) | recordExtendedData(node, strs, positionMap, extendedData, structuredData)
|
||||
default:
|
||||
panic("unreachable")
|
||||
}
|
||||
}
|
||||
|
||||
const noStructuredData = 0xFFFFFFFF
|
||||
|
||||
func recordExtendedData_SourceFile(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
|
||||
sf := node.AsSourceFile()
|
||||
textIndex := strs.add(sf.Text(), sf.Kind, sf.Pos(), sf.End())
|
||||
fileNameIndex := strs.add(sf.FileName(), 0, 0, 0)
|
||||
pathIndex := strs.add(string(sf.Path()), 0, 0, 0)
|
||||
referencedFilesOffset := encodeFileReferences(sf.ReferencedFiles, positionMap, structuredData)
|
||||
typeRefDirectivesOffset := encodeFileReferences(sf.TypeReferenceDirectives, positionMap, structuredData)
|
||||
libRefDirectivesOffset := encodeFileReferences(sf.LibReferenceDirectives, positionMap, structuredData)
|
||||
// imports, moduleAugmentations, ambientModuleNames offsets are placeholders;
|
||||
// they will be patched after the tree walk when node indices are known.
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, fileNameIndex, pathIndex, uint32(sf.LanguageVariant), uint32(sf.ScriptKind), referencedFilesOffset, typeRefDirectivesOffset, libRefDirectivesOffset, noStructuredData, noStructuredData, noStructuredData, 0)
|
||||
}
|
||||
|
||||
func recordExtendedData_TemplateHead(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
|
||||
n := node.AsTemplateHead()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))
|
||||
}
|
||||
|
||||
func recordExtendedData_TemplateMiddle(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
|
||||
n := node.AsTemplateMiddle()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))
|
||||
}
|
||||
|
||||
func recordExtendedData_TemplateTail(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
|
||||
n := node.AsTemplateTail()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))
|
||||
}
|
||||
|
||||
func boolToByte(b bool) byte {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// hasModifiers returns true if the modifier list is non-nil and has at least one modifier.
|
||||
func hasModifiers(modifiers *ast.ModifierList) bool {
|
||||
return modifiers != nil && len(modifiers.Nodes) > 0
|
||||
}
|
||||
|
||||
// encodeFileReferences encodes a slice of FileReferences as a msgpack array of tuples
|
||||
// into the structured data buffer. Returns the byte offset into the buffer, or
|
||||
// noStructuredData (0xFFFFFFFF) if the slice is empty.
|
||||
func encodeFileReferences(refs []*ast.FileReference, positionMap *ast.PositionMap, buf *[]byte) uint32 {
|
||||
if len(refs) == 0 {
|
||||
return noStructuredData
|
||||
}
|
||||
offset := uint32(len(*buf))
|
||||
*buf = msgpackWriteArrayHeader(*buf, len(refs))
|
||||
for _, ref := range refs {
|
||||
// Each entry is a 5-element tuple: [pos, end, fileName, resolutionMode, preserve]
|
||||
*buf = msgpackWriteArrayHeader(*buf, 5)
|
||||
*buf = msgpackWriteUint(*buf, uint32(positionMap.UTF8ToUTF16(ref.Pos())))
|
||||
*buf = msgpackWriteUint(*buf, uint32(positionMap.UTF8ToUTF16(ref.End())))
|
||||
*buf = msgpackWriteString(*buf, ref.FileName)
|
||||
*buf = msgpackWriteUint(*buf, uint32(ref.ResolutionMode))
|
||||
*buf = msgpackWriteBool(*buf, ref.Preserve)
|
||||
}
|
||||
return offset
|
||||
}
|
||||
|
||||
// encodeNodeIndexArray encodes a slice of LiteralLikeNodes as a msgpack array of
|
||||
// uint node indices. Returns the byte offset into the buffer, or noStructuredData
|
||||
// if the slice is empty.
|
||||
func encodeNodeIndexArray(nodes []*ast.LiteralLikeNode, indexMap map[*ast.Node]uint32, buf *[]byte) uint32 {
|
||||
if len(nodes) == 0 {
|
||||
return noStructuredData
|
||||
}
|
||||
offset := uint32(len(*buf))
|
||||
*buf = msgpackWriteArrayHeader(*buf, len(nodes))
|
||||
for _, node := range nodes {
|
||||
*buf = msgpackWriteUint(*buf, indexMap[node.AsNode()])
|
||||
}
|
||||
return offset
|
||||
}
|
||||
|
||||
// encodeModuleAugmentations encodes a slice of ModuleName nodes as a msgpack array
|
||||
// of uint node indices. Returns the byte offset into the buffer, or noStructuredData
|
||||
// if the slice is empty.
|
||||
func encodeModuleAugmentations(nodes []*ast.ModuleName, indexMap map[*ast.Node]uint32, buf *[]byte) uint32 {
|
||||
if len(nodes) == 0 {
|
||||
return noStructuredData
|
||||
}
|
||||
offset := uint32(len(*buf))
|
||||
*buf = msgpackWriteArrayHeader(*buf, len(nodes))
|
||||
for _, node := range nodes {
|
||||
*buf = msgpackWriteUint(*buf, indexMap[node.AsNode()])
|
||||
}
|
||||
return offset
|
||||
}
|
||||
|
||||
// encodeStringArray encodes a slice of strings as a msgpack array of strings.
|
||||
// Returns the byte offset into the buffer, or noStructuredData if the slice is empty.
|
||||
func encodeStringArray(strs []string, buf *[]byte) uint32 {
|
||||
if len(strs) == 0 {
|
||||
return noStructuredData
|
||||
}
|
||||
offset := uint32(len(*buf))
|
||||
*buf = msgpackWriteArrayHeader(*buf, len(strs))
|
||||
for _, s := range strs {
|
||||
*buf = msgpackWriteString(*buf, s)
|
||||
}
|
||||
return offset
|
||||
}
|
||||
|
||||
// Minimal msgpack writers for the structured data section.
|
||||
|
||||
func msgpackWriteArrayHeader(buf []byte, length int) []byte {
|
||||
if length <= 0x0f {
|
||||
return append(buf, byte(0x90|length))
|
||||
}
|
||||
if length <= 0xffff {
|
||||
return append(buf, 0xdc, byte(length>>8), byte(length))
|
||||
}
|
||||
return append(buf, 0xdd, byte(length>>24), byte(length>>16), byte(length>>8), byte(length))
|
||||
}
|
||||
|
||||
func msgpackWriteUint(buf []byte, value uint32) []byte {
|
||||
if value <= 0x7f {
|
||||
return append(buf, byte(value))
|
||||
}
|
||||
if value <= 0xff {
|
||||
return append(buf, 0xcc, byte(value))
|
||||
}
|
||||
if value <= 0xffff {
|
||||
return append(buf, 0xcd, byte(value>>8), byte(value))
|
||||
}
|
||||
return append(buf, 0xce, byte(value>>24), byte(value>>16), byte(value>>8), byte(value))
|
||||
}
|
||||
|
||||
func msgpackWriteString(buf []byte, s string) []byte {
|
||||
n := len(s)
|
||||
if n <= 0x1f {
|
||||
buf = append(buf, byte(0xa0|n))
|
||||
} else if n <= 0xff {
|
||||
buf = append(buf, 0xd9, byte(n))
|
||||
} else if n <= 0xffff {
|
||||
buf = append(buf, 0xda, byte(n>>8), byte(n))
|
||||
} else {
|
||||
buf = append(buf, 0xdb, byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
|
||||
}
|
||||
return append(buf, s...)
|
||||
}
|
||||
|
||||
func msgpackWriteBool(buf []byte, value bool) []byte {
|
||||
if value {
|
||||
return append(buf, 0xc3)
|
||||
}
|
||||
return append(buf, 0xc2)
|
||||
}
|
||||
|
||||
// Hand-written commonData encoding functions for nodes whose non-bool data
|
||||
// members cannot be automatically encoded by the generator. Each function
|
||||
// packs relevant fields into the 6-bit commonData area (bits 24-29) of the
|
||||
// 32-bit node data word.
|
||||
|
||||
func getNodeCommonData_SyntheticExpression(_ *ast.Node) uint32 {
|
||||
// SyntheticExpression is an internal compiler node that is never part of a parsed AST.
|
||||
// It should never be encoded.
|
||||
panic("SyntheticExpression should never be encoded")
|
||||
}
|
||||
|
||||
// Hand-written extended data encoding functions for literal nodes that were
|
||||
// previously string-type but whose TokenFlags/TemplateFlags cannot fit in 6 bits.
|
||||
|
||||
func recordExtendedData_StringLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) {
|
||||
n := node.AsStringLiteral()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))
|
||||
}
|
||||
|
||||
func recordExtendedData_NumericLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) {
|
||||
n := node.AsNumericLiteral()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))
|
||||
}
|
||||
|
||||
func recordExtendedData_BigIntLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) {
|
||||
n := node.AsBigIntLiteral()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))
|
||||
}
|
||||
|
||||
func recordExtendedData_RegularExpressionLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) {
|
||||
n := node.AsRegularExpressionLiteral()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))
|
||||
}
|
||||
|
||||
func recordExtendedData_NoSubstitutionTemplateLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) {
|
||||
n := node.AsNoSubstitutionTemplateLiteral()
|
||||
textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
|
||||
*extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TemplateFlags))
|
||||
}
|
||||
Reference in New Issue
Block a user