maxamundsen/kjol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vendor tsgo

Browse Source
This commit is contained in:
Max Amundsen
2026-07-09 16:50:43 -04:00
parent c06ea2e5a4
commit 98978e4930
5804 changed files with 1556156 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1076
tools/tsgo/internal/scanner/regexp.go Normal file
View File

File diff suppressed because it is too large Load Diff

2918
tools/tsgo/internal/scanner/scanner.go Normal file
View File

File diff suppressed because it is too large Load Diff

21
tools/tsgo/internal/scanner/scanner_test.go Normal file
View File

@@ -0,0 +1,21 @@
package scanner
import (
"testing"
"github.com/microsoft/typescript-go/internal/ast"
"github.com/microsoft/typescript-go/internal/stringutil"
"gotest.tools/v3/assert"
)
func TestScanStringPreservesLoneSurrogates(t *testing.T) {
t.Parallel()
s := NewScanner()
s.SetText(`"🦀\ud7ff\ud800\ud801\uD83E\uDD80"`)
assert.Equal(t, s.Scan(), ast.KindStringLiteral)
assert.Equal(t, s.TokenValue(), "🦀"+
stringutil.EncodeJSStringRune(0xD7FF)+
stringutil.EncodeJSStringRune(0xD800)+
stringutil.EncodeJSStringRune(0xD801)+
"🦀")
}

162
tools/tsgo/internal/scanner/unicodeproperties.go Normal file
View File

@@ -0,0 +1,162 @@
package scanner
import "github.com/microsoft/typescript-go/internal/collections"
// Table 66: Non-binary Unicode property aliases and their canonical property names
// https://tc39.es/ecma262/#table-nonbinary-unicode-properties
var nonBinaryUnicodeProperties = map[string]string{
"General_Category": "General_Category",
"gc": "General_Category",
"Script": "Script",
"sc": "Script",
"Script_Extensions": "Script_Extensions",
"scx": "Script_Extensions",
}
// Table 67: Binary Unicode property aliases and their canonical property names
// https://tc39.es/ecma262/#table-binary-unicode-properties
var binaryUnicodeProperties = collections.NewSetFromItems(
"ASCII", "ASCII_Hex_Digit", "AHex", "Alphabetic", "Alpha", "Any", "Assigned",
"Bidi_Control", "Bidi_C", "Bidi_Mirrored", "Bidi_M",
"Case_Ignorable", "CI", "Cased",
"Changes_When_Casefolded", "CWCF", "Changes_When_Casemapped", "CWCM",
"Changes_When_Lowercased", "CWL", "Changes_When_NFKC_Casefolded", "CWKCF",
"Changes_When_Titlecased", "CWT", "Changes_When_Uppercased", "CWU",
"Dash", "Default_Ignorable_Code_Point", "DI", "Deprecated", "Dep",
"Diacritic", "Dia",
"Emoji", "Emoji_Component", "EComp", "Emoji_Modifier", "EMod",
"Emoji_Modifier_Base", "EBase", "Emoji_Presentation", "EPres",
"Extended_Pictographic", "ExtPict", "Extender", "Ext",
"Grapheme_Base", "Gr_Base", "Grapheme_Extend", "Gr_Ext",
"Hex_Digit", "Hex",
"IDS_Binary_Operator", "IDSB", "IDS_Trinary_Operator", "IDST",
"ID_Continue", "IDC", "ID_Start", "IDS",
"Ideographic", "Ideo",
"Join_Control", "Join_C",
"Logical_Order_Exception", "LOE",
"Lowercase", "Lower", "Math",
"Noncharacter_Code_Point", "NChar",
"Pattern_Syntax", "Pat_Syn", "Pattern_White_Space", "Pat_WS",
"Quotation_Mark", "QMark",
"Radical",
"Regional_Indicator", "RI",
"Sentence_Terminal", "STerm",
"Soft_Dotted", "SD",
"Terminal_Punctuation", "Term",
"Unified_Ideograph", "UIdeo",
"Uppercase", "Upper",
"Variation_Selector", "VS",
"White_Space", "space",
"XID_Continue", "XIDC", "XID_Start", "XIDS",
)
// Table 68: Binary Unicode properties of strings
// https://tc39.es/ecma262/#table-binary-unicode-properties-of-strings
var binaryUnicodePropertiesOfStrings = collections.NewSetFromItems(
"Basic_Emoji", "Emoji_Keycap_Sequence", "RGI_Emoji_Modifier_Sequence",
"RGI_Emoji_Flag_Sequence", "RGI_Emoji_Tag_Sequence",
"RGI_Emoji_ZWJ_Sequence", "RGI_Emoji",
)
// Unicode 15.1
var scriptValues = collections.NewSetFromItems(
"Adlm", "Adlam", "Aghb", "Caucasian_Albanian", "Ahom", "Arab", "Arabic",
"Armi", "Imperial_Aramaic", "Armn", "Armenian", "Avst", "Avestan",
"Bali", "Balinese", "Bamu", "Bamum", "Bass", "Bassa_Vah", "Batk", "Batak",
"Beng", "Bengali", "Bhks", "Bhaiksuki", "Bopo", "Bopomofo", "Brah", "Brahmi",
"Brai", "Braille", "Bugi", "Buginese", "Buhd", "Buhid",
"Cakm", "Chakma", "Cans", "Canadian_Aboriginal", "Cari", "Carian",
"Cham", "Cher", "Cherokee", "Chrs", "Chorasmian",
"Copt", "Coptic", "Qaac", "Cpmn", "Cypro_Minoan", "Cprt", "Cypriot",
"Cyrl", "Cyrillic",
"Deva", "Devanagari", "Diak", "Dives_Akuru", "Dogr", "Dogra",
"Dsrt", "Deseret", "Dupl", "Duployan",
"Egyp", "Egyptian_Hieroglyphs", "Elba", "Elbasan", "Elym", "Elymaic",
"Ethi", "Ethiopic",
"Geor", "Georgian", "Glag", "Glagolitic",
"Gong", "Gunjala_Gondi", "Gonm", "Masaram_Gondi",
"Goth", "Gothic", "Gran", "Grantha", "Grek", "Greek",
"Gujr", "Gujarati", "Guru", "Gurmukhi",
"Hang", "Hangul", "Hani", "Han", "Hano", "Hanunoo",
"Hatr", "Hatran", "Hebr", "Hebrew",
"Hira", "Hiragana", "Hluw", "Anatolian_Hieroglyphs",
"Hmng", "Pahawh_Hmong", "Hmnp", "Nyiakeng_Puachue_Hmong",
"Hrkt", "Katakana_Or_Hiragana",
"Hung", "Old_Hungarian",
"Ital", "Old_Italic",
"Java", "Javanese",
"Kali", "Kayah_Li", "Kana", "Katakana", "Kawi",
"Khar", "Kharoshthi", "Khmr", "Khmer", "Khoj", "Khojki",
"Kits", "Khitan_Small_Script", "Knda", "Kannada", "Kthi", "Kaithi",
"Lana", "Tai_Tham", "Laoo", "Lao", "Latn", "Latin",
"Lepc", "Lepcha", "Limb", "Limbu",
"Lina", "Linear_A", "Linb", "Linear_B", "Lisu",
"Lyci", "Lycian", "Lydi", "Lydian",
"Mahj", "Mahajani", "Maka", "Makasar",
"Mand", "Mandaic", "Mani", "Manichaean", "Marc", "Marchen",
"Medf", "Medefaidrin", "Mend", "Mende_Kikakui",
"Merc", "Meroitic_Cursive", "Mero", "Meroitic_Hieroglyphs",
"Mlym", "Malayalam", "Modi", "Mong", "Mongolian",
"Mroo", "Mro", "Mtei", "Meetei_Mayek", "Mult", "Multani",
"Mymr", "Myanmar",
"Nagm", "Nag_Mundari", "Nand", "Nandinagari",
"Narb", "Old_North_Arabian", "Nbat", "Nabataean",
"Newa", "Nkoo", "Nko", "Nshu", "Nushu",
"Ogam", "Ogham", "Olck", "Ol_Chiki",
"Orkh", "Old_Turkic", "Orya", "Oriya",
"Osge", "Osage", "Osma", "Osmanya", "Ougr", "Old_Uyghur",
"Palm", "Palmyrene", "Pauc", "Pau_Cin_Hau",
"Perm", "Old_Permic", "Phag", "Phags_Pa",
"Phli", "Inscriptional_Pahlavi", "Phlp", "Psalter_Pahlavi",
"Phnx", "Phoenician", "Plrd", "Miao",
"Prti", "Inscriptional_Parthian",
"Rjng", "Rejang", "Rohg", "Hanifi_Rohingya",
"Runr", "Runic",
"Samr", "Samaritan", "Sarb", "Old_South_Arabian",
"Saur", "Saurashtra", "Sgnw", "SignWriting",
"Shaw", "Shavian", "Shrd", "Sharada",
"Sidd", "Siddham", "Sind", "Khudawadi", "Sinh", "Sinhala",
"Sogd", "Sogdian", "Sogo", "Old_Sogdian",
"Sora", "Sora_Sompeng", "Soyo", "Soyombo",
"Sund", "Sundanese", "Sylo", "Syloti_Nagri", "Syrc", "Syriac",
"Tagb", "Tagbanwa", "Takr", "Takri",
"Tale", "Tai_Le", "Talu", "New_Tai_Lue",
"Taml", "Tamil", "Tang", "Tangut", "Tavt", "Tai_Viet",
"Telu", "Telugu", "Tfng", "Tifinagh",
"Tglg", "Tagalog", "Thaa", "Thaana", "Thai", "Tibt", "Tibetan",
"Tirh", "Tirhuta", "Tnsa", "Tangsa", "Toto",
"Ugar", "Ugaritic",
"Vaii", "Vai", "Vith", "Vithkuqi",
"Wara", "Warang_Citi", "Wcho", "Wancho",
"Xpeo", "Old_Persian", "Xsux", "Cuneiform",
"Yezi", "Yezidi", "Yiii", "Yi",
"Zanb", "Zanabazar_Square",
"Zinh", "Inherited", "Qaai",
"Zyyy", "Common",
"Zzzz", "Unknown",
)
var valuesOfNonBinaryUnicodeProperties = map[string]*collections.Set[string]{
"General_Category": collections.NewSetFromItems(
"C", "Other", "Cc", "Control", "cntrl", "Cf", "Format", "Cn", "Unassigned",
"Co", "Private_Use", "Cs", "Surrogate",
"L", "Letter", "LC", "Cased_Letter", "Ll", "Lowercase_Letter", "Lm", "Modifier_Letter",
"Lo", "Other_Letter", "Lt", "Titlecase_Letter", "Lu", "Uppercase_Letter",
"M", "Mark", "Combining_Mark", "Mc", "Spacing_Mark", "Me", "Enclosing_Mark",
"Mn", "Nonspacing_Mark",
"N", "Number", "Nd", "Decimal_Number", "digit", "Nl", "Letter_Number", "No", "Other_Number",
"P", "Punctuation", "punct", "Pc", "Connector_Punctuation", "Pd", "Dash_Punctuation",
"Pe", "Close_Punctuation", "Pf", "Final_Punctuation", "Pi", "Initial_Punctuation",
"Po", "Other_Punctuation", "Ps", "Open_Punctuation",
"S", "Symbol", "Sc", "Currency_Symbol", "Sk", "Modifier_Symbol",
"Sm", "Math_Symbol", "So", "Other_Symbol",
"Z", "Separator", "Zl", "Line_Separator", "Zp", "Paragraph_Separator",
"Zs", "Space_Separator",
),
"Script": scriptValues,
// The Script_Extensions property of a character contains one or more Script values.
// See https://www.unicode.org/reports/tr24/#Script_Extensions
// Here, since each Unicode property value expression only allows a single value,
// its values can be considered the same as those of the Script property.
"Script_Extensions": scriptValues,
}

100
tools/tsgo/internal/scanner/utilities.go Normal file
View File

@@ -0,0 +1,100 @@
package scanner
import (
"strings"
"unicode"
"unicode/utf8"
"github.com/microsoft/typescript-go/internal/ast"
"github.com/microsoft/typescript-go/internal/core"
"github.com/microsoft/typescript-go/internal/debug"
)
func tokenIsIdentifierOrKeyword(token ast.Kind) bool {
return token >= ast.KindIdentifier
}
func IdentifierToKeywordKind(node *ast.Identifier) ast.Kind {
return textToKeyword[node.Text]
}
func GetSourceTextOfNodeFromSourceFile(sourceFile *ast.SourceFile, node *ast.Node, includeTrivia bool) string {
return GetTextOfNodeFromSourceText(sourceFile.Text(), node, includeTrivia)
}
func GetTextOfNodeFromSourceText(sourceText string, node *ast.Node, includeTrivia bool) string {
if ast.NodeIsMissing(node) {
return ""
}
pos := node.Pos()
if !includeTrivia {
pos = SkipTrivia(sourceText, pos)
}
text := sourceText[pos:node.End()]
if node.Flags&ast.NodeFlagsReparserTransformedLiteral != 0 {
// This is similar to `getLiteralTextOfNode` in the printer, but without the context of an `emitContext` to provide overrides
if ast.IsStringLiteral(node) {
if node.AsStringLiteral().TokenFlags&ast.TokenFlagsSingleQuote != 0 {
return "'" + text + "'"
}
return "\"" + text + "\""
} else if ast.IsIdentifier(node) {
return node.Text()
}
// Only the above node kinds are currently transformed into one another by the reparser, requiring the textual remapping.
// (Any reamppings done by emit transforms are handled by `getLiteralTextOfNode` in the printer)
// Fail on any other kinds.
debug.FailBadSyntaxKind(node, "Unexpected reparser-transformed node kind")
}
// if (isJSDocTypeExpressionOrChild(node)) {
// // strip space + asterisk at line start
// text = text.split(/\r\n|\n|\r/).map(line => line.replace(/^\s*\*/, "").trimStart()).join("\n");
// }
return text
}
func GetTextOfNode(node *ast.Node) string {
return GetSourceTextOfNodeFromSourceFile(ast.GetSourceFileOfNode(node), node, false /*includeTrivia*/)
}
func GetTextOfJSDocComment(comment *ast.NodeList) string {
if comment == nil {
return ""
}
var b strings.Builder
for _, n := range comment.Nodes {
switch n.Kind {
case ast.KindJSDocText:
b.WriteString(n.Text())
case ast.KindJSDocLink, ast.KindJSDocLinkCode, ast.KindJSDocLinkPlain:
b.WriteString(GetTextOfNode(n))
}
}
return strings.TrimRightFunc(b.String(), unicode.IsSpace)
}
func DeclarationNameToString(name *ast.Node) string {
if name == nil || name.Pos() == name.End() {
return "(Missing)"
}
return GetTextOfNode(name)
}
func IsIdentifierText(name string, languageVariant core.LanguageVariant) bool {
ch, size := utf8.DecodeRuneInString(name)
if !IsIdentifierStart(ch) {
return false
}
for i := size; i < len(name); {
ch, size = utf8.DecodeRuneInString(name[i:])
if !IsIdentifierPartEx(ch, languageVariant) {
return false
}
i += size
}
return true
}
func IsIntrinsicJsxName(name string) bool {
return len(name) != 0 && (name[0] >= 'a' && name[0] <= 'z' || strings.ContainsRune(name, '-'))
}