vendor tsgo
This commit is contained in:
460
tools/tsgo/internal/diagnostics/generate.go
Normal file
460
tools/tsgo/internal/diagnostics/generate.go
Normal file
@@ -0,0 +1,460 @@
|
||||
//go:build ignore
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"compress/gzip"
|
||||
"encoding/xml"
|
||||
"flag"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"go/token"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/microsoft/typescript-go/internal/collections"
|
||||
"github.com/microsoft/typescript-go/internal/json"
|
||||
"github.com/microsoft/typescript-go/internal/repo"
|
||||
"golang.org/x/text/language"
|
||||
)
|
||||
|
||||
type diagnosticMessage struct {
|
||||
Category string `json:"category"`
|
||||
Code int `json:"code"`
|
||||
ReportsUnnecessary bool `json:"reportsUnnecessary"`
|
||||
ReportsDeprecated bool `json:"reportsDeprecated"`
|
||||
// spelling error here is [sic] in Strada
|
||||
ElidedInCompatibilityPyramid bool `json:"elidedInCompatabilityPyramid"`
|
||||
|
||||
key string
|
||||
}
|
||||
|
||||
type LCX struct {
|
||||
TgtCul string `xml:"TgtCul,attr"`
|
||||
RootItems []RootItem `xml:"Item"`
|
||||
}
|
||||
|
||||
type RootItem struct {
|
||||
ItemId string `xml:"ItemId,attr"`
|
||||
Items []StringTableItem `xml:"Item"`
|
||||
}
|
||||
|
||||
type StringTableItem struct {
|
||||
ItemId string `xml:"ItemId,attr"`
|
||||
Items []LocalizedItem `xml:"Item"`
|
||||
}
|
||||
|
||||
type LocalizedItem struct {
|
||||
ItemId string `xml:"ItemId,attr"`
|
||||
Str Str `xml:"Str"`
|
||||
}
|
||||
|
||||
type Str struct {
|
||||
Val string `xml:"Val"`
|
||||
Tgt *Tgt `xml:"Tgt"`
|
||||
}
|
||||
|
||||
type Tgt struct {
|
||||
Val string `xml:"Val"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
|
||||
diagnosticsOutput := flag.String("diagnostics", "", "path to the output diagnostics_generated.go file")
|
||||
locOutput := flag.String("loc", "", "path to the output loc_generated.go file")
|
||||
locDir := flag.String("locdir", "", "directory to write locale .json.gz files")
|
||||
flag.Parse()
|
||||
|
||||
if *diagnosticsOutput == "" || *locOutput == "" || *locDir == "" {
|
||||
flag.Usage()
|
||||
return
|
||||
}
|
||||
|
||||
rawDiagnosticMessages := readRawMessages(filepath.Join(repo.TypeScriptSubmodulePath(), "src", "compiler", "diagnosticMessages.json"))
|
||||
|
||||
_, filename, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
panic("could not get current filename")
|
||||
}
|
||||
filename = filepath.FromSlash(filename) // runtime.Caller always returns forward slashes; https://go.dev/issues/3335, https://go.dev/cl/603275
|
||||
|
||||
rawExtraMessages := readRawMessages(filepath.Join(filepath.Dir(filename), "extraDiagnosticMessages.json"))
|
||||
|
||||
maps.Copy(rawDiagnosticMessages, rawExtraMessages)
|
||||
diagnosticMessages := slices.Collect(maps.Values(rawDiagnosticMessages))
|
||||
|
||||
slices.SortFunc(diagnosticMessages, func(a *diagnosticMessage, b *diagnosticMessage) int {
|
||||
return cmp.Compare(a.Code, b.Code)
|
||||
})
|
||||
|
||||
// Collect known keys for filtering localizations
|
||||
knownKeys := make(map[string]bool, len(diagnosticMessages))
|
||||
for _, m := range diagnosticMessages {
|
||||
_, key := convertPropertyName(m.key, m.Code)
|
||||
knownKeys[key] = true
|
||||
}
|
||||
|
||||
// Generate diagnostics file
|
||||
diagnosticsBuf := generateDiagnostics(diagnosticMessages)
|
||||
|
||||
formatted, err := format.Source(diagnosticsBuf.Bytes())
|
||||
if err != nil {
|
||||
log.Fatalf("failed to format diagnostics output: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.WriteFile(*diagnosticsOutput, formatted, 0o666); err != nil {
|
||||
log.Fatalf("failed to write diagnostics output: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Generate localizations file
|
||||
locBuf := generateLocalizations(knownKeys, *locDir)
|
||||
|
||||
formatted, err = format.Source(locBuf.Bytes())
|
||||
if err != nil {
|
||||
log.Fatalf("failed to format localizations output: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.WriteFile(*locOutput, formatted, 0o666); err != nil {
|
||||
log.Fatalf("failed to write localizations output: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func generateDiagnostics(diagnosticMessages []*diagnosticMessage) *bytes.Buffer {
|
||||
var buf bytes.Buffer
|
||||
|
||||
buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n")
|
||||
buf.WriteString("\n")
|
||||
buf.WriteString("package diagnostics\n")
|
||||
|
||||
for _, m := range diagnosticMessages {
|
||||
varName, key := convertPropertyName(m.key, m.Code)
|
||||
|
||||
fmt.Fprintf(&buf, "var %s = &Message{code: %d, category: Category%s, key: %q, text: %q", varName, m.Code, m.Category, key, m.key)
|
||||
|
||||
if m.ReportsUnnecessary {
|
||||
buf.WriteString(`, reportsUnnecessary: true`)
|
||||
}
|
||||
if m.ElidedInCompatibilityPyramid {
|
||||
buf.WriteString(`, elidedInCompatibilityPyramid: true`)
|
||||
}
|
||||
if m.ReportsDeprecated {
|
||||
buf.WriteString(`, reportsDeprecated: true`)
|
||||
}
|
||||
|
||||
buf.WriteString("}\n\n")
|
||||
}
|
||||
|
||||
buf.WriteString("func keyToMessage(key Key) *Message {\n")
|
||||
buf.WriteString("\tswitch key {\n")
|
||||
for _, m := range diagnosticMessages {
|
||||
_, key := convertPropertyName(m.key, m.Code)
|
||||
varName, _ := convertPropertyName(m.key, m.Code)
|
||||
fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %s\n", key, varName)
|
||||
}
|
||||
buf.WriteString("\tdefault:\n\t\treturn nil\n")
|
||||
buf.WriteString("\t}\n")
|
||||
buf.WriteString("}\n")
|
||||
|
||||
return &buf
|
||||
}
|
||||
|
||||
func generateLocalizations(knownKeys map[string]bool, locDir string) *bytes.Buffer {
|
||||
var buf bytes.Buffer
|
||||
|
||||
buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n")
|
||||
buf.WriteString("\n")
|
||||
buf.WriteString("package diagnostics\n")
|
||||
buf.WriteString("\n")
|
||||
buf.WriteString("import (\n")
|
||||
buf.WriteString("\t\"compress/gzip\"\n")
|
||||
buf.WriteString("\t_ \"embed\"\n")
|
||||
buf.WriteString("\t\"strings\"\n")
|
||||
buf.WriteString("\t\"sync\"\n")
|
||||
buf.WriteString("\t\"golang.org/x/text/language\"\n")
|
||||
buf.WriteString("\t\"github.com/microsoft/typescript-go/internal/json\"\n")
|
||||
buf.WriteString(")\n")
|
||||
|
||||
// Remove and recreate the loc directory for a clean state
|
||||
if err := os.RemoveAll(locDir); err != nil {
|
||||
log.Fatalf("failed to remove locale directory: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(locDir, 0o755); err != nil {
|
||||
log.Fatalf("failed to create locale directory: %v", err)
|
||||
}
|
||||
|
||||
// Generate locale maps
|
||||
localeFiles, err := filepath.Glob(filepath.Join(repo.TypeScriptSubmodulePath(), "src", "loc", "lcl", "*", "diagnosticMessages", "diagnosticMessages.generated.json.lcl"))
|
||||
if err != nil {
|
||||
log.Fatalf("failed to find locale files: %v", err)
|
||||
}
|
||||
if len(localeFiles) == 0 {
|
||||
log.Fatalf("no locale files found in %s", filepath.Join(repo.TypeScriptSubmodulePath(), "src", "loc", "lcl"))
|
||||
}
|
||||
slices.Sort(localeFiles)
|
||||
|
||||
type localeInfo struct {
|
||||
varName string
|
||||
tgtCul string
|
||||
canonical string // canonical lowercase form (e.g., "zh-cn", "pt-br")
|
||||
lang string
|
||||
messages map[string]string
|
||||
filename string
|
||||
}
|
||||
|
||||
var locales []localeInfo
|
||||
|
||||
for _, localeFile := range localeFiles {
|
||||
localizedMessages, tgtCul := readLocalizedMessages(localeFile)
|
||||
if len(localizedMessages) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Filter to only known keys
|
||||
for key := range localizedMessages {
|
||||
if !knownKeys[key] {
|
||||
delete(localizedMessages, key)
|
||||
}
|
||||
}
|
||||
|
||||
if len(localizedMessages) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert locale code to valid Go identifier
|
||||
localeVar := strings.ReplaceAll(strings.ReplaceAll(tgtCul, "-", ""), "_", "")
|
||||
|
||||
// Parse the locale using language.Tag to get canonical forms
|
||||
tag, err := language.Parse(tgtCul)
|
||||
if err != nil {
|
||||
log.Printf("failed to parse locale %q: %v", tgtCul, err)
|
||||
continue
|
||||
}
|
||||
|
||||
base, _ := tag.Base()
|
||||
lang := strings.ToLower(base.String())
|
||||
|
||||
// Get canonical form (lowercase with dash)
|
||||
canonical := strings.ToLower(tgtCul)
|
||||
|
||||
// Filename for the JSON.gz file (use the original tgtCul as standard language tag)
|
||||
filename := fmt.Sprintf("%s.json.gz", tgtCul)
|
||||
|
||||
// Write the JSON.gz file
|
||||
// Convert map to OrderedMap with sorted keys for consistent ordering
|
||||
keys := slices.Sorted(maps.Keys(localizedMessages))
|
||||
var orderedMessages collections.OrderedMap[string, string]
|
||||
for _, key := range keys {
|
||||
orderedMessages.Set(key, localizedMessages[key])
|
||||
}
|
||||
jsonData, err := json.Marshal(&orderedMessages)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to marshal locale %s: %v", tgtCul, err)
|
||||
}
|
||||
|
||||
var compressed bytes.Buffer
|
||||
gzipWriter, err := gzip.NewWriterLevel(&compressed, gzip.BestCompression)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to create gzip writer for locale %s: %v", tgtCul, err)
|
||||
}
|
||||
if _, err := gzipWriter.Write(jsonData); err != nil {
|
||||
log.Fatalf("failed to compress locale %s: %v", tgtCul, err)
|
||||
}
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
log.Fatalf("failed to close gzip writer for locale %s: %v", tgtCul, err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(locDir, filename)
|
||||
if err := os.WriteFile(outputPath, compressed.Bytes(), 0o644); err != nil {
|
||||
log.Fatalf("failed to write locale file %s: %v", outputPath, err)
|
||||
}
|
||||
|
||||
locales = append(locales, localeInfo{
|
||||
varName: localeVar,
|
||||
tgtCul: tgtCul,
|
||||
canonical: canonical,
|
||||
lang: lang,
|
||||
messages: localizedMessages,
|
||||
filename: filename,
|
||||
})
|
||||
}
|
||||
|
||||
// Generate matcher with inlined tags
|
||||
// English is first (index 0) as the default/fallback with no translation needed
|
||||
buf.WriteString("\nvar matcher = language.NewMatcher([]language.Tag{\n")
|
||||
buf.WriteString("\tlanguage.English,\n")
|
||||
for _, loc := range locales {
|
||||
fmt.Fprintf(&buf, "\tlanguage.MustParse(%q),\n", loc.tgtCul)
|
||||
}
|
||||
buf.WriteString("})\n")
|
||||
|
||||
// Generate index-to-function map for matcher results
|
||||
// English (index 0) returns nil since we use the default English text
|
||||
buf.WriteString("\nvar localeFuncs = []func() map[Key]string{\n")
|
||||
buf.WriteString("\tnil, // English (default)\n")
|
||||
for _, loc := range locales {
|
||||
fmt.Fprintf(&buf, "\t%s,\n", loc.varName)
|
||||
}
|
||||
buf.WriteString("}\n")
|
||||
|
||||
// Generate helper function for decompressing locale data
|
||||
buf.WriteString("\nfunc loadLocaleData(data string) map[Key]string {\n")
|
||||
buf.WriteString("\tgr, err := gzip.NewReader(strings.NewReader(data))\n")
|
||||
buf.WriteString("\tif err != nil {\n")
|
||||
buf.WriteString("\t\tpanic(\"failed to create gzip reader: \" + err.Error())\n")
|
||||
buf.WriteString("\t}\n")
|
||||
buf.WriteString("\tdefer gr.Close()\n")
|
||||
buf.WriteString("\tvar result map[Key]string\n")
|
||||
buf.WriteString("\tif err := json.UnmarshalRead(gr, &result); err != nil {\n")
|
||||
buf.WriteString("\t\tpanic(\"failed to unmarshal locale data: \" + err.Error())\n")
|
||||
buf.WriteString("\t}\n")
|
||||
buf.WriteString("\treturn result\n")
|
||||
buf.WriteString("}\n")
|
||||
|
||||
// Generate embed directives, vars, and loader functions interleaved at the bottom
|
||||
for _, loc := range locales {
|
||||
fmt.Fprintf(&buf, "\n//go:embed loc/%s\n", loc.filename)
|
||||
fmt.Fprintf(&buf, "var %sData string\n", loc.varName)
|
||||
fmt.Fprintf(&buf, "\nvar %s = sync.OnceValue(func() map[Key]string {\n", loc.varName)
|
||||
fmt.Fprintf(&buf, "\treturn loadLocaleData(%sData)\n", loc.varName)
|
||||
buf.WriteString("})\n")
|
||||
}
|
||||
|
||||
return &buf
|
||||
}
|
||||
|
||||
func readRawMessages(p string) map[int]*diagnosticMessage {
|
||||
file, err := os.Open(p)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to open file: %v", err)
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var rawMessages map[string]*diagnosticMessage
|
||||
if err := json.UnmarshalRead(file, &rawMessages); err != nil {
|
||||
log.Fatalf("failed to decode file: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
codeToMessage := make(map[int]*diagnosticMessage, len(rawMessages))
|
||||
for k, m := range rawMessages {
|
||||
m.key = k
|
||||
codeToMessage[m.Code] = m
|
||||
}
|
||||
|
||||
return codeToMessage
|
||||
}
|
||||
|
||||
func readLocalizedMessages(p string) (map[string]string, string) {
|
||||
file, err := os.Open(p)
|
||||
if err != nil {
|
||||
log.Printf("failed to open locale file %s: %v", p, err)
|
||||
return nil, ""
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var lcx LCX
|
||||
if err := xml.NewDecoder(file).Decode(&lcx); err != nil {
|
||||
log.Printf("failed to decode locale file %s: %v", p, err)
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
messages := make(map[string]string)
|
||||
|
||||
// Navigate the nested Item structure
|
||||
for _, rootItem := range lcx.RootItems {
|
||||
for _, stringTable := range rootItem.Items {
|
||||
for _, item := range stringTable.Items {
|
||||
// ItemId has format ";key_code", remove the leading semicolon
|
||||
itemId := strings.TrimPrefix(item.ItemId, ";")
|
||||
|
||||
// Get the localized text from Tgt if available, otherwise use Val
|
||||
var text string
|
||||
if item.Str.Tgt != nil && item.Str.Tgt.Val != "" {
|
||||
text = item.Str.Tgt.Val
|
||||
} else {
|
||||
text = item.Str.Val
|
||||
}
|
||||
|
||||
messages[itemId] = text
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return messages, lcx.TgtCul
|
||||
}
|
||||
|
||||
var (
|
||||
multipleUnderscoreRegexp = regexp.MustCompile(`_+`)
|
||||
leadingUnderscoreUnlessDigitRegexp = regexp.MustCompile(`^_+(\D)`)
|
||||
trailingUnderscoreRegexp = regexp.MustCompile(`_$`)
|
||||
)
|
||||
|
||||
func convertPropertyName(origName string, code int) (varName string, key string) {
|
||||
var b strings.Builder
|
||||
b.Grow(len(origName))
|
||||
|
||||
for _, r := range origName {
|
||||
switch r {
|
||||
case '*':
|
||||
b.WriteString("_Asterisk")
|
||||
case '/':
|
||||
b.WriteString("_Slash")
|
||||
case ':':
|
||||
b.WriteString("_Colon")
|
||||
default:
|
||||
if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
|
||||
b.WriteRune('_')
|
||||
} else {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
varName = b.String()
|
||||
// get rid of all multi-underscores
|
||||
varName = multipleUnderscoreRegexp.ReplaceAllString(varName, "_")
|
||||
// remove any leading underscore, unless it is followed by a number.
|
||||
varName = leadingUnderscoreUnlessDigitRegexp.ReplaceAllString(varName, "$1")
|
||||
// get rid of all trailing underscores.
|
||||
varName = trailingUnderscoreRegexp.ReplaceAllString(varName, "")
|
||||
|
||||
key = varName
|
||||
if len(key) > 100 {
|
||||
key = key[:100]
|
||||
}
|
||||
key = key + "_" + strconv.Itoa(code)
|
||||
|
||||
if !token.IsExported(varName) {
|
||||
var b strings.Builder
|
||||
b.Grow(len(varName) + 2)
|
||||
if varName[0] == '_' {
|
||||
b.WriteString("X")
|
||||
} else {
|
||||
b.WriteString("X_")
|
||||
}
|
||||
b.WriteString(varName)
|
||||
varName = b.String()
|
||||
}
|
||||
|
||||
if !token.IsIdentifier(varName) || !token.IsExported(varName) {
|
||||
log.Fatalf("failed to convert property name to exported identifier: %q", origName)
|
||||
}
|
||||
|
||||
return varName, key
|
||||
}
|
||||
Reference in New Issue
Block a user