//go:build ignore package main import ( "bytes" "cmp" "compress/gzip" "encoding/xml" "flag" "fmt" "go/format" "go/token" "log" "maps" "os" "path/filepath" "regexp" "runtime" "slices" "strconv" "strings" "unicode" "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/json" "github.com/microsoft/typescript-go/internal/repo" "golang.org/x/text/language" ) type diagnosticMessage struct { Category string `json:"category"` Code int `json:"code"` ReportsUnnecessary bool `json:"reportsUnnecessary"` ReportsDeprecated bool `json:"reportsDeprecated"` // spelling error here is [sic] in Strada ElidedInCompatibilityPyramid bool `json:"elidedInCompatabilityPyramid"` key string } type LCX struct { TgtCul string `xml:"TgtCul,attr"` RootItems []RootItem `xml:"Item"` } type RootItem struct { ItemId string `xml:"ItemId,attr"` Items []StringTableItem `xml:"Item"` } type StringTableItem struct { ItemId string `xml:"ItemId,attr"` Items []LocalizedItem `xml:"Item"` } type LocalizedItem struct { ItemId string `xml:"ItemId,attr"` Str Str `xml:"Str"` } type Str struct { Val string `xml:"Val"` Tgt *Tgt `xml:"Tgt"` } type Tgt struct { Val string `xml:"Val"` } func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) diagnosticsOutput := flag.String("diagnostics", "", "path to the output diagnostics_generated.go file") locOutput := flag.String("loc", "", "path to the output loc_generated.go file") locDir := flag.String("locdir", "", "directory to write locale .json.gz files") flag.Parse() if *diagnosticsOutput == "" || *locOutput == "" || *locDir == "" { flag.Usage() return } rawDiagnosticMessages := readRawMessages(filepath.Join(repo.TypeScriptSubmodulePath(), "src", "compiler", "diagnosticMessages.json")) _, filename, _, ok := runtime.Caller(0) if !ok { panic("could not get current filename") } filename = filepath.FromSlash(filename) // runtime.Caller always returns forward slashes; https://go.dev/issues/3335, https://go.dev/cl/603275 rawExtraMessages := readRawMessages(filepath.Join(filepath.Dir(filename), "extraDiagnosticMessages.json")) maps.Copy(rawDiagnosticMessages, rawExtraMessages) diagnosticMessages := slices.Collect(maps.Values(rawDiagnosticMessages)) slices.SortFunc(diagnosticMessages, func(a *diagnosticMessage, b *diagnosticMessage) int { return cmp.Compare(a.Code, b.Code) }) // Collect known keys for filtering localizations knownKeys := make(map[string]bool, len(diagnosticMessages)) for _, m := range diagnosticMessages { _, key := convertPropertyName(m.key, m.Code) knownKeys[key] = true } // Generate diagnostics file diagnosticsBuf := generateDiagnostics(diagnosticMessages) formatted, err := format.Source(diagnosticsBuf.Bytes()) if err != nil { log.Fatalf("failed to format diagnostics output: %v", err) return } if err := os.WriteFile(*diagnosticsOutput, formatted, 0o666); err != nil { log.Fatalf("failed to write diagnostics output: %v", err) return } // Generate localizations file locBuf := generateLocalizations(knownKeys, *locDir) formatted, err = format.Source(locBuf.Bytes()) if err != nil { log.Fatalf("failed to format localizations output: %v", err) return } if err := os.WriteFile(*locOutput, formatted, 0o666); err != nil { log.Fatalf("failed to write localizations output: %v", err) return } } func generateDiagnostics(diagnosticMessages []*diagnosticMessage) *bytes.Buffer { var buf bytes.Buffer buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n") buf.WriteString("\n") buf.WriteString("package diagnostics\n") for _, m := range diagnosticMessages { varName, key := convertPropertyName(m.key, m.Code) fmt.Fprintf(&buf, "var %s = &Message{code: %d, category: Category%s, key: %q, text: %q", varName, m.Code, m.Category, key, m.key) if m.ReportsUnnecessary { buf.WriteString(`, reportsUnnecessary: true`) } if m.ElidedInCompatibilityPyramid { buf.WriteString(`, elidedInCompatibilityPyramid: true`) } if m.ReportsDeprecated { buf.WriteString(`, reportsDeprecated: true`) } buf.WriteString("}\n\n") } buf.WriteString("func keyToMessage(key Key) *Message {\n") buf.WriteString("\tswitch key {\n") for _, m := range diagnosticMessages { _, key := convertPropertyName(m.key, m.Code) varName, _ := convertPropertyName(m.key, m.Code) fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %s\n", key, varName) } buf.WriteString("\tdefault:\n\t\treturn nil\n") buf.WriteString("\t}\n") buf.WriteString("}\n") return &buf } func generateLocalizations(knownKeys map[string]bool, locDir string) *bytes.Buffer { var buf bytes.Buffer buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n") buf.WriteString("\n") buf.WriteString("package diagnostics\n") buf.WriteString("\n") buf.WriteString("import (\n") buf.WriteString("\t\"compress/gzip\"\n") buf.WriteString("\t_ \"embed\"\n") buf.WriteString("\t\"strings\"\n") buf.WriteString("\t\"sync\"\n") buf.WriteString("\t\"golang.org/x/text/language\"\n") buf.WriteString("\t\"github.com/microsoft/typescript-go/internal/json\"\n") buf.WriteString(")\n") // Remove and recreate the loc directory for a clean state if err := os.RemoveAll(locDir); err != nil { log.Fatalf("failed to remove locale directory: %v", err) } if err := os.MkdirAll(locDir, 0o755); err != nil { log.Fatalf("failed to create locale directory: %v", err) } // Generate locale maps localeFiles, err := filepath.Glob(filepath.Join(repo.TypeScriptSubmodulePath(), "src", "loc", "lcl", "*", "diagnosticMessages", "diagnosticMessages.generated.json.lcl")) if err != nil { log.Fatalf("failed to find locale files: %v", err) } if len(localeFiles) == 0 { log.Fatalf("no locale files found in %s", filepath.Join(repo.TypeScriptSubmodulePath(), "src", "loc", "lcl")) } slices.Sort(localeFiles) type localeInfo struct { varName string tgtCul string canonical string // canonical lowercase form (e.g., "zh-cn", "pt-br") lang string messages map[string]string filename string } var locales []localeInfo for _, localeFile := range localeFiles { localizedMessages, tgtCul := readLocalizedMessages(localeFile) if len(localizedMessages) == 0 { continue } // Filter to only known keys for key := range localizedMessages { if !knownKeys[key] { delete(localizedMessages, key) } } if len(localizedMessages) == 0 { continue } // Convert locale code to valid Go identifier localeVar := strings.ReplaceAll(strings.ReplaceAll(tgtCul, "-", ""), "_", "") // Parse the locale using language.Tag to get canonical forms tag, err := language.Parse(tgtCul) if err != nil { log.Printf("failed to parse locale %q: %v", tgtCul, err) continue } base, _ := tag.Base() lang := strings.ToLower(base.String()) // Get canonical form (lowercase with dash) canonical := strings.ToLower(tgtCul) // Filename for the JSON.gz file (use the original tgtCul as standard language tag) filename := fmt.Sprintf("%s.json.gz", tgtCul) // Write the JSON.gz file // Convert map to OrderedMap with sorted keys for consistent ordering keys := slices.Sorted(maps.Keys(localizedMessages)) var orderedMessages collections.OrderedMap[string, string] for _, key := range keys { orderedMessages.Set(key, localizedMessages[key]) } jsonData, err := json.Marshal(&orderedMessages) if err != nil { log.Fatalf("failed to marshal locale %s: %v", tgtCul, err) } var compressed bytes.Buffer gzipWriter, err := gzip.NewWriterLevel(&compressed, gzip.BestCompression) if err != nil { log.Fatalf("failed to create gzip writer for locale %s: %v", tgtCul, err) } if _, err := gzipWriter.Write(jsonData); err != nil { log.Fatalf("failed to compress locale %s: %v", tgtCul, err) } if err := gzipWriter.Close(); err != nil { log.Fatalf("failed to close gzip writer for locale %s: %v", tgtCul, err) } outputPath := filepath.Join(locDir, filename) if err := os.WriteFile(outputPath, compressed.Bytes(), 0o644); err != nil { log.Fatalf("failed to write locale file %s: %v", outputPath, err) } locales = append(locales, localeInfo{ varName: localeVar, tgtCul: tgtCul, canonical: canonical, lang: lang, messages: localizedMessages, filename: filename, }) } // Generate matcher with inlined tags // English is first (index 0) as the default/fallback with no translation needed buf.WriteString("\nvar matcher = language.NewMatcher([]language.Tag{\n") buf.WriteString("\tlanguage.English,\n") for _, loc := range locales { fmt.Fprintf(&buf, "\tlanguage.MustParse(%q),\n", loc.tgtCul) } buf.WriteString("})\n") // Generate index-to-function map for matcher results // English (index 0) returns nil since we use the default English text buf.WriteString("\nvar localeFuncs = []func() map[Key]string{\n") buf.WriteString("\tnil, // English (default)\n") for _, loc := range locales { fmt.Fprintf(&buf, "\t%s,\n", loc.varName) } buf.WriteString("}\n") // Generate helper function for decompressing locale data buf.WriteString("\nfunc loadLocaleData(data string) map[Key]string {\n") buf.WriteString("\tgr, err := gzip.NewReader(strings.NewReader(data))\n") buf.WriteString("\tif err != nil {\n") buf.WriteString("\t\tpanic(\"failed to create gzip reader: \" + err.Error())\n") buf.WriteString("\t}\n") buf.WriteString("\tdefer gr.Close()\n") buf.WriteString("\tvar result map[Key]string\n") buf.WriteString("\tif err := json.UnmarshalRead(gr, &result); err != nil {\n") buf.WriteString("\t\tpanic(\"failed to unmarshal locale data: \" + err.Error())\n") buf.WriteString("\t}\n") buf.WriteString("\treturn result\n") buf.WriteString("}\n") // Generate embed directives, vars, and loader functions interleaved at the bottom for _, loc := range locales { fmt.Fprintf(&buf, "\n//go:embed loc/%s\n", loc.filename) fmt.Fprintf(&buf, "var %sData string\n", loc.varName) fmt.Fprintf(&buf, "\nvar %s = sync.OnceValue(func() map[Key]string {\n", loc.varName) fmt.Fprintf(&buf, "\treturn loadLocaleData(%sData)\n", loc.varName) buf.WriteString("})\n") } return &buf } func readRawMessages(p string) map[int]*diagnosticMessage { file, err := os.Open(p) if err != nil { log.Fatalf("failed to open file: %v", err) return nil } defer file.Close() var rawMessages map[string]*diagnosticMessage if err := json.UnmarshalRead(file, &rawMessages); err != nil { log.Fatalf("failed to decode file: %v", err) return nil } codeToMessage := make(map[int]*diagnosticMessage, len(rawMessages)) for k, m := range rawMessages { m.key = k codeToMessage[m.Code] = m } return codeToMessage } func readLocalizedMessages(p string) (map[string]string, string) { file, err := os.Open(p) if err != nil { log.Printf("failed to open locale file %s: %v", p, err) return nil, "" } defer file.Close() var lcx LCX if err := xml.NewDecoder(file).Decode(&lcx); err != nil { log.Printf("failed to decode locale file %s: %v", p, err) return nil, "" } messages := make(map[string]string) // Navigate the nested Item structure for _, rootItem := range lcx.RootItems { for _, stringTable := range rootItem.Items { for _, item := range stringTable.Items { // ItemId has format ";key_code", remove the leading semicolon itemId := strings.TrimPrefix(item.ItemId, ";") // Get the localized text from Tgt if available, otherwise use Val var text string if item.Str.Tgt != nil && item.Str.Tgt.Val != "" { text = item.Str.Tgt.Val } else { text = item.Str.Val } messages[itemId] = text } } } return messages, lcx.TgtCul } var ( multipleUnderscoreRegexp = regexp.MustCompile(`_+`) leadingUnderscoreUnlessDigitRegexp = regexp.MustCompile(`^_+(\D)`) trailingUnderscoreRegexp = regexp.MustCompile(`_$`) ) func convertPropertyName(origName string, code int) (varName string, key string) { var b strings.Builder b.Grow(len(origName)) for _, r := range origName { switch r { case '*': b.WriteString("_Asterisk") case '/': b.WriteString("_Slash") case ':': b.WriteString("_Colon") default: if !unicode.IsLetter(r) && !unicode.IsDigit(r) { b.WriteRune('_') } else { b.WriteRune(r) } } } varName = b.String() // get rid of all multi-underscores varName = multipleUnderscoreRegexp.ReplaceAllString(varName, "_") // remove any leading underscore, unless it is followed by a number. varName = leadingUnderscoreUnlessDigitRegexp.ReplaceAllString(varName, "$1") // get rid of all trailing underscores. varName = trailingUnderscoreRegexp.ReplaceAllString(varName, "") key = varName if len(key) > 100 { key = key[:100] } key = key + "_" + strconv.Itoa(code) if !token.IsExported(varName) { var b strings.Builder b.Grow(len(varName) + 2) if varName[0] == '_' { b.WriteString("X") } else { b.WriteString("X_") } b.WriteString(varName) varName = b.String() } if !token.IsIdentifier(varName) || !token.IsExported(varName) { log.Fatalf("failed to convert property name to exported identifier: %q", origName) } return varName, key }