kjol/tools/tsgo/internal/diagnostics/generate.go

//go:build ignore

package main

import (
	"bytes"
	"cmp"
	"compress/gzip"
	"encoding/xml"
	"flag"
	"fmt"
	"go/format"
	"go/token"
	"log"
	"maps"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"slices"
	"strconv"
	"strings"
	"unicode"

	"github.com/microsoft/typescript-go/internal/collections"
	"github.com/microsoft/typescript-go/internal/json"
	"github.com/microsoft/typescript-go/internal/repo"
	"golang.org/x/text/language"
)

type diagnosticMessage struct {
	Category           string `json:"category"`
	Code               int    `json:"code"`
	ReportsUnnecessary bool   `json:"reportsUnnecessary"`
	ReportsDeprecated  bool   `json:"reportsDeprecated"`
	// spelling error here is [sic] in Strada
	ElidedInCompatibilityPyramid bool `json:"elidedInCompatabilityPyramid"`

	key string
}

type LCX struct {
	TgtCul    string     `xml:"TgtCul,attr"`
	RootItems []RootItem `xml:"Item"`
}

type RootItem struct {
	ItemId string            `xml:"ItemId,attr"`
	Items  []StringTableItem `xml:"Item"`
}

type StringTableItem struct {
	ItemId string          `xml:"ItemId,attr"`
	Items  []LocalizedItem `xml:"Item"`
}

type LocalizedItem struct {
	ItemId string `xml:"ItemId,attr"`
	Str    Str    `xml:"Str"`
}

type Str struct {
	Val string `xml:"Val"`
	Tgt *Tgt   `xml:"Tgt"`
}

type Tgt struct {
	Val string `xml:"Val"`
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	diagnosticsOutput := flag.String("diagnostics", "", "path to the output diagnostics_generated.go file")
	locOutput := flag.String("loc", "", "path to the output loc_generated.go file")
	locDir := flag.String("locdir", "", "directory to write locale .json.gz files")
	flag.Parse()

	if *diagnosticsOutput == "" || *locOutput == "" || *locDir == "" {
		flag.Usage()
		return
	}

	rawDiagnosticMessages := readRawMessages(filepath.Join(repo.TypeScriptSubmodulePath(), "src", "compiler", "diagnosticMessages.json"))

	_, filename, _, ok := runtime.Caller(0)
	if !ok {
		panic("could not get current filename")
	}
	filename = filepath.FromSlash(filename) // runtime.Caller always returns forward slashes; https://go.dev/issues/3335, https://go.dev/cl/603275

	rawExtraMessages := readRawMessages(filepath.Join(filepath.Dir(filename), "extraDiagnosticMessages.json"))

	maps.Copy(rawDiagnosticMessages, rawExtraMessages)
	diagnosticMessages := slices.Collect(maps.Values(rawDiagnosticMessages))

	slices.SortFunc(diagnosticMessages, func(a *diagnosticMessage, b *diagnosticMessage) int {
		return cmp.Compare(a.Code, b.Code)
	})

	// Collect known keys for filtering localizations
	knownKeys := make(map[string]bool, len(diagnosticMessages))
	for _, m := range diagnosticMessages {
		_, key := convertPropertyName(m.key, m.Code)
		knownKeys[key] = true
	}

	// Generate diagnostics file
	diagnosticsBuf := generateDiagnostics(diagnosticMessages)

	formatted, err := format.Source(diagnosticsBuf.Bytes())
	if err != nil {
		log.Fatalf("failed to format diagnostics output: %v", err)
		return
	}

	if err := os.WriteFile(*diagnosticsOutput, formatted, 0o666); err != nil {
		log.Fatalf("failed to write diagnostics output: %v", err)
		return
	}

	// Generate localizations file
	locBuf := generateLocalizations(knownKeys, *locDir)

	formatted, err = format.Source(locBuf.Bytes())
	if err != nil {
		log.Fatalf("failed to format localizations output: %v", err)
		return
	}

	if err := os.WriteFile(*locOutput, formatted, 0o666); err != nil {
		log.Fatalf("failed to write localizations output: %v", err)
		return
	}
}

func generateDiagnostics(diagnosticMessages []*diagnosticMessage) *bytes.Buffer {
	var buf bytes.Buffer

	buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n")
	buf.WriteString("\n")
	buf.WriteString("package diagnostics\n")

	for _, m := range diagnosticMessages {
		varName, key := convertPropertyName(m.key, m.Code)

		fmt.Fprintf(&buf, "var %s = &Message{code: %d, category: Category%s, key: %q, text: %q", varName, m.Code, m.Category, key, m.key)

		if m.ReportsUnnecessary {
			buf.WriteString(`, reportsUnnecessary: true`)
		}
		if m.ElidedInCompatibilityPyramid {
			buf.WriteString(`, elidedInCompatibilityPyramid: true`)
		}
		if m.ReportsDeprecated {
			buf.WriteString(`, reportsDeprecated: true`)
		}

		buf.WriteString("}\n\n")
	}

	buf.WriteString("func keyToMessage(key Key) *Message {\n")
	buf.WriteString("\tswitch key {\n")
	for _, m := range diagnosticMessages {
		_, key := convertPropertyName(m.key, m.Code)
		varName, _ := convertPropertyName(m.key, m.Code)
		fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %s\n", key, varName)
	}
	buf.WriteString("\tdefault:\n\t\treturn nil\n")
	buf.WriteString("\t}\n")
	buf.WriteString("}\n")

	return &buf
}

func generateLocalizations(knownKeys map[string]bool, locDir string) *bytes.Buffer {
	var buf bytes.Buffer

	buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n")
	buf.WriteString("\n")
	buf.WriteString("package diagnostics\n")
	buf.WriteString("\n")
	buf.WriteString("import (\n")
	buf.WriteString("\t\"compress/gzip\"\n")
	buf.WriteString("\t_ \"embed\"\n")
	buf.WriteString("\t\"strings\"\n")
	buf.WriteString("\t\"sync\"\n")
	buf.WriteString("\t\"golang.org/x/text/language\"\n")
	buf.WriteString("\t\"github.com/microsoft/typescript-go/internal/json\"\n")
	buf.WriteString(")\n")

	// Remove and recreate the loc directory for a clean state
	if err := os.RemoveAll(locDir); err != nil {
		log.Fatalf("failed to remove locale directory: %v", err)
	}
	if err := os.MkdirAll(locDir, 0o755); err != nil {
		log.Fatalf("failed to create locale directory: %v", err)
	}

	// Generate locale maps
	localeFiles, err := filepath.Glob(filepath.Join(repo.TypeScriptSubmodulePath(), "src", "loc", "lcl", "*", "diagnosticMessages", "diagnosticMessages.generated.json.lcl"))
	if err != nil {
		log.Fatalf("failed to find locale files: %v", err)
	}
	if len(localeFiles) == 0 {
		log.Fatalf("no locale files found in %s", filepath.Join(repo.TypeScriptSubmodulePath(), "src", "loc", "lcl"))
	}
	slices.Sort(localeFiles)

	type localeInfo struct {
		varName   string
		tgtCul    string
		canonical string // canonical lowercase form (e.g., "zh-cn", "pt-br")
		lang      string
		messages  map[string]string
		filename  string
	}

	var locales []localeInfo

	for _, localeFile := range localeFiles {
		localizedMessages, tgtCul := readLocalizedMessages(localeFile)
		if len(localizedMessages) == 0 {
			continue
		}

		// Filter to only known keys
		for key := range localizedMessages {
			if !knownKeys[key] {
				delete(localizedMessages, key)
			}
		}

		if len(localizedMessages) == 0 {
			continue
		}

		// Convert locale code to valid Go identifier
		localeVar := strings.ReplaceAll(strings.ReplaceAll(tgtCul, "-", ""), "_", "")

		// Parse the locale using language.Tag to get canonical forms
		tag, err := language.Parse(tgtCul)
		if err != nil {
			log.Printf("failed to parse locale %q: %v", tgtCul, err)
			continue
		}

		base, _ := tag.Base()
		lang := strings.ToLower(base.String())

		// Get canonical form (lowercase with dash)
		canonical := strings.ToLower(tgtCul)

		// Filename for the JSON.gz file (use the original tgtCul as standard language tag)
		filename := fmt.Sprintf("%s.json.gz", tgtCul)

		// Write the JSON.gz file
		// Convert map to OrderedMap with sorted keys for consistent ordering
		keys := slices.Sorted(maps.Keys(localizedMessages))
		var orderedMessages collections.OrderedMap[string, string]
		for _, key := range keys {
			orderedMessages.Set(key, localizedMessages[key])
		}
		jsonData, err := json.Marshal(&orderedMessages)
		if err != nil {
			log.Fatalf("failed to marshal locale %s: %v", tgtCul, err)
		}

		var compressed bytes.Buffer
		gzipWriter, err := gzip.NewWriterLevel(&compressed, gzip.BestCompression)
		if err != nil {
			log.Fatalf("failed to create gzip writer for locale %s: %v", tgtCul, err)
		}
		if _, err := gzipWriter.Write(jsonData); err != nil {
			log.Fatalf("failed to compress locale %s: %v", tgtCul, err)
		}
		if err := gzipWriter.Close(); err != nil {
			log.Fatalf("failed to close gzip writer for locale %s: %v", tgtCul, err)
		}

		outputPath := filepath.Join(locDir, filename)
		if err := os.WriteFile(outputPath, compressed.Bytes(), 0o644); err != nil {
			log.Fatalf("failed to write locale file %s: %v", outputPath, err)
		}

		locales = append(locales, localeInfo{
			varName:   localeVar,
			tgtCul:    tgtCul,
			canonical: canonical,
			lang:      lang,
			messages:  localizedMessages,
			filename:  filename,
		})
	}

	// Generate matcher with inlined tags
	// English is first (index 0) as the default/fallback with no translation needed
	buf.WriteString("\nvar matcher = language.NewMatcher([]language.Tag{\n")
	buf.WriteString("\tlanguage.English,\n")
	for _, loc := range locales {
		fmt.Fprintf(&buf, "\tlanguage.MustParse(%q),\n", loc.tgtCul)
	}
	buf.WriteString("})\n")

	// Generate index-to-function map for matcher results
	// English (index 0) returns nil since we use the default English text
	buf.WriteString("\nvar localeFuncs = []func() map[Key]string{\n")
	buf.WriteString("\tnil, // English (default)\n")
	for _, loc := range locales {
		fmt.Fprintf(&buf, "\t%s,\n", loc.varName)
	}
	buf.WriteString("}\n")

	// Generate helper function for decompressing locale data
	buf.WriteString("\nfunc loadLocaleData(data string) map[Key]string {\n")
	buf.WriteString("\tgr, err := gzip.NewReader(strings.NewReader(data))\n")
	buf.WriteString("\tif err != nil {\n")
	buf.WriteString("\t\tpanic(\"failed to create gzip reader: \" + err.Error())\n")
	buf.WriteString("\t}\n")
	buf.WriteString("\tdefer gr.Close()\n")
	buf.WriteString("\tvar result map[Key]string\n")
	buf.WriteString("\tif err := json.UnmarshalRead(gr, &result); err != nil {\n")
	buf.WriteString("\t\tpanic(\"failed to unmarshal locale data: \" + err.Error())\n")
	buf.WriteString("\t}\n")
	buf.WriteString("\treturn result\n")
	buf.WriteString("}\n")

	// Generate embed directives, vars, and loader functions interleaved at the bottom
	for _, loc := range locales {
		fmt.Fprintf(&buf, "\n//go:embed loc/%s\n", loc.filename)
		fmt.Fprintf(&buf, "var %sData string\n", loc.varName)
		fmt.Fprintf(&buf, "\nvar %s = sync.OnceValue(func() map[Key]string {\n", loc.varName)
		fmt.Fprintf(&buf, "\treturn loadLocaleData(%sData)\n", loc.varName)
		buf.WriteString("})\n")
	}

	return &buf
}

func readRawMessages(p string) map[int]*diagnosticMessage {
	file, err := os.Open(p)
	if err != nil {
		log.Fatalf("failed to open file: %v", err)
		return nil
	}
	defer file.Close()

	var rawMessages map[string]*diagnosticMessage
	if err := json.UnmarshalRead(file, &rawMessages); err != nil {
		log.Fatalf("failed to decode file: %v", err)
		return nil
	}

	codeToMessage := make(map[int]*diagnosticMessage, len(rawMessages))
	for k, m := range rawMessages {
		m.key = k
		codeToMessage[m.Code] = m
	}

	return codeToMessage
}

func readLocalizedMessages(p string) (map[string]string, string) {
	file, err := os.Open(p)
	if err != nil {
		log.Printf("failed to open locale file %s: %v", p, err)
		return nil, ""
	}
	defer file.Close()

	var lcx LCX
	if err := xml.NewDecoder(file).Decode(&lcx); err != nil {
		log.Printf("failed to decode locale file %s: %v", p, err)
		return nil, ""
	}

	messages := make(map[string]string)

	// Navigate the nested Item structure
	for _, rootItem := range lcx.RootItems {
		for _, stringTable := range rootItem.Items {
			for _, item := range stringTable.Items {
				// ItemId has format ";key_code", remove the leading semicolon
				itemId := strings.TrimPrefix(item.ItemId, ";")

				// Get the localized text from Tgt if available, otherwise use Val
				var text string
				if item.Str.Tgt != nil && item.Str.Tgt.Val != "" {
					text = item.Str.Tgt.Val
				} else {
					text = item.Str.Val
				}

				messages[itemId] = text
			}
		}
	}

	return messages, lcx.TgtCul
}

var (
	multipleUnderscoreRegexp           = regexp.MustCompile(`_+`)
	leadingUnderscoreUnlessDigitRegexp = regexp.MustCompile(`^_+(\D)`)
	trailingUnderscoreRegexp           = regexp.MustCompile(`_$`)
)

func convertPropertyName(origName string, code int) (varName string, key string) {
	var b strings.Builder
	b.Grow(len(origName))

	for _, r := range origName {
		switch r {
		case '*':
			b.WriteString("_Asterisk")
		case '/':
			b.WriteString("_Slash")
		case ':':
			b.WriteString("_Colon")
		default:
			if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
				b.WriteRune('_')
			} else {
				b.WriteRune(r)
			}
		}
	}

	varName = b.String()
	// get rid of all multi-underscores
	varName = multipleUnderscoreRegexp.ReplaceAllString(varName, "_")
	// remove any leading underscore, unless it is followed by a number.
	varName = leadingUnderscoreUnlessDigitRegexp.ReplaceAllString(varName, "$1")
	// get rid of all trailing underscores.
	varName = trailingUnderscoreRegexp.ReplaceAllString(varName, "")

	key = varName
	if len(key) > 100 {
		key = key[:100]
	}
	key = key + "_" + strconv.Itoa(code)

	if !token.IsExported(varName) {
		var b strings.Builder
		b.Grow(len(varName) + 2)
		if varName[0] == '_' {
			b.WriteString("X")
		} else {
			b.WriteString("X_")
		}
		b.WriteString(varName)
		varName = b.String()
	}

	if !token.IsIdentifier(varName) || !token.IsExported(varName) {
		log.Fatalf("failed to convert property name to exported identifier: %q", origName)
	}

	return varName, key
}