package scanner
import (
"fmt"
"iter"
"maps"
"strconv"
"strings"
"unicode"
"unicode/utf16"
"unicode/utf8"
"github.com/microsoft/typescript-go/internal/ast"
"github.com/microsoft/typescript-go/internal/core"
"github.com/microsoft/typescript-go/internal/debug"
"github.com/microsoft/typescript-go/internal/diagnostics"
"github.com/microsoft/typescript-go/internal/jsnum"
"github.com/microsoft/typescript-go/internal/stringutil"
)
type EscapeSequenceScanningFlags int32
const (
EscapeSequenceScanningFlagsString EscapeSequenceScanningFlags = 1 << 0
EscapeSequenceScanningFlagsReportErrors EscapeSequenceScanningFlags = 1 << 1
EscapeSequenceScanningFlagsRegularExpression EscapeSequenceScanningFlags = 1 << 2
EscapeSequenceScanningFlagsAnnexB EscapeSequenceScanningFlags = 1 << 3
EscapeSequenceScanningFlagsAnyUnicodeMode EscapeSequenceScanningFlags = 1 << 4
EscapeSequenceScanningFlagsAtomEscape EscapeSequenceScanningFlags = 1 << 5
EscapeSequenceScanningFlagsReportInvalidEscapeErrors EscapeSequenceScanningFlags = EscapeSequenceScanningFlagsRegularExpression | EscapeSequenceScanningFlagsReportErrors
EscapeSequenceScanningFlagsAllowExtendedUnicodeEscape EscapeSequenceScanningFlags = EscapeSequenceScanningFlagsString | EscapeSequenceScanningFlagsAnyUnicodeMode
)
type ErrorCallback func(diagnostic *diagnostics.Message, start, length int, args ...any)
var textToKeyword = map[string]ast.Kind{
"abstract": ast.KindAbstractKeyword,
"accessor": ast.KindAccessorKeyword,
"any": ast.KindAnyKeyword,
"as": ast.KindAsKeyword,
"asserts": ast.KindAssertsKeyword,
"assert": ast.KindAssertKeyword,
"bigint": ast.KindBigIntKeyword,
"boolean": ast.KindBooleanKeyword,
"break": ast.KindBreakKeyword,
"case": ast.KindCaseKeyword,
"catch": ast.KindCatchKeyword,
"class": ast.KindClassKeyword,
"continue": ast.KindContinueKeyword,
"const": ast.KindConstKeyword,
"constructor": ast.KindConstructorKeyword,
"debugger": ast.KindDebuggerKeyword,
"declare": ast.KindDeclareKeyword,
"default": ast.KindDefaultKeyword,
"defer": ast.KindDeferKeyword,
"delete": ast.KindDeleteKeyword,
"do": ast.KindDoKeyword,
"else": ast.KindElseKeyword,
"enum": ast.KindEnumKeyword,
"export": ast.KindExportKeyword,
"extends": ast.KindExtendsKeyword,
"false": ast.KindFalseKeyword,
"finally": ast.KindFinallyKeyword,
"for": ast.KindForKeyword,
"from": ast.KindFromKeyword,
"function": ast.KindFunctionKeyword,
"get": ast.KindGetKeyword,
"if": ast.KindIfKeyword,
"immediate": ast.KindImmediateKeyword,
"implements": ast.KindImplementsKeyword,
"import": ast.KindImportKeyword,
"in": ast.KindInKeyword,
"infer": ast.KindInferKeyword,
"instanceof": ast.KindInstanceOfKeyword,
"interface": ast.KindInterfaceKeyword,
"intrinsic": ast.KindIntrinsicKeyword,
"is": ast.KindIsKeyword,
"keyof": ast.KindKeyOfKeyword,
"let": ast.KindLetKeyword,
"module": ast.KindModuleKeyword,
"namespace": ast.KindNamespaceKeyword,
"never": ast.KindNeverKeyword,
"new": ast.KindNewKeyword,
"null": ast.KindNullKeyword,
"number": ast.KindNumberKeyword,
"object": ast.KindObjectKeyword,
"package": ast.KindPackageKeyword,
"private": ast.KindPrivateKeyword,
"protected": ast.KindProtectedKeyword,
"public": ast.KindPublicKeyword,
"override": ast.KindOverrideKeyword,
"out": ast.KindOutKeyword,
"readonly": ast.KindReadonlyKeyword,
"require": ast.KindRequireKeyword,
"global": ast.KindGlobalKeyword,
"return": ast.KindReturnKeyword,
"satisfies": ast.KindSatisfiesKeyword,
"set": ast.KindSetKeyword,
"static": ast.KindStaticKeyword,
"string": ast.KindStringKeyword,
"super": ast.KindSuperKeyword,
"switch": ast.KindSwitchKeyword,
"symbol": ast.KindSymbolKeyword,
"this": ast.KindThisKeyword,
"throw": ast.KindThrowKeyword,
"true": ast.KindTrueKeyword,
"try": ast.KindTryKeyword,
"type": ast.KindTypeKeyword,
"typeof": ast.KindTypeOfKeyword,
"undefined": ast.KindUndefinedKeyword,
"unique": ast.KindUniqueKeyword,
"unknown": ast.KindUnknownKeyword,
"using": ast.KindUsingKeyword,
"var": ast.KindVarKeyword,
"void": ast.KindVoidKeyword,
"while": ast.KindWhileKeyword,
"with": ast.KindWithKeyword,
"yield": ast.KindYieldKeyword,
"async": ast.KindAsyncKeyword,
"await": ast.KindAwaitKeyword,
"of": ast.KindOfKeyword,
}
var textToToken = func() map[string]ast.Kind {
m := map[string]ast.Kind{
"{": ast.KindOpenBraceToken,
"}": ast.KindCloseBraceToken,
"(": ast.KindOpenParenToken,
")": ast.KindCloseParenToken,
"[": ast.KindOpenBracketToken,
"]": ast.KindCloseBracketToken,
".": ast.KindDotToken,
"...": ast.KindDotDotDotToken,
";": ast.KindSemicolonToken,
",": ast.KindCommaToken,
"<": ast.KindLessThanToken,
">": ast.KindGreaterThanToken,
"<=": ast.KindLessThanEqualsToken,
">=": ast.KindGreaterThanEqualsToken,
"==": ast.KindEqualsEqualsToken,
"!=": ast.KindExclamationEqualsToken,
"===": ast.KindEqualsEqualsEqualsToken,
"!==": ast.KindExclamationEqualsEqualsToken,
"=>": ast.KindEqualsGreaterThanToken,
"+": ast.KindPlusToken,
"-": ast.KindMinusToken,
"**": ast.KindAsteriskAsteriskToken,
"*": ast.KindAsteriskToken,
"/": ast.KindSlashToken,
"%": ast.KindPercentToken,
"++": ast.KindPlusPlusToken,
"--": ast.KindMinusMinusToken,
"<<": ast.KindLessThanLessThanToken,
">": ast.KindGreaterThanGreaterThanToken,
">>>": ast.KindGreaterThanGreaterThanGreaterThanToken,
"&": ast.KindAmpersandToken,
"|": ast.KindBarToken,
"^": ast.KindCaretToken,
"!": ast.KindExclamationToken,
"~": ast.KindTildeToken,
"&&": ast.KindAmpersandAmpersandToken,
"||": ast.KindBarBarToken,
"?": ast.KindQuestionToken,
"??": ast.KindQuestionQuestionToken,
"?.": ast.KindQuestionDotToken,
":": ast.KindColonToken,
"=": ast.KindEqualsToken,
"+=": ast.KindPlusEqualsToken,
"-=": ast.KindMinusEqualsToken,
"*=": ast.KindAsteriskEqualsToken,
"**=": ast.KindAsteriskAsteriskEqualsToken,
"/=": ast.KindSlashEqualsToken,
"%=": ast.KindPercentEqualsToken,
"<<=": ast.KindLessThanLessThanEqualsToken,
">>=": ast.KindGreaterThanGreaterThanEqualsToken,
">>>=": ast.KindGreaterThanGreaterThanGreaterThanEqualsToken,
"&=": ast.KindAmpersandEqualsToken,
"|=": ast.KindBarEqualsToken,
"^=": ast.KindCaretEqualsToken,
"||=": ast.KindBarBarEqualsToken,
"&&=": ast.KindAmpersandAmpersandEqualsToken,
"??=": ast.KindQuestionQuestionEqualsToken,
"@": ast.KindAtToken,
"#": ast.KindHashToken,
"`": ast.KindBacktickToken,
}
maps.Copy(m, textToKeyword)
return m
}()
type ScannerState struct {
pos int // Current position in text (and ending position of current token)
fullStartPos int // Starting position of current token including preceding whitespace
tokenStart int // Starting position of non-whitespace part of current token
token ast.Kind // Kind of current token
tokenValue string // Parsed value of current token
tokenFlags ast.TokenFlags // Flags for current token
commentDirectives []ast.CommentDirective
skipJSDocLeadingAsterisks int // Leading asterisks to skip when scanning types inside JSDoc. Should be 0 outside JSDoc
}
type Scanner struct {
text string
end int
languageVariant core.LanguageVariant
scriptTarget core.ScriptTarget
onError ErrorCallback
skipTrivia bool
ScannerState
numberCache map[string]string
hexNumberCache map[string]string
hexDigitCache map[string]string
}
func defaultScanner() Scanner {
// Using a function rather than a global is intentional; this function is
// inlined as pure code (zeroing + moves), whereas a global requires write
// barriers since the memory is mutable.
return Scanner{skipTrivia: true}
}
func NewScanner() *Scanner {
s := defaultScanner()
return &s
}
func (s *Scanner) Reset() {
numberCache := cleared(s.numberCache)
hexNumberCache := cleared(s.hexNumberCache)
hexDigitCache := cleared(s.hexDigitCache)
*s = defaultScanner()
s.numberCache = numberCache
s.hexNumberCache = hexNumberCache
s.hexDigitCache = hexDigitCache
}
func cleared[M ~map[K]V, K comparable, V any](m M) M {
clear(m)
return m
}
func (s *Scanner) Text() string {
return s.text
}
func (s *Scanner) Token() ast.Kind {
return s.token
}
func (s *Scanner) TokenFlags() ast.TokenFlags {
return s.tokenFlags
}
func (s *Scanner) TokenFullStart() int {
return s.fullStartPos
}
func (s *Scanner) TokenStart() int {
return s.tokenStart
}
func (s *Scanner) TokenEnd() int {
return s.pos
}
func (s *Scanner) TokenText() string {
return s.text[s.tokenStart:s.pos]
}
func (s *Scanner) TokenValue() string {
return s.tokenValue
}
func (s *Scanner) TokenRange() core.TextRange {
return core.NewTextRange(s.tokenStart, s.pos)
}
func (s *Scanner) CommentDirectives() []ast.CommentDirective {
return s.commentDirectives
}
func (s *Scanner) Mark() ScannerState {
return s.ScannerState
}
func (s *Scanner) Rewind(state ScannerState) {
s.ScannerState = state
}
func (s *Scanner) ResetPos(pos int) {
if pos < 0 {
panic("Cannot reset token state to negative position")
}
s.pos = pos
s.fullStartPos = pos
s.tokenStart = pos
}
func (s *Scanner) ResetTokenState(pos int) {
s.ResetPos(pos)
s.token = ast.KindUnknown
s.tokenValue = ""
s.tokenFlags = ast.TokenFlagsNone
}
func (scanner *Scanner) SetSkipJSDocLeadingAsterisks(skip bool) {
if skip {
scanner.skipJSDocLeadingAsterisks += 1
} else {
scanner.skipJSDocLeadingAsterisks += -1
}
}
func (scanner *Scanner) SetSkipTrivia(skip bool) {
scanner.skipTrivia = skip
}
func (s *Scanner) HasUnicodeEscape() bool {
return s.tokenFlags&ast.TokenFlagsUnicodeEscape != 0
}
func (s *Scanner) HasExtendedUnicodeEscape() bool {
return s.tokenFlags&ast.TokenFlagsExtendedUnicodeEscape != 0
}
func (s *Scanner) HasPrecedingLineBreak() bool {
return s.tokenFlags&ast.TokenFlagsPrecedingLineBreak != 0
}
func (s *Scanner) HasPrecedingJSDocComment() bool {
return s.tokenFlags&ast.TokenFlagsPrecedingJSDocComment != 0
}
func (s *Scanner) HasPrecedingJSDocLeadingAsterisks() bool {
return s.tokenFlags&ast.TokenFlagsPrecedingJSDocLeadingAsterisks != 0
}
func (s *Scanner) HasPrecedingJSDocWithDeprecatedTag() bool {
return s.tokenFlags&ast.TokenFlagsPrecedingJSDocWithDeprecated != 0
}
func (s *Scanner) HasPrecedingJSDocWithSeeOrLink() bool {
return s.tokenFlags&ast.TokenFlagsPrecedingJSDocWithSeeOrLink != 0
}
// scanJSDocCommentForTags scans a JSDoc comment for @deprecated, @see, and @link tags,
// setting the appropriate token flags. Called during scanning when a JSDoc comment is detected.
func (s *Scanner) scanJSDocCommentForTags(commentText string) {
for {
i := strings.IndexByte(commentText, '@')
if i < 0 {
return
}
commentText = commentText[i+1:]
if s.tokenFlags&ast.TokenFlagsPrecedingJSDocWithDeprecated == 0 && hasJSDocTag(commentText, "deprecated") {
s.tokenFlags |= ast.TokenFlagsPrecedingJSDocWithDeprecated
}
if s.tokenFlags&ast.TokenFlagsPrecedingJSDocWithSeeOrLink == 0 && hasJSDocTag(commentText, "see", "link", "linkcode", "linkplain") {
s.tokenFlags |= ast.TokenFlagsPrecedingJSDocWithSeeOrLink
}
if s.tokenFlags&(ast.TokenFlagsPrecedingJSDocWithDeprecated|ast.TokenFlagsPrecedingJSDocWithSeeOrLink) ==
(ast.TokenFlagsPrecedingJSDocWithDeprecated | ast.TokenFlagsPrecedingJSDocWithSeeOrLink) {
return
}
}
}
// hasJSDocTag reports whether text starts with one of the given tag names followed
// by a valid JSDoc tag terminator (whitespace, '}', '*', or end-of-string).
func hasJSDocTag(text string, tags ...string) bool {
for _, tag := range tags {
if !strings.HasPrefix(text, tag) {
continue
}
if len(text) == len(tag) {
return true
}
ch := text[len(tag)]
if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '}' || ch == '*' {
return true
}
}
return false
}
func (s *Scanner) SetText(text string) {
s.text = text
s.end = len(text)
s.ScannerState = ScannerState{}
}
func (s *Scanner) SetOnError(errorCallback ErrorCallback) {
s.onError = errorCallback
}
func (s *Scanner) SetLanguageVariant(languageVariant core.LanguageVariant) {
s.languageVariant = languageVariant
}
func (s *Scanner) SetScriptTarget(scriptTarget core.ScriptTarget) {
s.scriptTarget = scriptTarget
}
func (s *Scanner) languageVersion() core.ScriptTarget {
if s.scriptTarget == core.ScriptTargetNone {
return core.ScriptTargetLatest
}
return s.scriptTarget
}
func (s *Scanner) error(diagnostic *diagnostics.Message) {
s.errorAt(diagnostic, s.pos, 0)
}
func (s *Scanner) errorAt(diagnostic *diagnostics.Message, pos int, length int, args ...any) {
if s.onError != nil {
s.onError(diagnostic, pos, length, args...)
}
}
// NOTE: even though this returns a rune, it only decodes the current byte.
// It must be checked against utf8.RuneSelf to verify that a call to charAndSize
// is not needed.
func (s *Scanner) char() rune {
if s.pos < s.end {
return rune(s.text[s.pos])
}
return -1
}
// NOTE: this returns a rune, but only decodes the byte at the offset.
func (s *Scanner) charAt(offset int) rune {
if s.pos+offset < s.end {
return rune(s.text[s.pos+offset])
}
return -1
}
func (s *Scanner) charAndSize() (rune, int) {
// Fast path: a single ASCII byte. The vast majority of source bytes are
// ASCII; handling them here avoids constructing a string slice header and
// calling the non-inlined utf8.DecodeRuneInString on every byte.
if s.pos < s.end {
if b := s.text[s.pos]; b < utf8.RuneSelf {
return rune(b), 1
}
}
return utf8.DecodeRuneInString(s.text[s.pos:])
}
// scanASCIIWhile advances s.pos over the longest run of ASCII bytes for which
// pred returns true. It stops at end-of-text, the first non-ASCII byte, or the
// first byte where pred is false.
func (s *Scanner) scanASCIIWhile(pred func(byte) bool) {
text := s.text[s.pos:s.end]
i := 0
for i < len(text) {
b := text[i]
if b >= utf8.RuneSelf || !pred(b) {
break
}
i++
}
s.pos += i
}
func (s *Scanner) Scan() ast.Kind {
s.fullStartPos = s.pos
s.tokenFlags = ast.TokenFlagsNone
for {
ch := s.char()
s.tokenStart = s.pos
switch ch {
case '\t', '\v', '\f', ' ':
s.pos++
if s.skipTrivia {
continue
}
for {
ch, size := s.charAndSize()
if !stringutil.IsWhiteSpaceSingleLine(ch) {
break
}
s.pos += size
}
s.token = ast.KindWhitespaceTrivia
case '\n', '\r':
s.tokenFlags |= ast.TokenFlagsPrecedingLineBreak
if s.skipTrivia {
s.pos++
s.scanASCIIWhile(func(b byte) bool {
return b == ' ' || (b >= '\t' && b <= '\r')
})
continue
}
if ch == '\r' && s.charAt(1) == '\n' {
s.pos += 2
} else {
s.pos++
}
s.token = ast.KindNewLineTrivia
case '!':
if s.charAt(1) == '=' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindExclamationEqualsEqualsToken
} else {
s.pos += 2
s.token = ast.KindExclamationEqualsToken
}
} else {
s.pos++
s.token = ast.KindExclamationToken
}
case '"', '\'':
s.tokenValue = s.scanString(false /*jsxAttributeString*/)
s.token = ast.KindStringLiteral
case '`':
s.token = s.scanTemplateAndSetTokenValue(false /*shouldEmitInvalidEscapeError*/)
case '%':
if s.charAt(1) == '=' {
s.pos += 2
s.token = ast.KindPercentEqualsToken
} else {
s.pos++
s.token = ast.KindPercentToken
}
case '&':
next := s.charAt(1)
if next == '&' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindAmpersandAmpersandEqualsToken
} else {
s.pos += 2
s.token = ast.KindAmpersandAmpersandToken
}
} else if next == '=' {
s.pos += 2
s.token = ast.KindAmpersandEqualsToken
} else {
s.pos++
s.token = ast.KindAmpersandToken
}
case '(':
s.pos++
s.token = ast.KindOpenParenToken
case ')':
s.pos++
s.token = ast.KindCloseParenToken
case '*':
next := s.charAt(1)
if next == '=' {
s.pos += 2
s.token = ast.KindAsteriskEqualsToken
} else if next == '*' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindAsteriskAsteriskEqualsToken
} else {
s.pos += 2
s.token = ast.KindAsteriskAsteriskToken
}
} else {
s.pos++
if s.skipJSDocLeadingAsterisks != 0 &&
(s.tokenFlags&ast.TokenFlagsPrecedingJSDocLeadingAsterisks) == 0 &&
(s.tokenFlags&ast.TokenFlagsPrecedingLineBreak) != 0 {
s.tokenFlags |= ast.TokenFlagsPrecedingJSDocLeadingAsterisks
continue
}
s.token = ast.KindAsteriskToken
}
case '+':
next := s.charAt(1)
if next == '=' {
s.pos += 2
s.token = ast.KindPlusEqualsToken
} else if next == '+' {
s.pos += 2
s.token = ast.KindPlusPlusToken
} else {
s.pos++
s.token = ast.KindPlusToken
}
case ',':
s.pos++
s.token = ast.KindCommaToken
case '-':
next := s.charAt(1)
if next == '=' {
s.pos += 2
s.token = ast.KindMinusEqualsToken
} else if next == '-' {
s.pos += 2
s.token = ast.KindMinusMinusToken
} else {
s.pos++
s.token = ast.KindMinusToken
}
case '.':
next := s.charAt(1)
if stringutil.IsDigit(next) {
s.token = s.scanNumber()
} else if next == '.' && s.charAt(2) == '.' {
s.pos += 3
s.token = ast.KindDotDotDotToken
} else {
s.pos++
s.token = ast.KindDotToken
}
case '/':
// Single-line comment
if s.charAt(1) == '/' {
s.pos += 2
for {
s.scanASCIIWhile(func(b byte) bool {
return b != '\n' && b != '\r'
})
ch1, size := s.charAndSize()
if size == 0 || stringutil.IsLineBreak(ch1) {
break
}
s.pos += size
}
s.processCommentDirective(s.tokenStart, s.pos, false)
if s.skipTrivia {
continue
}
s.token = ast.KindSingleLineCommentTrivia
return s.token
}
// Multi-line comment
if s.charAt(1) == '*' {
s.pos += 2
isJSDoc := s.char() == '*' && s.charAt(1) != '/'
commentClosed := false
lastLineStart := s.tokenStart
for {
s.scanASCIIWhile(func(b byte) bool {
return b != '*' && b != '\n' && b != '\r'
})
ch1, size := s.charAndSize()
if size == 0 {
break
}
if ch1 == '*' && s.charAt(1) == '/' {
s.pos += 2
commentClosed = true
break
}
s.pos += size
if stringutil.IsLineBreak(ch1) {
lastLineStart = s.pos
s.tokenFlags |= ast.TokenFlagsPrecedingLineBreak
}
}
if isJSDoc {
s.tokenFlags |= ast.TokenFlagsPrecedingJSDocComment
s.scanJSDocCommentForTags(s.text[s.tokenStart:s.pos])
}
s.processCommentDirective(lastLineStart, s.pos, true)
if !commentClosed {
s.error(diagnostics.Asterisk_Slash_expected)
}
if s.skipTrivia {
continue
}
if !commentClosed {
s.tokenFlags |= ast.TokenFlagsUnterminated
}
s.token = ast.KindMultiLineCommentTrivia
return s.token
}
if s.charAt(1) == '=' {
s.pos += 2
s.token = ast.KindSlashEqualsToken
} else {
s.pos++
s.token = ast.KindSlashToken
}
case '0':
if s.charAt(1) == 'X' || s.charAt(1) == 'x' {
start := s.pos
s.pos += 2
digits := s.scanHexDigits(1, true, true)
if digits == "" {
s.error(diagnostics.Hexadecimal_digit_expected)
digits = "0"
}
if s.hexNumberCache == nil {
s.hexNumberCache = make(map[string]string)
}
if cachedValue, ok := s.hexNumberCache[digits]; ok {
s.tokenValue = cachedValue
} else {
rawText := s.text[start:s.pos]
if strings.HasPrefix(rawText, "0x") && rawText[2:] == digits {
s.tokenValue = rawText
} else {
s.tokenValue = "0x" + digits
}
s.hexNumberCache[digits] = s.tokenValue
}
s.tokenFlags |= ast.TokenFlagsHexSpecifier
s.token = s.scanBigIntSuffix()
break
}
if s.charAt(1) == 'B' || s.charAt(1) == 'b' {
s.pos += 2
digits := s.scanBinaryOrOctalDigits(2)
if digits == "" {
s.error(diagnostics.Binary_digit_expected)
digits = "0"
}
s.tokenValue = "0b" + digits
s.tokenFlags |= ast.TokenFlagsBinarySpecifier
s.token = s.scanBigIntSuffix()
break
}
if s.charAt(1) == 'O' || s.charAt(1) == 'o' {
s.pos += 2
digits := s.scanBinaryOrOctalDigits(8)
if digits == "" {
s.error(diagnostics.Octal_digit_expected)
digits = "0"
}
s.tokenValue = "0o" + digits
s.tokenFlags |= ast.TokenFlagsOctalSpecifier
s.token = s.scanBigIntSuffix()
break
}
fallthrough
case '1', '2', '3', '4', '5', '6', '7', '8', '9':
s.token = s.scanNumber()
case ':':
s.pos++
s.token = ast.KindColonToken
case ';':
s.pos++
s.token = ast.KindSemicolonToken
case '<':
if s.charAt(1) == '<' && isConflictMarkerTrivia(s.text, s.pos) {
s.pos = scanConflictMarkerTrivia(s.text, s.pos, s.errorAt)
if s.skipTrivia {
continue
} else {
s.token = ast.KindConflictMarkerTrivia
return s.token
}
}
if s.charAt(1) == '<' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindLessThanLessThanEqualsToken
} else {
s.pos += 2
s.token = ast.KindLessThanLessThanToken
}
} else if s.charAt(1) == '=' {
s.pos += 2
s.token = ast.KindLessThanEqualsToken
} else if s.languageVariant == core.LanguageVariantJSX && s.charAt(1) == '/' && s.charAt(2) != '*' {
s.pos += 2
s.token = ast.KindLessThanSlashToken
} else {
s.pos++
s.token = ast.KindLessThanToken
}
case '=':
if s.charAt(1) == '=' && isConflictMarkerTrivia(s.text, s.pos) {
s.pos = scanConflictMarkerTrivia(s.text, s.pos, s.errorAt)
if s.skipTrivia {
continue
} else {
s.token = ast.KindConflictMarkerTrivia
return s.token
}
}
if s.charAt(1) == '=' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindEqualsEqualsEqualsToken
} else {
s.pos += 2
s.token = ast.KindEqualsEqualsToken
}
} else if s.charAt(1) == '>' {
s.pos += 2
s.token = ast.KindEqualsGreaterThanToken
} else {
s.pos++
s.token = ast.KindEqualsToken
}
case '>':
if s.charAt(1) == '>' && isConflictMarkerTrivia(s.text, s.pos) {
s.pos = scanConflictMarkerTrivia(s.text, s.pos, s.errorAt)
if s.skipTrivia {
continue
} else {
s.token = ast.KindConflictMarkerTrivia
return s.token
}
}
s.pos++
s.token = ast.KindGreaterThanToken
case '?':
if s.charAt(1) == '.' && !stringutil.IsDigit(s.charAt(2)) {
s.pos += 2
s.token = ast.KindQuestionDotToken
} else if s.charAt(1) == '?' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindQuestionQuestionEqualsToken
} else {
s.pos += 2
s.token = ast.KindQuestionQuestionToken
}
} else {
s.pos++
s.token = ast.KindQuestionToken
}
case '[':
s.pos++
s.token = ast.KindOpenBracketToken
case ']':
s.pos++
s.token = ast.KindCloseBracketToken
case '^':
if s.charAt(1) == '=' {
s.pos += 2
s.token = ast.KindCaretEqualsToken
} else {
s.pos++
s.token = ast.KindCaretToken
}
case '{':
s.pos++
s.token = ast.KindOpenBraceToken
case '|':
if s.charAt(1) == '|' && isConflictMarkerTrivia(s.text, s.pos) {
s.pos = scanConflictMarkerTrivia(s.text, s.pos, s.errorAt)
if s.skipTrivia {
continue
} else {
s.token = ast.KindConflictMarkerTrivia
return s.token
}
}
if s.charAt(1) == '|' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindBarBarEqualsToken
} else {
s.pos += 2
s.token = ast.KindBarBarToken
}
} else if s.charAt(1) == '=' {
s.pos += 2
s.token = ast.KindBarEqualsToken
} else {
s.pos++
s.token = ast.KindBarToken
}
case '}':
s.pos++
s.token = ast.KindCloseBraceToken
case '~':
s.pos++
s.token = ast.KindTildeToken
case '@':
s.pos++
s.token = ast.KindAtToken
case '\\':
cp := s.peekUnicodeEscape()
if cp >= 0 && IsIdentifierStart(cp) {
s.tokenValue = string(s.scanUnicodeEscape(true)) + s.scanIdentifierParts()
s.token = GetIdentifierToken(s.tokenValue)
} else {
s.scanInvalidCharacter()
}
case '#':
if s.charAt(1) == '!' {
if s.pos == 0 {
s.pos += 2
for ch, size := s.charAndSize(); size > 0 && !stringutil.IsLineBreak(ch); ch, size = s.charAndSize() {
s.pos += size
}
continue
}
s.errorAt(diagnostics.X_can_only_be_used_at_the_start_of_a_file, s.pos, 2)
s.pos++
s.token = ast.KindUnknown
break
}
if s.charAt(1) == '\\' {
s.pos++
cp := s.peekUnicodeEscape()
if cp >= 0 && IsIdentifierStart(cp) {
s.tokenValue = "#" + string(s.scanUnicodeEscape(true)) + s.scanIdentifierParts()
s.token = ast.KindPrivateIdentifier
break
}
s.pos--
}
if !s.scanIdentifier(1) {
s.errorAt(diagnostics.Invalid_character, s.pos-1, 1)
s.tokenValue = "#"
}
s.token = ast.KindPrivateIdentifier
default:
if ch < 0 {
s.token = ast.KindEndOfFile
break
}
if s.scanIdentifier(0) {
s.token = GetIdentifierToken(s.tokenValue)
break
}
ch, size := s.charAndSize()
if ch == utf8.RuneError {
s.errorAt(diagnostics.File_appears_to_be_binary, 0, 0)
s.pos = len(s.text)
s.token = ast.KindNonTextFileMarkerTrivia
break
}
if stringutil.IsWhiteSpaceSingleLine(ch) {
s.pos += size
// If we get here and it's not 0x0085 (nextLine), then we're handling non-ASCII whitespace.
// Handle skipTrivia like we do in the space case above.
if ch == 0x0085 || s.skipTrivia {
continue
}
for {
ch, size = s.charAndSize()
if !stringutil.IsWhiteSpaceSingleLine(ch) {
break
}
s.pos += size
}
s.token = ast.KindWhitespaceTrivia
return s.token
}
if stringutil.IsLineBreak(ch) {
s.tokenFlags |= ast.TokenFlagsPrecedingLineBreak
s.pos += size
continue
}
s.scanInvalidCharacter()
}
return s.token
}
}
func (s *Scanner) processCommentDirective(start int, end int, multiline bool) {
// Skip starting slashes and whitespace
pos := start
if multiline {
// Skip whitespace
for pos < end && (s.text[pos] == ' ' || s.text[pos] == '\t') {
pos++
}
// Skip combinations of / and *
for pos < end && (s.text[pos] == '/' || s.text[pos] == '*') {
pos++
}
} else {
// Skip opening //
pos += 2
// Skip another / if present
for pos < end && s.text[pos] == '/' {
pos++
}
}
// Skip whitespace
for pos < end && (s.text[pos] == ' ' || s.text[pos] == '\t') {
pos++
}
// Directive must start with '@'
if !(pos < end && s.text[pos] == '@') {
return
}
pos++
var kind ast.CommentDirectiveKind
switch {
case strings.HasPrefix(s.text[pos:], "ts-expect-error"):
kind = ast.CommentDirectiveKindExpectError
case strings.HasPrefix(s.text[pos:], "ts-ignore"):
kind = ast.CommentDirectiveKindIgnore
default:
return
}
s.commentDirectives = append(s.commentDirectives, ast.CommentDirective{Loc: core.NewTextRange(start, end), Kind: kind})
}
func (s *Scanner) ReScanLessThanToken() ast.Kind {
if s.token == ast.KindLessThanLessThanToken {
s.pos = s.tokenStart + 1
s.token = ast.KindLessThanToken
}
return s.token
}
func (s *Scanner) ReScanGreaterThanToken() ast.Kind {
if s.token == ast.KindGreaterThanToken {
s.reScanGreaterThanTokenInner()
}
return s.token
}
func (s *Scanner) reScanGreaterThanTokenInner() {
s.pos = s.tokenStart + 1
if s.char() == '>' {
if s.charAt(1) == '>' {
if s.charAt(2) == '=' {
s.pos += 3
s.token = ast.KindGreaterThanGreaterThanGreaterThanEqualsToken
} else {
s.pos += 2
s.token = ast.KindGreaterThanGreaterThanGreaterThanToken
}
} else if s.charAt(1) == '=' {
s.pos += 2
s.token = ast.KindGreaterThanGreaterThanEqualsToken
} else {
s.pos++
s.token = ast.KindGreaterThanGreaterThanToken
}
} else if s.char() == '=' {
s.pos++
s.token = ast.KindGreaterThanEqualsToken
}
}
func (s *Scanner) ReScanTemplateToken(isTaggedTemplate bool) ast.Kind {
s.pos = s.tokenStart
s.token = s.scanTemplateAndSetTokenValue(!isTaggedTemplate)
return s.token
}
func (s *Scanner) ReScanAsteriskEqualsToken() ast.Kind {
if s.token != ast.KindAsteriskEqualsToken {
panic("'ReScanAsteriskEqualsToken' should only be called on a '*='")
}
s.pos = s.tokenStart + 1
s.token = ast.KindEqualsToken
return s.token
}
func (s *Scanner) ReScanSlashToken(reportErrors ...bool) ast.Kind {
shouldReportErrors := len(reportErrors) > 0 && reportErrors[0]
if s.token == ast.KindSlashToken || s.token == ast.KindSlashEqualsToken {
// Quickly get to the end of regex such that we know the flags
startOfRegExpBody := s.tokenStart + 1
p := startOfRegExpBody
inEscape := false
namedCaptureGroups := false
// Although nested character classes are allowed in Unicode Sets mode,
// an unescaped slash is nevertheless invalid even in a character class in any Unicode mode.
// This is indicated by Section 12.9.5 Regular Expression Literals of the specification,
// where nested character classes are not considered at all. (A `[` RegularExpressionClassChar
// does nothing in a RegularExpressionClass, and a `]` always closes the class.)
// Additionally, parsing nested character classes will misinterpret regexes like `/[[]/`
// as unterminated, consuming characters beyond the slash. (This even applies to `/[[]/v`,
// which should be parsed as a well-terminated regex with an incomplete character class.)
// Thus we must not handle nested character classes in the first pass.
inCharacterClass := false
loop:
for {
// If we reach the end of a file, or hit a newline, then this is an unterminated
// regex. Report error and return what we have so far.
if p >= s.end {
s.tokenFlags |= ast.TokenFlagsUnterminated
break loop
}
ch := rune(s.text[p])
switch {
case stringutil.IsLineBreak(ch):
s.tokenFlags |= ast.TokenFlagsUnterminated
break loop
case inEscape:
// Parsing an escape character;
// reset the flag and just advance to the next char.
inEscape = false
case ch == '/' && !inCharacterClass:
// A slash within a character class is permissible,
// but in general it signals the end of the regexp literal.
break loop
case ch == '[':
inCharacterClass = true
case ch == '\\':
inEscape = true
case ch == ']':
inCharacterClass = false
case !inCharacterClass && ch == '(' &&
p+1 < s.end && s.text[p+1] == '?' &&
p+2 < s.end && s.text[p+2] == '<' &&
(p+3 >= s.end || (s.text[p+3] != '=' && s.text[p+3] != '!')):
namedCaptureGroups = true
}
p++
}
endOfRegExpBody := p
if s.tokenFlags&ast.TokenFlagsUnterminated != 0 {
// Search for the nearest unbalanced bracket for better recovery. Since the expression is
// invalid anyways, we take nested square brackets into consideration for the best guess.
p = startOfRegExpBody
inEscape = false
characterClassDepth := 0
inDecimalQuantifier := false
groupDepth := 0
for p < endOfRegExpBody {
ch := rune(s.text[p])
if inEscape {
inEscape = false
} else if ch == '\\' {
inEscape = true
} else if ch == '[' {
characterClassDepth++
} else if ch == ']' && characterClassDepth != 0 {
characterClassDepth--
} else if characterClassDepth == 0 {
if ch == '{' {
inDecimalQuantifier = true
} else if ch == '}' && inDecimalQuantifier {
inDecimalQuantifier = false
} else if !inDecimalQuantifier {
if ch == '(' {
groupDepth++
} else if ch == ')' && groupDepth != 0 {
groupDepth--
} else if ch == ')' || ch == ']' || ch == '}' {
// We encountered an unbalanced bracket outside a character class. Treat this position as the end of regex.
break
}
}
}
p++
}
// Whitespaces and semicolons at the end are not likely to be part of the regex
for p > startOfRegExpBody {
ch, size := utf8.DecodeLastRuneInString(s.text[:p])
if stringutil.IsWhiteSpaceLike(ch) || ch == ';' {
p -= size
} else {
break
}
}
s.errorAt(diagnostics.Unterminated_regular_expression_literal, s.tokenStart, p-s.tokenStart)
} else {
// Consume the slash character
p++
var regExpFlags regularExpressionFlags
for p < s.end {
ch, size := utf8.DecodeRuneInString(s.text[p:])
if ch == utf8.RuneError || !IsIdentifierPart(ch) {
break
}
if shouldReportErrors {
flag, ok := charCodeToRegExpFlag[ch]
if !ok {
s.errorAt(diagnostics.Unknown_regular_expression_flag, p, size)
} else if regExpFlags&flag != 0 {
s.errorAt(diagnostics.Duplicate_regular_expression_flag, p, size)
} else if (regExpFlags|flag)®ularExpressionFlagsAnyUnicodeMode == regularExpressionFlagsAnyUnicodeMode {
s.errorAt(diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, p, size)
} else {
regExpFlags |= flag
s.checkRegularExpressionFlagAvailability(flag, p, size)
}
}
p += size
}
if shouldReportErrors {
s.pos = startOfRegExpBody
saveEnd := s.end
saveTokenPos := s.tokenStart
saveTokenFlags := s.tokenFlags
s.end = endOfRegExpBody
parser := ®ExpParser{
scanner: s,
end: endOfRegExpBody,
regExpFlags: regExpFlags,
anyUnicodeMode: regExpFlags®ularExpressionFlagsAnyUnicodeMode != 0,
unicodeSetsMode: regExpFlags®ularExpressionFlagsUnicodeSets != 0,
annexB: true,
namedCaptureGroups: namedCaptureGroups,
groupSpecifiers: make(map[string]bool),
}
parser.run()
s.end = saveEnd
s.pos = p
s.tokenStart = saveTokenPos
s.tokenFlags = saveTokenFlags
} else {
s.pos = p
}
}
s.pos = p
s.tokenValue = s.text[s.tokenStart:s.pos]
s.token = ast.KindRegularExpressionLiteral
}
return s.token
}
func (s *Scanner) ReScanJsxToken(allowMultilineJsxText bool) ast.Kind {
s.pos = s.fullStartPos
s.tokenStart = s.fullStartPos
s.token = s.ScanJsxTokenEx(allowMultilineJsxText)
return s.token
}
func (s *Scanner) ReScanHashToken() ast.Kind {
if s.token == ast.KindPrivateIdentifier {
s.pos = s.tokenStart + 1
s.token = ast.KindHashToken
}
return s.token
}
func (s *Scanner) ReScanQuestionToken() ast.Kind {
if s.token != ast.KindQuestionQuestionToken {
panic("'reScanQuestionToken' should only be called on a '??'")
}
s.pos = s.tokenStart + 1
s.token = ast.KindQuestionToken
return s.token
}
func (s *Scanner) ScanJsxToken() ast.Kind {
return s.ScanJsxTokenEx(true /*allowMultilineJsxText*/)
}
func (s *Scanner) ScanJsxTokenEx(allowMultilineJsxText bool) ast.Kind {
s.fullStartPos = s.pos
s.tokenStart = s.pos
ch := s.char()
switch {
case ch < 0:
s.token = ast.KindEndOfFile
case ch == '<':
if s.charAt(1) == '/' {
s.pos += 2
s.token = ast.KindLessThanSlashToken
} else {
s.pos++
s.token = ast.KindLessThanToken
}
case ch == '{':
s.pos++
s.token = ast.KindOpenBraceToken
default:
// First non-whitespace character on this line.
firstNonWhitespace := 0
// These initial values are special because the first line is:
// firstNonWhitespace = 0 to indicate that we want leading whitespace
for {
ch, size := s.charAndSize()
if size == 0 || ch == '{' {
break
}
if ch == '<' {
if isConflictMarkerTrivia(s.text, s.pos) {
s.pos = scanConflictMarkerTrivia(s.text, s.pos, s.errorAt)
s.token = ast.KindConflictMarkerTrivia
return s.token
}
break
}
if ch == '>' {
s.errorAt(diagnostics.Unexpected_token_Did_you_mean_or_gt, s.pos, 1)
} else if ch == '}' {
s.errorAt(diagnostics.Unexpected_token_Did_you_mean_or_rbrace, s.pos, 1)
}
// FirstNonWhitespace is 0, then we only see whitespaces so far. If we see a linebreak, we want to ignore that whitespaces.
// i.e (- : whitespace)
// ----
//
becomes
//
// ----
becomes ----
if stringutil.IsLineBreak(ch) && firstNonWhitespace == 0 {
firstNonWhitespace = -1
} else if !allowMultilineJsxText && stringutil.IsLineBreak(ch) && firstNonWhitespace > 0 {
// Stop JsxText on each line during formatting. This allows the formatter to
// indent each line correctly.
break
} else if !stringutil.IsWhiteSpaceLike(ch) {
firstNonWhitespace = s.pos
}
s.pos += size
}
s.tokenValue = s.text[s.fullStartPos:s.pos]
s.token = ast.KindJsxText
if firstNonWhitespace == -1 {
s.token = ast.KindJsxTextAllWhiteSpaces
}
}
return s.token
}
// Scans a JSX identifier; these differ from normal identifiers in that they allow dashes
func (s *Scanner) ScanJsxIdentifier() ast.Kind {
if tokenIsIdentifierOrKeyword(s.token) {
// An identifier or keyword has already been parsed - check for a `-` or a single instance of `:` and then append it and
// everything after it to the token
// Do note that this means that `scanJsxIdentifier` effectively _mutates_ the visible token without advancing to a new token
// Any caller should be expecting this behavior and should only read the pos or token value after calling it.
for {
ch := s.char()
if ch < 0 {
break
}
if ch == '-' {
s.tokenValue += "-"
s.pos++
continue
}
oldPos := s.pos
s.tokenValue += s.scanIdentifierParts() // reuse `scanIdentifierParts` so unicode escapes are handled
if s.pos == oldPos {
break
}
}
s.token = GetIdentifierToken(s.tokenValue)
}
return s.token
}
func (s *Scanner) ScanJsxAttributeValue() ast.Kind {
s.fullStartPos = s.pos
// Skip whitespace between '=' and the value so tokenStart lands on the
// opening quote, not on trivia.
for ch, size := s.charAndSize(); size > 0 && stringutil.IsWhiteSpaceLike(ch); ch, size = s.charAndSize() {
s.pos += size
}
s.tokenStart = s.pos
switch s.char() {
case '"', '\'':
s.tokenValue = s.scanString(true /*jsxAttributeString*/)
s.token = ast.KindStringLiteral
return s.token
default:
// If this scans anything other than `{`, it's a parse error.
return s.Scan()
}
}
func (s *Scanner) ReScanJsxAttributeValue() ast.Kind {
s.pos = s.fullStartPos
s.tokenStart = s.fullStartPos
return s.ScanJsxAttributeValue()
}
/** In addition to the usual JSDoc ast.Kinds, can also return ast.KindJSDocCommentTextToken */
func (s *Scanner) ScanJSDocCommentTextToken(inBackticks bool) ast.Kind {
s.fullStartPos = s.pos
s.tokenFlags = ast.TokenFlagsNone
if s.pos >= len(s.text) {
s.token = ast.KindEndOfFile
return s.token
}
s.tokenStart = s.pos
for ch, size := s.charAndSize(); s.pos < len(s.text) && !stringutil.IsLineBreak(ch) && ch != '`'; ch, size = s.charAndSize() {
if !inBackticks {
if ch == '{' {
break
} else if ch == '@' && s.pos >= 0 {
// @ doesn't start a new tag inside ``, and elsewhere, only after whitespace and before identifier
previous, _ := utf8.DecodeLastRuneInString(s.text[:s.pos])
if stringutil.IsWhiteSpaceSingleLine(previous) {
next, _ := utf8.DecodeRuneInString(s.text[s.pos+size:])
if IsIdentifierStart(next) {
break
}
}
}
}
s.pos += size
}
if s.pos == s.tokenStart {
return s.ScanJSDocToken()
}
s.tokenValue = s.text[s.tokenStart:s.pos]
s.token = ast.KindJSDocCommentTextToken
return s.token
}
// Peek at the character at the current scanner position (expected to be right after '@')
// and return true if a JSDoc tag can follow. Identifier starts indicate a tag name.
// Whitespace, newlines, and EOF are also accepted to support incomplete tags for code completion.
func (s *Scanner) CanFollowJSDocAt() bool {
if s.pos >= len(s.text) {
return true
}
ch, _ := utf8.DecodeRuneInString(s.text[s.pos:])
return IsIdentifierStart(ch) || stringutil.IsWhiteSpaceSingleLine(ch) || stringutil.IsLineBreak(ch)
}
func (s *Scanner) ScanJSDocToken() ast.Kind {
s.fullStartPos = s.pos
s.tokenFlags = ast.TokenFlagsNone
if s.pos >= len(s.text) {
s.token = ast.KindEndOfFile
return s.token
}
s.tokenStart = s.pos
ch, size := s.charAndSize()
s.pos += size
switch ch {
case '\t', '\v', '\f', ' ':
for ch2, size2 := s.charAndSize(); size2 > 0 && stringutil.IsWhiteSpaceSingleLine(ch2); ch2, size2 = s.charAndSize() {
s.pos += size2
}
s.token = ast.KindWhitespaceTrivia
return s.token
case '@':
s.token = ast.KindAtToken
return s.token
case '\r':
if s.char() == '\n' {
s.pos++
}
fallthrough
case '\n':
s.tokenFlags |= ast.TokenFlagsPrecedingLineBreak
s.token = ast.KindNewLineTrivia
return s.token
case '*':
s.token = ast.KindAsteriskToken
return s.token
case '{':
s.token = ast.KindOpenBraceToken
return s.token
case '}':
s.token = ast.KindCloseBraceToken
return s.token
case '[':
s.token = ast.KindOpenBracketToken
return s.token
case ']':
s.token = ast.KindCloseBracketToken
return s.token
case '(':
s.token = ast.KindOpenParenToken
return s.token
case ')':
s.token = ast.KindCloseParenToken
return s.token
case '<':
s.token = ast.KindLessThanToken
return s.token
case '>':
s.token = ast.KindGreaterThanToken
return s.token
case '=':
s.token = ast.KindEqualsToken
return s.token
case ',':
s.token = ast.KindCommaToken
return s.token
case '.':
s.token = ast.KindDotToken
return s.token
case '`':
s.token = ast.KindBacktickToken
return s.token
case '#':
s.token = ast.KindHashToken
return s.token
case '\\':
s.pos--
cp := s.peekUnicodeEscape()
if cp >= 0 && IsIdentifierStart(cp) {
s.tokenValue = string(s.scanUnicodeEscape(true)) + s.scanIdentifierParts()
s.token = GetIdentifierToken(s.tokenValue)
} else {
s.pos++
s.token = ast.KindUnknown
}
return s.token
}
if IsIdentifierStart(ch) {
char := ch
for {
if s.pos >= len(s.text) {
break
}
char, size = s.charAndSize()
if !IsIdentifierPart(char) && char != '-' {
break
}
s.pos += size
}
s.tokenValue = s.text[s.tokenStart:s.pos]
if char == '\\' {
s.tokenValue += s.scanIdentifierParts()
}
s.token = GetIdentifierToken(s.tokenValue)
return s.token
} else {
s.token = ast.KindUnknown
return s.token
}
}
func (s *Scanner) scanIdentifier(prefixLength int) bool {
start := s.pos
s.pos += prefixLength
ch := s.char()
// Fast path for simple ASCII identifiers
if stringutil.IsASCIILetter(ch) || ch == '_' || ch == '$' {
s.pos++
s.scanASCIIWhile(func(b byte) bool {
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' || b == '$'
})
ch = s.char()
if ch < utf8.RuneSelf && ch != '\\' {
s.tokenValue = s.text[start:s.pos]
return true
}
s.pos = start + prefixLength
}
ch, size := s.charAndSize()
if IsIdentifierStart(ch) {
for {
s.pos += size
ch, size = s.charAndSize()
if !IsIdentifierPart(ch) {
break
}
}
s.tokenValue = s.text[start:s.pos]
if ch == '\\' {
s.tokenValue += s.scanIdentifierParts()
}
return true
}
return false
}
func (s *Scanner) scanIdentifierParts() string {
var sb strings.Builder
start := s.pos
for {
ch, size := s.charAndSize()
if IsIdentifierPart(ch) {
s.pos += size
continue
}
if ch == '\\' {
escaped := s.peekUnicodeEscape()
if escaped >= 0 && IsIdentifierPart(escaped) {
sb.WriteString(s.text[start:s.pos])
sb.WriteRune(s.scanUnicodeEscape(true))
start = s.pos
continue
}
}
break
}
sb.WriteString(s.text[start:s.pos])
return sb.String()
}
func (s *Scanner) scanString(jsxAttributeString bool) string {
quote := s.char()
if quote == '\'' {
s.tokenFlags |= ast.TokenFlagsSingleQuote
}
s.pos++
// Fast path for simple strings without escape sequences.
strLen := strings.IndexByte(s.text[s.pos:], byte(quote))
if strLen == 0 {
s.pos++
return ""
}
if strLen > 0 {
str := s.text[s.pos : s.pos+strLen]
if jsxAttributeString ||
strings.IndexByte(str, '\\') < 0 && strings.IndexByte(str, '\r') < 0 && strings.IndexByte(str, '\n') < 0 {
s.pos += strLen + 1
return str
}
}
var sb strings.Builder
start := s.pos
for {
ch := s.char()
if ch < 0 {
sb.WriteString(s.text[start:s.pos])
s.tokenFlags |= ast.TokenFlagsUnterminated
s.error(diagnostics.Unterminated_string_literal)
break
}
if ch == quote {
sb.WriteString(s.text[start:s.pos])
s.pos++
break
}
if ch == '\\' && !jsxAttributeString {
sb.WriteString(s.text[start:s.pos])
sb.WriteString(s.scanEscapeSequence(EscapeSequenceScanningFlagsString | EscapeSequenceScanningFlagsReportErrors))
start = s.pos
continue
}
if (ch == '\n' || ch == '\r') && !jsxAttributeString {
sb.WriteString(s.text[start:s.pos])
s.tokenFlags |= ast.TokenFlagsUnterminated
s.error(diagnostics.Unterminated_string_literal)
break
}
s.pos++
}
return sb.String()
}
func (s *Scanner) scanTemplateAndSetTokenValue(shouldEmitInvalidEscapeError bool) ast.Kind {
startedWithBacktick := s.char() == '`'
s.pos++
start := s.pos
parts := make([]string, 0, 4)
var token ast.Kind
for {
s.scanASCIIWhile(func(b byte) bool {
return b != '`' && b != '$' && b != '\\' && b != '\r'
})
ch := s.char()
if ch < 0 || ch == '`' {
parts = append(parts, s.text[start:s.pos])
if ch == '`' {
s.pos++
} else {
s.tokenFlags |= ast.TokenFlagsUnterminated
s.error(diagnostics.Unterminated_template_literal)
}
token = core.IfElse(startedWithBacktick, ast.KindNoSubstitutionTemplateLiteral, ast.KindTemplateTail)
break
}
if ch == '$' && s.charAt(1) == '{' {
parts = append(parts, s.text[start:s.pos])
s.pos += 2
token = core.IfElse(startedWithBacktick, ast.KindTemplateHead, ast.KindTemplateMiddle)
break
}
if ch == '\\' {
parts = append(parts, s.text[start:s.pos])
parts = append(parts, s.scanEscapeSequence(EscapeSequenceScanningFlagsString|core.IfElse(shouldEmitInvalidEscapeError, EscapeSequenceScanningFlagsReportErrors, 0)))
start = s.pos
continue
}
// Speculated ECMAScript 6 Spec 11.8.6.1:
// and LineTerminatorSequences are normalized to for Template Values
if ch == '\r' {
parts = append(parts, s.text[start:s.pos])
s.pos++
if s.char() == '\n' {
s.pos++
}
parts = append(parts, "\n")
start = s.pos
continue
}
s.pos++
}
s.tokenValue = strings.Join(parts, "")
return token
}
func (s *Scanner) scanEscapeSequence(flags EscapeSequenceScanningFlags) string {
start := s.pos
s.pos++
ch := s.char()
if ch < 0 {
s.error(diagnostics.Unexpected_end_of_text)
return ""
}
s.pos++
switch ch {
case '0':
// Although '0' preceding any digit is treated as LegacyOctalEscapeSequence,
// '\08' should separately be interpreted as '\0' + '8'.
if !stringutil.IsDigit(s.char()) {
return "\x00"
}
// '\01', '\011'
fallthrough
case '1', '2', '3':
// '\1', '\17', '\177'
if stringutil.IsOctalDigit(s.char()) {
s.pos++
}
// '\17', '\177'
fallthrough
case '4', '5', '6', '7':
// '\4', '\47' but not '\477'
if stringutil.IsOctalDigit(s.char()) {
s.pos++
}
// '\47'
s.tokenFlags |= ast.TokenFlagsContainsInvalidEscape
if flags&EscapeSequenceScanningFlagsReportInvalidEscapeErrors != 0 {
code, _ := strconv.ParseInt(s.text[start+1:s.pos], 8, 32)
if flags&EscapeSequenceScanningFlagsRegularExpression != 0 && flags&EscapeSequenceScanningFlagsAtomEscape == 0 && ch != '0' {
s.errorAt(diagnostics.Octal_escape_sequences_and_backreferences_are_not_allowed_in_a_character_class_If_this_was_intended_as_an_escape_sequence_use_the_syntax_0_instead, start, s.pos-start, fmt.Sprintf("\\x%02x", code))
} else {
s.errorAt(diagnostics.Octal_escape_sequences_are_not_allowed_Use_the_syntax_0, start, s.pos-start, fmt.Sprintf("\\x%02x", code))
}
return string(rune(code))
}
return s.text[start:s.pos]
case '8', '9':
// the invalid '\8' and '\9'
s.tokenFlags |= ast.TokenFlagsContainsInvalidEscape
if flags&EscapeSequenceScanningFlagsReportInvalidEscapeErrors != 0 {
if flags&EscapeSequenceScanningFlagsRegularExpression != 0 && flags&EscapeSequenceScanningFlagsAtomEscape == 0 {
s.errorAt(diagnostics.Decimal_escape_sequences_and_backreferences_are_not_allowed_in_a_character_class, start, s.pos-start)
} else {
s.errorAt(diagnostics.Escape_sequence_0_is_not_allowed, start, s.pos-start, s.text[start:s.pos])
}
return string(ch)
}
return s.text[start:s.pos]
case 'b':
return "\b"
case 't':
return "\t"
case 'n':
return "\n"
case 'v':
return "\v"
case 'f':
return "\f"
case 'r':
return "\r"
case '\'':
return "'"
case '"':
return "\""
case 'u':
// '\uDDDD' and '\u{DDDDDD}'
extended := s.char() == '{'
s.pos -= 2
codePoint := s.scanUnicodeEscape(flags&EscapeSequenceScanningFlagsReportInvalidEscapeErrors != 0)
if extended {
if flags&EscapeSequenceScanningFlagsAllowExtendedUnicodeEscape == 0 {
s.tokenFlags |= ast.TokenFlagsContainsInvalidEscape
if flags&EscapeSequenceScanningFlagsReportInvalidEscapeErrors != 0 {
s.errorAt(diagnostics.Unicode_escape_sequences_are_only_available_when_the_Unicode_u_flag_or_the_Unicode_Sets_v_flag_is_set, start, s.pos-start)
}
}
if codePoint < 0 {
return s.text[start:s.pos]
}
// In string literals, a high surrogate \u{...} followed by a low
// surrogate escape forms a single code point, exactly as adjacent
// UTF-16 code units would in a JavaScript string.
if flags&EscapeSequenceScanningFlagsRegularExpression == 0 && stringutil.IsHighSurrogate(codePoint) {
if combined, ok := s.scanLowSurrogateEscape(codePoint); ok {
return string(combined)
}
}
return stringutil.EncodeJSStringRune(codePoint)
}
if codePoint < 0 {
return s.text[start:s.pos]
} else if stringutil.IsHighSurrogate(codePoint) {
if flags&EscapeSequenceScanningFlagsRegularExpression == 0 {
// Combine \uHigh followed by any low surrogate escape (\uLow or
// \u{Low}) into a single code point in string literals, matching
// how adjacent UTF-16 code units pair in a JavaScript string.
if combined, ok := s.scanLowSurrogateEscape(codePoint); ok {
return string(combined)
}
} else if flags&EscapeSequenceScanningFlagsAnyUnicodeMode != 0 &&
s.char() == '\\' && s.charAt(1) == 'u' && s.charAt(2) != '{' {
// In regex AnyUnicodeMode, combine \uHigh\uLow so scanClassRanges
// can compare the pair numerically. In non-unicode regex mode they
// are separate atoms, and extended \u{...} escapes never combine.
savedPos := s.pos
nextCodePoint := s.scanUnicodeEscape(flags&EscapeSequenceScanningFlagsReportInvalidEscapeErrors != 0)
if stringutil.IsLowSurrogate(nextCodePoint) {
return string(stringutil.SurrogatePairToCodePoint(codePoint, nextCodePoint))
}
s.pos = savedPos
}
}
// Lone surrogate: encode as CESU-8 so it survives losslessly. In a
// non-unicode regex this also lets scanClassRanges compare it numerically.
return stringutil.EncodeJSStringRune(codePoint)
case 'x':
// '\xDD'
for ; s.pos < start+4; s.pos++ {
if !stringutil.IsHexDigit(s.char()) {
s.tokenFlags |= ast.TokenFlagsContainsInvalidEscape
if flags&EscapeSequenceScanningFlagsReportInvalidEscapeErrors != 0 {
s.error(diagnostics.Hexadecimal_digit_expected)
}
return s.text[start:s.pos]
}
}
s.tokenFlags |= ast.TokenFlagsHexEscape
escapedValue, _ := strconv.ParseInt(s.text[start+2:s.pos], 16, 32)
return string(rune(escapedValue))
case '\r':
// when encountering a LineContinuation (i.e. a backslash and a line terminator sequence),
// the line terminator is interpreted to be "the empty code unit sequence".
if s.char() == '\n' {
s.pos++
}
fallthrough
case '\n':
return ""
default:
// ch was read as a single byte; for multi-byte UTF-8 characters,
// we need to decode the full rune and advance past all its bytes.
if ch >= utf8.RuneSelf {
s.pos-- // back up past the single-byte advance
var size int
ch, size = utf8.DecodeRuneInString(s.text[s.pos:])
s.pos += size
}
// LineContinuation: a backslash followed by a line terminator is "the empty code unit sequence".
if ch == '\u2028' || ch == '\u2029' {
return ""
}
if flags&EscapeSequenceScanningFlagsAnyUnicodeMode != 0 || flags&EscapeSequenceScanningFlagsRegularExpression != 0 && flags&EscapeSequenceScanningFlagsAnnexB == 0 && IsIdentifierPart(ch) {
s.errorAt(diagnostics.This_character_cannot_be_escaped_in_a_regular_expression, start, s.pos-start)
}
return string(ch)
}
}
// Known to be at \u
func (s *Scanner) scanUnicodeEscape(shouldEmitInvalidEscapeError bool) rune {
s.pos += 2
start := s.pos
extended := s.char() == '{'
var hexDigits string
if extended {
s.pos++
hexDigits = s.scanHexDigits(1, true, false)
} else {
s.tokenFlags |= ast.TokenFlagsUnicodeEscape
hexDigits = s.scanHexDigits(4, false, false)
}
if hexDigits == "" {
s.tokenFlags |= ast.TokenFlagsContainsInvalidEscape
if shouldEmitInvalidEscapeError {
s.error(diagnostics.Hexadecimal_digit_expected)
}
return -1
}
hexValue, _ := strconv.ParseInt(hexDigits, 16, 32)
if extended {
isInvalidExtendedEscape := false
if hexValue > 0x10FFFF {
if shouldEmitInvalidEscapeError {
s.errorAt(diagnostics.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive, start+1, s.pos-start-1)
}
isInvalidExtendedEscape = true
}
if s.pos >= s.end {
if shouldEmitInvalidEscapeError {
s.error(diagnostics.Unexpected_end_of_text)
}
isInvalidExtendedEscape = true
} else if s.char() == '}' {
s.pos++
} else {
if shouldEmitInvalidEscapeError {
s.error(diagnostics.Unterminated_Unicode_escape_sequence)
}
isInvalidExtendedEscape = true
}
if isInvalidExtendedEscape {
s.tokenFlags |= ast.TokenFlagsContainsInvalidEscape
return -1
}
s.tokenFlags |= ast.TokenFlagsExtendedUnicodeEscape
}
return rune(hexValue)
}
// scanLowSurrogateEscape attempts to consume a low-surrogate Unicode escape
// (either '\uLow' or '\u{Low}') immediately following an already-scanned high
// surrogate and combine them into a single supplementary code point. This
// mirrors how adjacent UTF-16 code units form a surrogate pair in a JavaScript
// string, regardless of which escape syntax produced each half. On success it
// returns the combined code point and true; otherwise it restores the scanner
// position and returns false.
func (s *Scanner) scanLowSurrogateEscape(high rune) (rune, bool) {
if s.char() != '\\' || s.charAt(1) != 'u' {
return 0, false
}
savedPos := s.pos
savedTokenFlags := s.tokenFlags
// Speculatively scan the escape with diagnostics suppressed: if it isn't a
// low surrogate we rewind below, and the caller re-scans the same escape and
// reports any error then, so reporting here would duplicate diagnostics.
low := s.scanUnicodeEscape(false)
if stringutil.IsLowSurrogate(low) {
return stringutil.SurrogatePairToCodePoint(high, low), true
}
s.pos = savedPos
s.tokenFlags = savedTokenFlags
return 0, false
}
// Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX'
// or '\u{XXXXXX}' and return code point value if valid Unicode escape is found. Otherwise return -1.
func (s *Scanner) peekUnicodeEscape() rune {
if s.charAt(1) == 'u' {
savePos := s.pos
saveTokenFlags := s.tokenFlags
codePoint := s.scanUnicodeEscape(false)
s.pos = savePos
s.tokenFlags = saveTokenFlags
return codePoint
}
return -1
}
func (s *Scanner) scanNumber() ast.Kind {
start := s.pos
var fixedPart string
if s.char() == '0' {
s.pos++
if s.char() == '_' {
s.tokenFlags |= ast.TokenFlagsContainsSeparator | ast.TokenFlagsContainsInvalidSeparator
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos, 1)
s.pos = start
fixedPart = s.scanNumberFragment()
} else {
digits, isOctal := s.scanDigits()
if digits == "" {
fixedPart = "0"
} else if !isOctal {
s.tokenFlags |= ast.TokenFlagsContainsLeadingZero
fixedPart = digits
} else {
val, _ := strconv.ParseInt(digits, 8, 64)
s.tokenValue = strconv.FormatInt(val, 10)
s.tokenFlags |= ast.TokenFlagsOctal
withMinus := s.token == ast.KindMinusToken
literal := core.IfElse(withMinus, "-", "") + "0o" + strconv.FormatInt(val, 8)
if withMinus {
start--
}
s.errorAt(diagnostics.Octal_literals_are_not_allowed_Use_the_syntax_0, start, s.pos-start, literal)
return ast.KindNumericLiteral
}
}
} else {
fixedPart = s.scanNumberFragment()
}
fixedPartEnd := s.pos
fractionalPart := ""
exponentPreamble := ""
exponentPart := ""
if s.char() == '.' {
s.pos++
fractionalPart = s.scanNumberFragment()
}
end := s.pos
if s.char() == 'E' || s.char() == 'e' {
s.pos++
s.tokenFlags |= ast.TokenFlagsScientific
if s.char() == '+' || s.char() == '-' {
s.pos++
}
startNumericPart := s.pos
exponentPart = s.scanNumberFragment()
if exponentPart == "" {
s.error(diagnostics.Digit_expected)
} else {
exponentPreamble = s.text[end:startNumericPart]
end = s.pos
}
}
if s.tokenFlags&ast.TokenFlagsContainsSeparator != 0 {
s.tokenValue = fixedPart
if fractionalPart != "" {
s.tokenValue += "." + fractionalPart
}
if exponentPart != "" {
s.tokenValue += exponentPreamble + exponentPart
}
} else {
s.tokenValue = s.text[start:end]
}
if s.tokenFlags&ast.TokenFlagsContainsLeadingZero != 0 {
s.errorAt(diagnostics.Decimals_with_leading_zeros_are_not_allowed, start, s.pos-start)
s.tokenValue = jsnum.FromString(s.tokenValue).String()
return ast.KindNumericLiteral
}
var result ast.Kind
if fixedPartEnd == s.pos {
result = s.scanBigIntSuffix()
} else {
s.tokenValue = jsnum.FromString(s.tokenValue).String()
result = ast.KindNumericLiteral
}
ch, _ := s.charAndSize()
if IsIdentifierStart(ch) {
idStart := s.pos
id := s.scanIdentifierParts()
if result != ast.KindBigIntLiteral && len(id) == 1 && s.text[idStart] == 'n' {
if s.tokenFlags&ast.TokenFlagsScientific != 0 {
s.errorAt(diagnostics.A_bigint_literal_cannot_use_exponential_notation, start, s.pos-start)
return result
}
if fixedPartEnd < idStart {
s.errorAt(diagnostics.A_bigint_literal_must_be_an_integer, start, s.pos-start)
return result
}
}
s.errorAt(diagnostics.An_identifier_or_keyword_cannot_immediately_follow_a_numeric_literal, idStart, s.pos-idStart)
s.pos = idStart
}
return result
}
func (s *Scanner) scanNumberFragment() string {
start := s.pos
allowSeparator := false
isPreviousTokenSeparator := false
var result strings.Builder
for {
before := s.pos
s.scanASCIIWhile(func(b byte) bool {
return b >= '0' && b <= '9'
})
if s.pos > before {
allowSeparator = true
isPreviousTokenSeparator = false
}
ch := s.char()
if ch == '_' {
s.tokenFlags |= ast.TokenFlagsContainsSeparator
if allowSeparator {
allowSeparator = false
isPreviousTokenSeparator = true
result.WriteString(s.text[start:s.pos])
} else {
s.tokenFlags |= ast.TokenFlagsContainsInvalidSeparator
if isPreviousTokenSeparator {
s.errorAt(diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, s.pos, 1)
} else {
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos, 1)
}
}
s.pos++
start = s.pos
continue
}
break
}
if isPreviousTokenSeparator {
s.tokenFlags |= ast.TokenFlagsContainsInvalidSeparator
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos-1, 1)
}
if result.Len() == 0 {
return s.text[start:s.pos]
}
result.WriteString(s.text[start:s.pos])
return result.String()
}
func (s *Scanner) scanDigits() (string, bool) {
start := s.pos
isOctal := true
for stringutil.IsDigit(s.char()) {
if !stringutil.IsOctalDigit(s.char()) {
isOctal = false
}
s.pos++
}
return s.text[start:s.pos], isOctal
}
func (s *Scanner) scanHexDigits(minCount int, scanAsManyAsPossible bool, canHaveSeparators bool) string {
digitCount := 0
start := s.pos
allowSeparator := false
isPreviousTokenSeparator := false
for digitCount < minCount || scanAsManyAsPossible {
ch := s.char()
if stringutil.IsHexDigit(ch) {
allowSeparator = canHaveSeparators
isPreviousTokenSeparator = false
digitCount++
} else if canHaveSeparators && ch == '_' {
s.tokenFlags |= ast.TokenFlagsContainsSeparator
if allowSeparator {
allowSeparator = false
isPreviousTokenSeparator = true
} else if isPreviousTokenSeparator {
s.errorAt(diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, s.pos, 1)
} else {
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos, 1)
}
} else {
break
}
s.pos++
}
if isPreviousTokenSeparator {
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos-1, 1)
}
if digitCount < minCount {
return ""
}
digits := s.text[start:s.pos]
if s.hexDigitCache == nil {
s.hexDigitCache = make(map[string]string)
}
if cached, ok := s.hexDigitCache[digits]; ok {
return cached
} else {
original := digits
if s.tokenFlags&ast.TokenFlagsContainsSeparator != 0 {
digits = strings.ReplaceAll(digits, "_", "")
}
digits = strings.ToLower(digits) // standardize hex literals to lowercase
s.hexDigitCache[original] = digits
return digits
}
}
func (s *Scanner) scanBinaryOrOctalDigits(base int32) string {
var sb strings.Builder
allowSeparator := false
isPreviousTokenSeparator := false
for {
ch := s.char()
if stringutil.IsDigit(ch) && ch-'0' < base {
sb.WriteByte(byte(ch))
allowSeparator = true
isPreviousTokenSeparator = false
} else if ch == '_' {
s.tokenFlags |= ast.TokenFlagsContainsSeparator
if allowSeparator {
allowSeparator = false
isPreviousTokenSeparator = true
} else if isPreviousTokenSeparator {
s.errorAt(diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, s.pos, 1)
} else {
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos, 1)
}
} else {
break
}
s.pos++
}
if isPreviousTokenSeparator {
s.errorAt(diagnostics.Numeric_separators_are_not_allowed_here, s.pos-1, 1)
}
return sb.String()
}
func (s *Scanner) scanBigIntSuffix() ast.Kind {
if s.char() == 'n' {
s.tokenValue += "n"
if s.tokenFlags&ast.TokenFlagsBinaryOrOctalSpecifier != 0 {
s.tokenValue = jsnum.ParsePseudoBigInt(s.tokenValue) + "n"
}
s.pos++
return ast.KindBigIntLiteral
}
if s.numberCache == nil {
s.numberCache = make(map[string]string)
}
if cached, ok := s.numberCache[s.tokenValue]; ok {
s.tokenValue = cached
} else {
tokenValue := jsnum.FromString(s.tokenValue).String()
if tokenValue == s.tokenValue {
tokenValue = s.tokenValue
}
s.numberCache[s.tokenValue] = tokenValue
s.tokenValue = tokenValue
}
return ast.KindNumericLiteral
}
func (s *Scanner) scanInvalidCharacter() {
_, size := s.charAndSize()
s.errorAt(diagnostics.Invalid_character, s.pos, size)
s.pos += size
s.token = ast.KindUnknown
}
func GetIdentifierToken(str string) ast.Kind {
if len(str) >= 2 && len(str) <= 12 && str[0] >= 'a' && str[0] <= 'z' {
keyword := textToKeyword[str]
if keyword != ast.KindUnknown {
return keyword
}
}
return ast.KindIdentifier
}
func IsValidIdentifier(s string) bool {
if len(s) == 0 {
return false
}
for i, ch := range s {
if i == 0 && !IsIdentifierStart(ch) || i != 0 && !IsIdentifierPart(ch) {
return false
}
}
return true
}
// Section 6.1.4
func isWordCharacter(ch rune) bool {
return stringutil.IsASCIILetter(ch) || stringutil.IsDigit(ch) || ch == '_'
}
func IsIdentifierStart(ch rune) bool {
return stringutil.IsASCIILetter(ch) || ch == '_' || ch == '$' || ch >= utf8.RuneSelf && stringutil.IsUnicodeIdentifierStart(ch)
}
func IsIdentifierPart(ch rune) bool {
return IsIdentifierPartEx(ch, core.LanguageVariantStandard)
}
func IsIdentifierPartEx(ch rune, languageVariant core.LanguageVariant) bool {
return isWordCharacter(ch) || ch == '$' ||
ch >= utf8.RuneSelf && stringutil.IsUnicodeIdentifierPart(ch) ||
languageVariant == core.LanguageVariantJSX && (ch == '-' || ch == ':') // "-" and ":" are valid in JSX Identifiers
}
var tokenToText = func() [ast.KindCount]string {
var result [ast.KindCount]string
for text, kind := range textToToken {
result[kind] = text
}
return result
}()
func TokenToString(token ast.Kind) string {
return tokenToText[token]
}
func StringToToken(s string) ast.Kind {
kind, ok := textToToken[s]
if ok {
return kind
}
return ast.KindUnknown
}
func GetViableKeywordSuggestions() []string {
result := make([]string, 0, len(textToKeyword))
for text := range textToKeyword {
if len(text) > 2 {
result = append(result, text)
}
}
return result
}
func couldStartTrivia(text string, pos int) bool {
// Keep in sync with skipTrivia
switch ch := text[pos]; ch {
// Characters that could start normal trivia
case '\r', '\n', '\t', '\v', '\f', ' ', '/',
// Characters that could start conflict marker trivia
'<', '|', '=', '>':
return true
case '#':
// Only if its the beginning can we have #! trivia
return pos == 0
default:
return ch > maxAsciiCharacter
}
}
type SkipTriviaOptions struct {
StopAfterLineBreak bool
StopAtComments bool
InJSDoc bool
}
func SkipTrivia(text string, pos int) int {
return SkipTriviaEx(text, pos, nil)
}
func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int {
if ast.PositionIsSynthesized(pos) {
return pos
}
if options == nil {
options = &SkipTriviaOptions{}
}
textLen := len(text)
canConsumeStar := false
// Keep in sync with couldStartTrivia
for {
if pos >= textLen {
return pos
}
ch, size := utf8.DecodeRuneInString(text[pos:])
switch ch {
case '\r':
if pos+1 < textLen && text[pos+1] == '\n' {
pos++
}
fallthrough
case '\n':
pos++
if options.StopAfterLineBreak {
return pos
}
canConsumeStar = options.InJSDoc
continue
case '\t', '\v', '\f', ' ':
pos++
continue
case '/':
if options.StopAtComments {
break
}
if pos+1 < textLen {
if text[pos+1] == '/' {
pos += 2
for pos < textLen {
ch, size := utf8.DecodeRuneInString(text[pos:])
if stringutil.IsLineBreak(ch) {
break
}
pos += size
}
canConsumeStar = false
continue
}
if text[pos+1] == '*' {
pos += 2
for pos < textLen {
if text[pos] == '*' && (pos+1 < textLen) && text[pos+1] == '/' {
pos += 2
break
}
_, size := utf8.DecodeRuneInString(text[pos:])
pos += size
}
canConsumeStar = false
continue
}
}
case '<', '|', '=', '>':
if isConflictMarkerTrivia(text, pos) {
pos = scanConflictMarkerTrivia(text, pos, nil)
canConsumeStar = false
continue
}
case '#':
if pos == 0 && isShebangTrivia(text, pos) {
pos = scanShebangTrivia(text, pos)
canConsumeStar = false
continue
}
case '*':
if canConsumeStar {
pos++
canConsumeStar = false
continue
}
default:
if ch > rune(maxAsciiCharacter) && stringutil.IsWhiteSpaceLike(ch) {
pos += size
continue
}
}
return pos
}
}
// All conflict markers consist of the same character repeated seven times. If it is
// a <<<<<<< or >>>>>>> marker then it is also followed by a space.
var (
mergeConflictMarkerLength = len("<<<<<<<")
maxAsciiCharacter byte = 127
)
func isConflictMarkerTrivia(text string, pos int) bool {
if pos < 0 {
panic("pos < 0")
}
// Fast reject: a conflict marker is the same byte repeated seven times. If the
// second byte differs (the overwhelmingly common case for `<`, `>`, `=`, `|`
// tokens), it cannot be a marker, so skip the line-start check entirely.
if pos+1 >= len(text) || text[pos+1] != text[pos] {
return false
}
// Conflict markers must be at the start of a line.
atLineStart := pos == 0 || stringutil.IsLineBreak(rune(text[pos-1]))
if !atLineStart && pos >= 2 {
prev, _ := utf8.DecodeLastRuneInString(text[:pos-2])
atLineStart = stringutil.IsLineBreak(prev)
}
if atLineStart {
ch := text[pos]
if (pos + mergeConflictMarkerLength) < len(text) {
for i := range mergeConflictMarkerLength {
if text[pos+i] != ch {
return false
}
}
return ch == '=' || text[pos+mergeConflictMarkerLength] == ' '
}
}
return false
}
func scanConflictMarkerTrivia(text string, pos int, reportError func(diag *diagnostics.Message, pos int, length int, args ...any)) int {
if reportError != nil {
reportError(diagnostics.Merge_conflict_marker_encountered, pos, mergeConflictMarkerLength)
}
ch, size := utf8.DecodeRuneInString(text[pos:])
length := len(text)
if ch == '<' || ch == '>' {
for pos < length && !stringutil.IsLineBreak(ch) {
pos += size
ch, size = utf8.DecodeRuneInString(text[pos:])
}
} else {
if ch != '|' && ch != '=' {
panic("Assertion failed: ch must be either '|' or '='")
}
// Consume everything from the start of a ||||||| or ======= marker to the start
// of the next ======= or >>>>>>> marker.
for pos < length {
currentChar := text[pos]
if (currentChar == '=' || currentChar == '>') && rune(currentChar) != ch && isConflictMarkerTrivia(text, pos) {
break
}
pos++
}
}
return pos
}
func isShebangTrivia(text string, pos int) bool {
if len(text) < 2 {
return false
}
if pos != 0 {
panic("Shebangs check must only be done at the start of the file")
}
return text[0] == '#' && text[1] == '!'
}
func scanShebangTrivia(text string, pos int) int {
pos += 2
for pos < len(text) {
ch, size := utf8.DecodeRuneInString(text[pos:])
if stringutil.IsLineBreak(ch) {
break
}
pos += size
}
return pos
}
func GetShebang(text string) string {
if !isShebangTrivia(text, 0) {
return ""
}
end := scanShebangTrivia(text, 0)
return text[:end]
}
func GetScannerForSourceFile(sourceFile *ast.SourceFile, pos int) *Scanner {
s := NewScanner()
s.text = sourceFile.Text()
s.pos = pos
s.end = len(s.text)
s.languageVariant = sourceFile.LanguageVariant
s.Scan()
return s
}
func ScanTokenAtPosition(sourceFile *ast.SourceFile, pos int) ast.Kind {
s := GetScannerForSourceFile(sourceFile, pos)
return s.token
}
func GetRangeOfTokenAtPosition(sourceFile *ast.SourceFile, pos int) core.TextRange {
s := GetScannerForSourceFile(sourceFile, pos)
return core.NewTextRange(s.tokenStart, s.pos)
}
func GetTokenPosOfNode(node *ast.Node, sourceFile *ast.SourceFile, includeJSDoc bool) int {
// With nodes that have no width (i.e. 'Missing' nodes), we actually *don't*
// want to skip trivia because this will launch us forward to the next token.
if ast.NodeIsMissing(node) {
return node.Pos()
}
if ast.IsJSDocNode(node) || node.Kind == ast.KindJsxText {
// JsxText cannot actually contain comments, even though the scanner will think it sees comments
return SkipTriviaEx(sourceFile.Text(), node.Pos(), &SkipTriviaOptions{StopAtComments: true})
}
if includeJSDoc && len(node.JSDoc(sourceFile)) > 0 {
return GetTokenPosOfNode(node.JSDoc(sourceFile)[0], sourceFile, false /*includeJSDoc*/)
}
return SkipTriviaEx(sourceFile.Text(), node.Pos(), &SkipTriviaOptions{InJSDoc: node.Flags&ast.NodeFlagsJSDoc != 0})
}
func getErrorRangeForArrowFunction(sourceFile *ast.SourceFile, node *ast.Node) core.TextRange {
pos := SkipTrivia(sourceFile.Text(), node.Pos())
body := node.Body()
if body != nil && body.Kind == ast.KindBlock {
startLine := GetECMALineOfPosition(sourceFile, body.Pos())
endLine := GetECMALineOfPosition(sourceFile, body.End())
if startLine < endLine {
// The arrow function spans multiple lines, make the error span be the first line, inclusive.
return core.NewTextRange(pos, GetECMAEndLinePosition(sourceFile, startLine)+1)
}
}
return core.NewTextRange(pos, node.End())
}
func findOriginatingJSDocSatisfiesTag(sourceFile *ast.SourceFile, node *ast.Node) *ast.Node {
targetType := node.AsSatisfiesExpression().Type
if targetType.Flags&ast.NodeFlagsReparsed == 0 {
return nil
}
for current := node.Parent; current != nil; current = current.Parent {
if current.Flags&ast.NodeFlagsHasJSDoc == 0 {
continue
}
var firstSatisfiesTag *ast.Node
for _, jsDoc := range current.EagerJSDoc(sourceFile) {
if tags := jsDoc.AsJSDoc().Tags; tags != nil {
for _, tag := range tags.Nodes {
if !ast.IsJSDocSatisfiesTag(tag) {
continue
}
if firstSatisfiesTag == nil {
firstSatisfiesTag = tag
}
if typeExpr := tag.AsJSDocSatisfiesTag().TypeExpression; typeExpr != nil {
if t := typeExpr.Type(); t != nil && t.Loc == targetType.Loc {
return tag
}
}
}
}
}
return firstSatisfiesTag
}
return nil
}
func GetErrorRangeForNode(sourceFile *ast.SourceFile, node *ast.Node) core.TextRange {
errorNode := node
switch node.Kind {
case ast.KindSourceFile:
pos := SkipTrivia(sourceFile.Text(), 0)
if pos == len(sourceFile.Text()) {
return core.NewTextRange(0, 0)
}
return GetRangeOfTokenAtPosition(sourceFile, pos)
// This list is a work in progress. Add missing node kinds to improve their error spans
case ast.KindFunctionDeclaration, ast.KindMethodDeclaration:
if node.Flags&ast.NodeFlagsReparsed != 0 {
errorNode = node
break
}
fallthrough
case ast.KindVariableDeclaration, ast.KindBindingElement, ast.KindClassDeclaration, ast.KindInterfaceDeclaration,
ast.KindModuleDeclaration, ast.KindEnumDeclaration, ast.KindEnumMember, ast.KindFunctionExpression,
ast.KindGetAccessor, ast.KindSetAccessor, ast.KindTypeAliasDeclaration, ast.KindJSTypeAliasDeclaration, ast.KindPropertyDeclaration,
ast.KindPropertySignature, ast.KindNamespaceImport:
errorNode = ast.GetNameOfDeclaration(node)
case ast.KindClassExpression:
errorNode = node.Name()
case ast.KindArrowFunction:
return getErrorRangeForArrowFunction(sourceFile, node)
case ast.KindCaseClause, ast.KindDefaultClause:
start := SkipTrivia(sourceFile.Text(), node.Pos())
end := node.End()
statements := node.Statements()
if len(statements) != 0 {
end = statements[0].Pos()
}
return core.NewTextRange(start, end)
case ast.KindReturnStatement, ast.KindYieldExpression:
pos := SkipTrivia(sourceFile.Text(), node.Pos())
return GetRangeOfTokenAtPosition(sourceFile, pos)
case ast.KindSatisfiesExpression:
if jsDocSatisfiesTag := findOriginatingJSDocSatisfiesTag(sourceFile, node); jsDocSatisfiesTag != nil {
pos := SkipTrivia(sourceFile.Text(), jsDocSatisfiesTag.TagName().Pos())
return GetRangeOfTokenAtPosition(sourceFile, pos)
}
pos := SkipTrivia(sourceFile.Text(), node.AsSatisfiesExpression().Expression.End())
return GetRangeOfTokenAtPosition(sourceFile, pos)
case ast.KindConstructor:
if node.Flags&ast.NodeFlagsReparsed != 0 {
errorNode = node
break
}
scanner := GetScannerForSourceFile(sourceFile, node.Pos())
start := scanner.TokenStart()
for scanner.Token() != ast.KindConstructorKeyword && scanner.Token() != ast.KindStringLiteral && scanner.Token() != ast.KindEndOfFile {
scanner.Scan()
}
return core.NewTextRange(start, scanner.TokenEnd())
}
if errorNode == nil {
// If we don't have a better node, then just set the error on the first token of
// construct.
return GetRangeOfTokenAtPosition(sourceFile, node.Pos())
}
pos := errorNode.Pos()
if !ast.NodeIsMissing(errorNode) && !ast.IsJsxText(errorNode) {
pos = SkipTrivia(sourceFile.Text(), pos)
}
return core.NewTextRange(pos, errorNode.End())
}
func ComputeLineOfPosition(lineStarts []core.TextPos, pos int) int {
low := 0
high := len(lineStarts) - 1
for low <= high {
middle := low + ((high - low) >> 1)
value := int(lineStarts[middle])
if value < pos {
low = middle + 1
} else if value > pos {
high = middle - 1
} else {
return middle
}
}
return low - 1
}
func GetECMALineStarts(sourceFile ast.SourceFileLike) []core.TextPos {
return sourceFile.ECMALineMap()
}
func GetECMALineOfPosition(sourceFile ast.SourceFileLike, pos int) int {
lineMap := GetECMALineStarts(sourceFile)
return ComputeLineOfPosition(lineMap, pos)
}
// GetECMALineAndUTF16CharacterOfPosition returns the 0-based line number and the
// UTF-16 code unit offset from the start of that line for the given byte position.
// Uses ECMAScript line separators (LF, CR, CRLF, LS, PS).
func GetECMALineAndUTF16CharacterOfPosition(sourceFile ast.SourceFileLike, pos int) (line int, character core.UTF16Offset) {
lineMap := GetECMALineStarts(sourceFile)
line = ComputeLineOfPosition(lineMap, pos)
character = core.UTF16Len(sourceFile.Text()[lineMap[line]:pos])
return line, character
}
// GetECMALineAndByteOffsetOfPosition returns the 0-based line number and the
// raw UTF-8 byte offset from the start of that line for the given byte position.
// Uses ECMAScript line separators (LF, CR, CRLF, LS, PS).
// Unlike GetECMALineAndUTF16CharacterOfPosition, the offset is in bytes, not UTF-16 code units.
func GetECMALineAndByteOffsetOfPosition(sourceFile ast.SourceFileLike, pos int) (line int, byteOffset int) {
lineMap := GetECMALineStarts(sourceFile)
line = ComputeLineOfPosition(lineMap, pos)
byteOffset = pos - int(lineMap[line])
return line, byteOffset
}
func GetECMAEndLinePosition(sourceFile *ast.SourceFile, line int) int {
pos := int(GetECMALineStarts(sourceFile)[line])
for {
ch, size := utf8.DecodeRuneInString(sourceFile.Text()[pos:])
if size == 0 || stringutil.IsLineBreak(ch) {
return pos - 1
}
pos += size
}
}
// GetECMAPositionOfLineAndUTF16Character converts a 0-based line number and UTF-16
// code unit character offset back to an absolute byte position in the source text.
// Uses ECMAScript line separators.
func GetECMAPositionOfLineAndUTF16Character(sourceFile ast.SourceFileLike, line int, character core.UTF16Offset) int {
lineStarts := GetECMALineStarts(sourceFile)
return ComputePositionOfLineAndUTF16Character(lineStarts, line, character, sourceFile.Text(), false)
}
// GetECMAPositionOfLineAndByteOffset converts a 0-based line number and byte offset
// from line start back to an absolute byte position in the source text.
// Uses ECMAScript line separators.
func GetECMAPositionOfLineAndByteOffset(sourceFile ast.SourceFileLike, line int, byteOffset int) int {
return ComputePositionOfLineAndByteOffset(GetECMALineStarts(sourceFile), line, byteOffset)
}
// ComputePositionOfLineAndByteOffset computes a byte position from a line and
// raw byte offset from the line start. This is a simple addition with validation.
func ComputePositionOfLineAndByteOffset(lineStarts []core.TextPos, line int, byteOffset int) int {
if line < 0 || line >= len(lineStarts) {
panic(fmt.Sprintf("Bad line number. Line: %d, lineStarts.length: %d.", line, len(lineStarts)))
}
return int(lineStarts[line]) + byteOffset
}
// ComputePositionOfLineAndUTF16Character converts a line and UTF-16 character offset
// back to a byte position. The character parameter is measured in UTF-16 code units.
// It scans from the line start to correctly handle multi-byte characters.
// When allowEdits is true, out-of-range values are clamped instead of panicking.
func ComputePositionOfLineAndUTF16Character(lineStarts []core.TextPos, line int, character core.UTF16Offset, text string, allowEdits bool) int {
if line < 0 || line >= len(lineStarts) {
if allowEdits {
// Clamp line to nearest allowable value
if line < 0 {
line = 0
} else if line >= len(lineStarts) {
line = len(lineStarts) - 1
}
} else {
panic(fmt.Sprintf("Bad line number. Line: %d, lineStarts.length: %d.", line, len(lineStarts)))
}
}
lineStart := int(lineStarts[line])
if character > 0 {
// UTF-16 character offset: scan from line start counting UTF-16 code units.
lineEnd := len(text)
if line+1 < len(lineStarts) {
lineEnd = int(lineStarts[line+1])
}
utf16Count := core.UTF16Offset(0)
pos := lineStart
for pos < lineEnd {
if utf16Count >= character {
break
}
r, size := utf8.DecodeRuneInString(text[pos:])
utf16Count += core.UTF16Offset(utf16.RuneLen(r))
pos += size
}
if !allowEdits {
if pos == lineEnd && utf16Count < character {
panic(fmt.Sprintf("Bad UTF-16 character offset. Line: %d, character: %d.", line, character))
}
debug.Assert(pos <= len(text))
return pos
}
if pos > len(text) {
return len(text)
}
return pos
}
// Character is 0: line start position.
res := lineStart
if allowEdits {
if res > len(text) {
return len(text)
}
return res
}
debug.Assert(res <= len(text)) // Allow single character overflow for trailing newline
return res
}
func GetLeadingCommentRanges(f *ast.NodeFactory, text string, pos int) iter.Seq[ast.CommentRange] {
return iterateCommentRanges(f, text, pos, false)
}
func GetTrailingCommentRanges(f *ast.NodeFactory, text string, pos int) iter.Seq[ast.CommentRange] {
return iterateCommentRanges(f, text, pos, true)
}
/*
Returns an iterator over each comment range following the provided position.
Single-line comment ranges include the leading double-slash characters but not the ending
line break. Multi-line comment ranges include the leading slash-asterisk and trailing
asterisk-slash characters.
*/
func iterateCommentRanges(f *ast.NodeFactory, text string, pos int, trailing bool) iter.Seq[ast.CommentRange] {
return func(yield func(ast.CommentRange) bool) {
var pendingPos int
var pendingEnd int
var pendingKind ast.Kind
var pendingHasTrailingNewLine bool
hasPendingCommentRange := false
collecting := trailing
if pos == 0 {
collecting = true
if isShebangTrivia(text, pos) {
pos = scanShebangTrivia(text, pos)
}
}
scan:
for pos >= 0 && pos < len(text) {
ch, size := utf8.DecodeRuneInString(text[pos:])
switch ch {
case '\r':
if pos+1 < len(text) && text[pos+1] == '\n' {
pos++
}
fallthrough
case '\n':
pos++
if trailing {
break scan
}
collecting = true
if hasPendingCommentRange {
pendingHasTrailingNewLine = true
}
continue
case '\t', '\v', '\f', ' ':
pos++
continue
case '/':
var nextChar byte
if pos+1 < len(text) {
nextChar = text[pos+1]
}
hasTrailingNewLine := false
if nextChar == '/' || nextChar == '*' {
var kind ast.Kind
if nextChar == '/' {
kind = ast.KindSingleLineCommentTrivia
} else {
kind = ast.KindMultiLineCommentTrivia
}
startPos := pos
pos += 2
if nextChar == '/' {
for pos < len(text) {
c, s := utf8.DecodeRuneInString(text[pos:])
if stringutil.IsLineBreak(c) {
hasTrailingNewLine = true
break
}
pos += s
}
} else {
if i := strings.Index(text[pos:], "*/"); i >= 0 {
pos += i + 2
} else {
pos = len(text)
}
}
if collecting {
if hasPendingCommentRange {
if !yield(f.NewCommentRange(pendingKind, pendingPos, pendingEnd, pendingHasTrailingNewLine)) {
return
}
}
pendingPos = startPos
pendingEnd = pos
pendingKind = kind
pendingHasTrailingNewLine = hasTrailingNewLine
hasPendingCommentRange = true
}
continue
}
break scan
default:
if ch > unicode.MaxASCII && stringutil.IsWhiteSpaceLike(ch) {
if hasPendingCommentRange && stringutil.IsLineBreak(ch) {
pendingHasTrailingNewLine = true
}
pos += size
continue
}
break scan
}
}
if hasPendingCommentRange {
yield(f.NewCommentRange(pendingKind, pendingPos, pendingEnd, pendingHasTrailingNewLine))
}
}
}