fix ampersand ordering bug for fuzzy matching
This commit is contained in:
@@ -37,7 +37,12 @@ const FUZZY_UNMATCHED_PENALTY = -1;
|
|||||||
const FUZZY_RECURSION_LIMIT = 10;
|
const FUZZY_RECURSION_LIMIT = 10;
|
||||||
const FUZZY_TRANSPOSE_PENALTY = -20;
|
const FUZZY_TRANSPOSE_PENALTY = -20;
|
||||||
const FUZZY_EXACT_SUBSTRING_BONUS = 100;
|
const FUZZY_EXACT_SUBSTRING_BONUS = 100;
|
||||||
const FUZZY_AND_AMPERSAND_PENALTY = -20;
|
// Per swapped "&" <-> "and" token. Each swap lengthens the matched run by two
|
||||||
|
// characters, worth up to 2*FUZZY_SEQUENTIAL_BONUS to the longer "and" spelling;
|
||||||
|
// penalize each swap by more than that so the literal spelling the user typed always
|
||||||
|
// ranks above the substituted one. (This lowers rank, not visibility — the
|
||||||
|
// substituted match still appears in the results.)
|
||||||
|
const FUZZY_AND_AMPERSAND_PENALTY = -(2 * FUZZY_SEQUENTIAL_BONUS + 20); // -50
|
||||||
|
|
||||||
// Options that turn on extra, penalized match passes. Off by default so the base
|
// Options that turn on extra, penalized match passes. Off by default so the base
|
||||||
// matcher stays exact-only.
|
// matcher stays exact-only.
|
||||||
@@ -103,17 +108,25 @@ export function fuzzyMatch(query: string, target: string): FuzzyMatchResult | nu
|
|||||||
return { score, positions: matches };
|
return { score, positions: matches };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An alternative spelling of the query plus the number of tokens swapped to
|
||||||
|
// produce it, so the penalty can scale with how far it strayed.
|
||||||
|
interface FuzzyVariant {
|
||||||
|
text: string;
|
||||||
|
swaps: number;
|
||||||
|
}
|
||||||
|
|
||||||
// Alternative spellings of `query` with the word "and" and the symbol "&" swapped
|
// Alternative spellings of `query` with the word "and" and the symbol "&" swapped
|
||||||
// for each other, excluding `query` itself. `\band\b` only rewrites a standalone
|
// for each other, excluding `query` itself. `\band\b` only rewrites a standalone
|
||||||
// "and", so "brand" and "island" are left alone. These are the candidates the
|
// "and", so "brand" and "island" are left alone. These are the candidates the
|
||||||
// matcher tries (penalized) so "... and ..." can still hit a "... & ..." target
|
// matcher tries (penalized per swap) so "... and ..." can still hit a "... & ..."
|
||||||
// and vice versa, while the exact spelling ranks first.
|
// target and vice versa, while the exact spelling ranks first.
|
||||||
function andAmpersandVariants(query: string): string[] {
|
function andAmpersandVariants(query: string): FuzzyVariant[] {
|
||||||
const out: string[] = [];
|
const out: FuzzyVariant[] = [];
|
||||||
const amp = query.replace(/\band\b/gi, "&");
|
let ands = 0;
|
||||||
if (amp !== query) out.push(amp);
|
const amp = query.replace(/\band\b/gi, () => { ands++; return "&"; });
|
||||||
const and = query.replace(/&/g, "and");
|
if (ands > 0) out.push({ text: amp, swaps: ands });
|
||||||
if (and !== query) out.push(and);
|
const amps = (query.match(/&/g) || []).length;
|
||||||
|
if (amps > 0) out.push({ text: query.replace(/&/g, "and"), swaps: amps });
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,9 +146,9 @@ export function fuzzyMatchTypoTolerant(query: string, target: string, opts?: Fuz
|
|||||||
}
|
}
|
||||||
if (opts?.andAmpersand) {
|
if (opts?.andAmpersand) {
|
||||||
for (const variant of andAmpersandVariants(query)) {
|
for (const variant of andAmpersandVariants(query)) {
|
||||||
const m = fuzzyMatch(variant, target);
|
const m = fuzzyMatch(variant.text, target);
|
||||||
if (!m) continue;
|
if (!m) continue;
|
||||||
const score = m.score + FUZZY_AND_AMPERSAND_PENALTY;
|
const score = m.score + variant.swaps * FUZZY_AND_AMPERSAND_PENALTY;
|
||||||
if (!best || score > best.score) best = { score, positions: m.positions };
|
if (!best || score > best.score) best = { score, positions: m.positions };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,7 +57,12 @@ const (
|
|||||||
fuzzyRecursionLimit = 10
|
fuzzyRecursionLimit = 10
|
||||||
fuzzyTransposePenalty = -20
|
fuzzyTransposePenalty = -20
|
||||||
fuzzyExactSubstrBonus = 100
|
fuzzyExactSubstrBonus = 100
|
||||||
fuzzyAndAmpersandPen = -20
|
// Per swapped "&"<->"and" token. Each swap lengthens the matched run by two
|
||||||
|
// characters, worth up to 2*fuzzySequentialBonus to the longer "and" spelling;
|
||||||
|
// penalize each swap by more than that so the literal spelling the user typed
|
||||||
|
// always ranks above the substituted one. (This lowers rank, not visibility —
|
||||||
|
// the substituted match still appears in the results.)
|
||||||
|
fuzzyAndAmpersandPen = -(2*fuzzySequentialBonus + 20) // -50
|
||||||
)
|
)
|
||||||
|
|
||||||
// FuzzyOptions turns on extra, penalized match passes. The zero value is exact-
|
// FuzzyOptions turns on extra, penalized match passes. The zero value is exact-
|
||||||
@@ -176,10 +181,12 @@ func fuzzyByteAt(s string, i int) byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// fuzzyReplaceAndWord replaces every standalone "and" (case-insensitive) in s
|
// fuzzyReplaceAndWord replaces every standalone "and" (case-insensitive) in s
|
||||||
// with repl, leaving substrings like "brand" or "island" untouched. "and", "&"
|
// with repl, leaving substrings like "brand" or "island" untouched, and reports
|
||||||
// and the ASCII separators are all single-byte, so a byte scan suffices.
|
// how many it replaced. "and", "&" and the ASCII separators are all single-byte,
|
||||||
func fuzzyReplaceAndWord(s, repl string) string {
|
// so a byte scan suffices.
|
||||||
|
func fuzzyReplaceAndWord(s, repl string) (string, int) {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
|
n := 0
|
||||||
for i := 0; i < len(s); {
|
for i := 0; i < len(s); {
|
||||||
isAnd := i+3 <= len(s) &&
|
isAnd := i+3 <= len(s) &&
|
||||||
(s[i] == 'a' || s[i] == 'A') &&
|
(s[i] == 'a' || s[i] == 'A') &&
|
||||||
@@ -187,26 +194,35 @@ func fuzzyReplaceAndWord(s, repl string) string {
|
|||||||
(s[i+2] == 'd' || s[i+2] == 'D')
|
(s[i+2] == 'd' || s[i+2] == 'D')
|
||||||
if isAnd && !fuzzyIsWordByte(fuzzyByteAt(s, i-1)) && !fuzzyIsWordByte(fuzzyByteAt(s, i+3)) {
|
if isAnd && !fuzzyIsWordByte(fuzzyByteAt(s, i-1)) && !fuzzyIsWordByte(fuzzyByteAt(s, i+3)) {
|
||||||
b.WriteString(repl)
|
b.WriteString(repl)
|
||||||
|
n++
|
||||||
i += 3
|
i += 3
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
b.WriteByte(s[i])
|
b.WriteByte(s[i])
|
||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
return b.String()
|
return b.String(), n
|
||||||
|
}
|
||||||
|
|
||||||
|
// fuzzyVariant is an alternative spelling of the query plus the number of tokens
|
||||||
|
// swapped to produce it, so the penalty can scale with how far it strayed.
|
||||||
|
type fuzzyVariant struct {
|
||||||
|
text string
|
||||||
|
swaps int
|
||||||
}
|
}
|
||||||
|
|
||||||
// fuzzyAndAmpersandVariants returns alternative spellings of query with the word
|
// fuzzyAndAmpersandVariants returns alternative spellings of query with the word
|
||||||
// "and" and the symbol "&" swapped for each other, excluding query itself. These
|
// "and" and the symbol "&" swapped for each other, excluding query itself. These
|
||||||
// are the candidates the matcher tries (penalized) so "... and ..." can still hit
|
// are the candidates the matcher tries (penalized per swap) so "... and ..." can
|
||||||
// a "... & ..." target and vice versa, while the exact spelling ranks first.
|
// still hit a "... & ..." target and vice versa, while the exact spelling ranks
|
||||||
func fuzzyAndAmpersandVariants(query string) []string {
|
// first.
|
||||||
var out []string
|
func fuzzyAndAmpersandVariants(query string) []fuzzyVariant {
|
||||||
if amp := fuzzyReplaceAndWord(query, "&"); amp != query {
|
var out []fuzzyVariant
|
||||||
out = append(out, amp)
|
if amp, n := fuzzyReplaceAndWord(query, "&"); n > 0 {
|
||||||
|
out = append(out, fuzzyVariant{amp, n})
|
||||||
}
|
}
|
||||||
if and := strings.ReplaceAll(query, "&", "and"); and != query {
|
if n := strings.Count(query, "&"); n > 0 {
|
||||||
out = append(out, and)
|
out = append(out, fuzzyVariant{strings.ReplaceAll(query, "&", "and"), n})
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
@@ -232,11 +248,11 @@ func FuzzyMatchTypoTolerant(query, target string, opts ...FuzzyOptions) *FuzzyMa
|
|||||||
}
|
}
|
||||||
if fuzzyOpt(opts).AndAmpersand {
|
if fuzzyOpt(opts).AndAmpersand {
|
||||||
for _, variant := range fuzzyAndAmpersandVariants(query) {
|
for _, variant := range fuzzyAndAmpersandVariants(query) {
|
||||||
m := FuzzyMatchOne(variant, target)
|
m := FuzzyMatchOne(variant.text, target)
|
||||||
if m == nil {
|
if m == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
score := m.Score + fuzzyAndAmpersandPen
|
score := m.Score + variant.swaps*fuzzyAndAmpersandPen
|
||||||
if best == nil || score > best.Score {
|
if best == nil || score > best.Score {
|
||||||
best = &FuzzyMatchResult{Score: score, Positions: m.Positions}
|
best = &FuzzyMatchResult{Score: score, Positions: m.Positions}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user