sweet-search 2.4.2 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/cli.js +19 -5
- package/core/embedding/embedding-cache.js +177 -15
- package/core/embedding/embedding-service.js +18 -4
- package/core/graph/graph-expansion.js +52 -12
- package/core/graph/graph-extractor.js +30 -1
- package/core/indexing/ast-chunker.js +331 -16
- package/core/indexing/chunking/chunk-builder.js +34 -1
- package/core/indexing/index.js +6 -3
- package/core/indexing/indexer-ann.js +45 -6
- package/core/indexing/indexer-build.js +9 -1
- package/core/indexing/indexer-phases.js +6 -4
- package/core/indexing/indexing-file-policy.js +140 -0
- package/core/indexing/li-skip-policy.js +11 -220
- package/core/infrastructure/codebase-repository.js +21 -0
- package/core/infrastructure/config/embedding.js +20 -1
- package/core/infrastructure/config/graph.js +2 -2
- package/core/infrastructure/config/ranking.js +10 -0
- package/core/infrastructure/config/vector-store.js +1 -1
- package/core/infrastructure/coreml-cascade.js +236 -30
- package/core/infrastructure/coreml-cascade.json +25 -0
- package/core/infrastructure/index.js +15 -0
- package/core/infrastructure/init-config.js +78 -0
- package/core/infrastructure/language-patterns/registry-core.js +18 -0
- package/core/infrastructure/model-registry.js +12 -0
- package/core/infrastructure/native-inference.js +143 -51
- package/core/infrastructure/tree-sitter-provider.js +92 -2
- package/core/ranking/cascaded-scorer.js +6 -2
- package/core/ranking/file-kind-ranking.js +264 -0
- package/core/ranking/late-interaction-index.js +10 -4
- package/core/ranking/late-interaction-policy.js +304 -0
- package/core/search/context-expander.js +267 -28
- package/core/search/index.js +4 -0
- package/core/search/search-cli.js +3 -1
- package/core/search/search-pattern.js +4 -3
- package/core/search/search-postprocess.js +189 -8
- package/core/search/search-read-semantic.js +717 -0
- package/core/search/search-read.js +481 -0
- package/core/search/search-server.js +6 -4
- package/core/search/sweet-search.js +119 -15
- package/mcp/server.js +41 -0
- package/mcp/tool-handlers.js +117 -6
- package/package.json +9 -7
- package/scripts/init.js +386 -5
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Intent-aware file-kind ranking (conservative variant).
|
|
3
|
+
*
|
|
4
|
+
* Background: real-codebase miss analysis found that documentation, test, and
|
|
5
|
+
* TypeScript-declaration files often outrank the implementation file users
|
|
6
|
+
* were actually looking for on multi-file codebases. The first version of
|
|
7
|
+
* this rule (commit f6fcfd1) lifted graph-2hop R@1 from 47.46 % → 64.41 %
|
|
8
|
+
* but catastrophically regressed GenCodeSearchNet under the dense profile
|
|
9
|
+
* (full-6 000 dense run: MRR@10 84.4 % → 47.4 %, Recall@5 92.0 % → 48.4 %).
|
|
10
|
+
* Root cause: the legacy LI-rerank pipeline assembles
|
|
11
|
+
* `results = [...liScored, ...tail]`, where `liScored` carries MaxSim
|
|
12
|
+
* scores that are sometimes *lower* (in absolute value) than the int8
|
|
13
|
+
* cosine scores already on the un-reranked tail. The concatenated list is
|
|
14
|
+
* therefore not globally score-monotonic. The old helper unconditionally
|
|
15
|
+
* spread and re-sorted *all* results by `score`, which floated the
|
|
16
|
+
* int8-only tail above the LI-reranked head and undid the rerank — even
|
|
17
|
+
* when every multiplier was 1 (GenCodeSearchNet is a single-source
|
|
18
|
+
* corpus, so no docs/tests/types kind ever matches there).
|
|
19
|
+
*
|
|
20
|
+
* Conservative variant fixes both regressions with three guards:
|
|
21
|
+
*
|
|
22
|
+
* 1. Confident-intent gating. `classifyFileKindIntent` now returns
|
|
23
|
+
* `'unknown'` for queries with no implementation-seeking signal. Only
|
|
24
|
+
* explicit `'implementation'` intent triggers demotion. `'unknown'`,
|
|
25
|
+
* `'docs'`, `'tests'`, `'types'` are no-ops.
|
|
26
|
+
*
|
|
27
|
+
* 2. Structural skip. The rule looks at the top-N candidates (default 30).
|
|
28
|
+
* If the window has zero docs/tests/types files (single-source corpus
|
|
29
|
+
* like GCSN) or zero implementation files (nothing to promote), the
|
|
30
|
+
* input is returned untouched. No re-sort, no new objects.
|
|
31
|
+
*
|
|
32
|
+
* 3. Window-bounded re-sort. When the rule does fire, only the top-N
|
|
33
|
+
* window is re-ranked. The tail — where the rerank/non-rerank score-
|
|
34
|
+
* scale boundary usually lives — is concatenated unchanged. This
|
|
35
|
+
* keeps mixed-scale damage contained.
|
|
36
|
+
*
|
|
37
|
+
* Disable at runtime with `SWEET_SEARCH_FILE_KIND_RANKING=0`. Tune the soft
|
|
38
|
+
* factor with `SWEET_SEARCH_FILE_KIND_FACTOR` (default 0.85; range (0, 1]).
|
|
39
|
+
* Tune the window with `SWEET_SEARCH_FILE_KIND_WINDOW` (default 30).
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
const DOCS_RE = /\.md$|\.mdx$|\.rst$|(?:^|\/)docs?\//i;
|
|
43
|
+
const TESTS_RE = /(?:^|\/)tests?\/|(?:^|\/)spec\/|\.test\.[a-z0-9]+$|_test\.[a-z0-9]+$|\.spec\.[a-z0-9]+$|_spec\.[a-z0-9]+$/i;
|
|
44
|
+
const TYPES_RE = /\.d\.ts$|(?:^|\/)types\//i;
|
|
45
|
+
|
|
46
|
+
// Strong implementation-seeking signals. A query that fires one of these is
|
|
47
|
+
// confidently asking for source code; anything else is treated as `'unknown'`.
|
|
48
|
+
// Curated to cover the validated guard-set queries plus common phrasings,
|
|
49
|
+
// without matching pure descriptive corpus prose like "Convert XML to URL List".
|
|
50
|
+
const IMPL_INTENT_RE = new RegExp(
|
|
51
|
+
'\\b(' + [
|
|
52
|
+
// English wh-questions about location/behaviour
|
|
53
|
+
'where', 'how does', 'how do',
|
|
54
|
+
// Definition / implementation phrasing
|
|
55
|
+
'implements?', 'implementation', 'defines?', 'definition', 'declared?',
|
|
56
|
+
// Code-structure nouns
|
|
57
|
+
'function', 'functions', 'method', 'methods', 'class', 'classes',
|
|
58
|
+
'constructor', 'module', 'library', 'crate', 'package',
|
|
59
|
+
// Verbs that strongly signal a code unit
|
|
60
|
+
'dispatch(?:es|er)?', 'handles?', 'handler', 'handlers',
|
|
61
|
+
'parses?', 'parser', 'parsers',
|
|
62
|
+
'router?', 'routes?', 'routing',
|
|
63
|
+
'register(?:s|ed|ing)?',
|
|
64
|
+
'builds?', 'builder', 'builders',
|
|
65
|
+
'generat(?:es?|or|ors|ed|ing)',
|
|
66
|
+
'creat(?:es?|or|ed|ion|ing)',
|
|
67
|
+
'loads?', 'loader',
|
|
68
|
+
'writes?', 'writer',
|
|
69
|
+
'reads?', 'reader',
|
|
70
|
+
'sends?', 'receives?',
|
|
71
|
+
'computes?', 'computed',
|
|
72
|
+
'encodes?', 'encoder', 'decodes?', 'decoder',
|
|
73
|
+
'transforms?', 'transformer',
|
|
74
|
+
'invokes?', 'calls?', 'returns?',
|
|
75
|
+
'valid(?:ate|ates|ator|ation)',
|
|
76
|
+
'serial(?:ize|izes|izer)', 'deserial(?:ize|izes|izer)',
|
|
77
|
+
'wrap(?:s|per|ped|ping)?',
|
|
78
|
+
'matchers?', 'matches?',
|
|
79
|
+
'printers?', 'prints?',
|
|
80
|
+
'searchers?', 'searches?',
|
|
81
|
+
// Specific terms common in real-repo guard queries
|
|
82
|
+
'callback', 'callbacks',
|
|
83
|
+
'factory', 'factories',
|
|
84
|
+
'controller', 'controllers',
|
|
85
|
+
'middleware',
|
|
86
|
+
'fallback', 'fallbacks',
|
|
87
|
+
'entrypoint', 'entry-point', 'main',
|
|
88
|
+
'init', 'initialise', 'initialize', 'initialiser', 'initializer',
|
|
89
|
+
'kernel', 'engine',
|
|
90
|
+
'wrapper', 'wrappers',
|
|
91
|
+
'singleton',
|
|
92
|
+
'factory',
|
|
93
|
+
'decorator', 'decorators',
|
|
94
|
+
'closure', 'closures',
|
|
95
|
+
].join('|') + ')\\b',
|
|
96
|
+
'i',
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
const DOCS_INTENT_RE = /\b(doc|docs|documentation|readme|guide|tutorial|reference|example)\b/i;
|
|
100
|
+
const TESTS_INTENT_RE = /\b(test|tests|spec|specs|fixture|fixtures|mock|mocks)\b/i;
|
|
101
|
+
const TYPES_INTENT_RE = /\b(type|types|interface|declaration|signature|typings|typedef)\b/i;
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Detect the file kind from a result path.
|
|
105
|
+
* @returns {'docs'|'tests'|'types'|'implementation'}
|
|
106
|
+
*/
|
|
107
|
+
export function detectFileKind(filePath) {
|
|
108
|
+
if (!filePath || typeof filePath !== 'string') return 'implementation';
|
|
109
|
+
if (DOCS_RE.test(filePath)) return 'docs';
|
|
110
|
+
if (TESTS_RE.test(filePath)) return 'tests';
|
|
111
|
+
if (TYPES_RE.test(filePath)) return 'types';
|
|
112
|
+
return 'implementation';
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Detect file-kind intent of a query along the docs/tests/types/implementation
|
|
117
|
+
* axis. Conservative: a query with no implementation-seeking signal returns
|
|
118
|
+
* `'unknown'`, and the helper treats `'unknown'` as a no-op (just like the
|
|
119
|
+
* docs/tests/types intents).
|
|
120
|
+
*
|
|
121
|
+
* @returns {'docs'|'tests'|'types'|'implementation'|'unknown'}
|
|
122
|
+
*/
|
|
123
|
+
export function classifyFileKindIntent(query) {
|
|
124
|
+
const q = (query || '').toLowerCase();
|
|
125
|
+
if (!q) return 'unknown';
|
|
126
|
+
// Type-seeking trumps test-seeking when both fire (existing convention).
|
|
127
|
+
if (TYPES_INTENT_RE.test(q)) return 'types';
|
|
128
|
+
if (DOCS_INTENT_RE.test(q)) return 'docs';
|
|
129
|
+
if (TESTS_INTENT_RE.test(q)) return 'tests';
|
|
130
|
+
if (IMPL_INTENT_RE.test(q)) return 'implementation';
|
|
131
|
+
return 'unknown';
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function resolveFilePath(r) {
|
|
135
|
+
return r?.file
|
|
136
|
+
|| r?.file_path
|
|
137
|
+
|| r?.path
|
|
138
|
+
|| r?.metadata?.file
|
|
139
|
+
|| r?.metadata?.file_path
|
|
140
|
+
|| r?.metadata?.path
|
|
141
|
+
|| '';
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function envOff() {
|
|
145
|
+
return process.env.SWEET_SEARCH_FILE_KIND_RANKING === '0'
|
|
146
|
+
|| process.env.SWEET_SEARCH_FILE_KIND_RANKING === 'false';
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function envFactor(name, fallback) {
|
|
150
|
+
const v = process.env[name];
|
|
151
|
+
if (!v) return fallback;
|
|
152
|
+
const n = Number.parseFloat(v);
|
|
153
|
+
return Number.isFinite(n) && n > 0 && n <= 1 ? n : fallback;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function envWindow(name, fallback) {
|
|
157
|
+
const v = process.env[name];
|
|
158
|
+
if (!v) return fallback;
|
|
159
|
+
const n = Number.parseInt(v, 10);
|
|
160
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const DEFAULT_FACTOR = 0.85;
|
|
164
|
+
const DEFAULT_WINDOW = 30;
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Apply intent-aware file-kind score multipliers, then re-sort the top-N
|
|
168
|
+
* window descending. The original array is not mutated.
|
|
169
|
+
*
|
|
170
|
+
* Demotion fires only when:
|
|
171
|
+
* - intent === 'implementation' (confident, NOT 'unknown'), AND
|
|
172
|
+
* - the top-N window contains at least one docs/tests/types candidate, AND
|
|
173
|
+
* - the top-N window contains at least one implementation candidate.
|
|
174
|
+
*
|
|
175
|
+
* In every other case the original `results` array is returned unchanged
|
|
176
|
+
* (same reference, no copy, no re-sort) — this is critical so the helper is
|
|
177
|
+
* a structural no-op on single-source corpora (GCSN) and on cascades whose
|
|
178
|
+
* top-N has no demotable competition.
|
|
179
|
+
*
|
|
180
|
+
* @param {Array} results - search results carrying .score and a file-path
|
|
181
|
+
* field (.file / .file_path / .path / .metadata.*).
|
|
182
|
+
* @param {Object} [opts]
|
|
183
|
+
* @param {string} [opts.query] - raw query (used to infer intent
|
|
184
|
+
* if opts.intent isn't supplied)
|
|
185
|
+
* @param {'docs'|'tests'|'types'|'implementation'|'unknown'} [opts.intent]
|
|
186
|
+
* - explicit intent override
|
|
187
|
+
* @param {number} [opts.docFactor] - default from env / 0.85
|
|
188
|
+
* @param {number} [opts.testFactor] - default from env / 0.85
|
|
189
|
+
* @param {number} [opts.typeFactor] - default from env / 0.85
|
|
190
|
+
* @param {number} [opts.window] - top-N window for analysis +
|
|
191
|
+
* bounded re-sort (default 30)
|
|
192
|
+
* @returns {Array} either the original `results` (no-op) or a new array
|
|
193
|
+
* whose head is sorted by adjusted score and whose tail is
|
|
194
|
+
* the unchanged input tail. Stable on ties.
|
|
195
|
+
*/
|
|
196
|
+
export function applyFileKindRanking(results, opts = {}) {
|
|
197
|
+
if (envOff()) return results;
|
|
198
|
+
if (!Array.isArray(results) || results.length === 0) return results;
|
|
199
|
+
|
|
200
|
+
const intent = opts.intent != null
|
|
201
|
+
? opts.intent
|
|
202
|
+
: classifyFileKindIntent(opts.query || '');
|
|
203
|
+
|
|
204
|
+
// Conservative gate: only confident 'implementation' intent fires.
|
|
205
|
+
if (intent !== 'implementation') return results;
|
|
206
|
+
|
|
207
|
+
const window = opts.window != null
|
|
208
|
+
? opts.window
|
|
209
|
+
: envWindow('SWEET_SEARCH_FILE_KIND_WINDOW', DEFAULT_WINDOW);
|
|
210
|
+
const windowSize = Math.min(window, results.length);
|
|
211
|
+
|
|
212
|
+
// Walk the window once: classify kinds and check for competition.
|
|
213
|
+
const kinds = new Array(windowSize);
|
|
214
|
+
let demotableCount = 0;
|
|
215
|
+
let implCount = 0;
|
|
216
|
+
for (let i = 0; i < windowSize; i++) {
|
|
217
|
+
const k = detectFileKind(resolveFilePath(results[i]));
|
|
218
|
+
kinds[i] = k;
|
|
219
|
+
if (k === 'docs' || k === 'tests' || k === 'types') demotableCount++;
|
|
220
|
+
else if (k === 'implementation') implCount++;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Structural skip: nothing to demote, or nothing to promote.
|
|
224
|
+
if (demotableCount === 0 || implCount === 0) return results;
|
|
225
|
+
|
|
226
|
+
const factor = envFactor('SWEET_SEARCH_FILE_KIND_FACTOR', DEFAULT_FACTOR);
|
|
227
|
+
const docFactor = opts.docFactor != null ? opts.docFactor : factor;
|
|
228
|
+
const testFactor = opts.testFactor != null ? opts.testFactor : factor;
|
|
229
|
+
const typeFactor = opts.typeFactor != null ? opts.typeFactor : factor;
|
|
230
|
+
|
|
231
|
+
const reranked = new Array(windowSize);
|
|
232
|
+
for (let i = 0; i < windowSize; i++) {
|
|
233
|
+
const r = results[i];
|
|
234
|
+
const kind = kinds[i];
|
|
235
|
+
let mult = 1;
|
|
236
|
+
if (kind === 'docs') mult = docFactor;
|
|
237
|
+
else if (kind === 'tests') mult = testFactor;
|
|
238
|
+
else if (kind === 'types') mult = typeFactor;
|
|
239
|
+
const baseScore = (typeof r.score === 'number') ? r.score : 0;
|
|
240
|
+
reranked[i] = {
|
|
241
|
+
...r,
|
|
242
|
+
_fileKindOrigScore: baseScore,
|
|
243
|
+
_fileKindMult: mult,
|
|
244
|
+
_fileKindKind: kind,
|
|
245
|
+
_fileKindOrigIndex: i,
|
|
246
|
+
score: baseScore * mult,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Stable sort: descending score, tie-break on original index.
|
|
251
|
+
reranked.sort((a, b) => {
|
|
252
|
+
const d = (b.score || 0) - (a.score || 0);
|
|
253
|
+
return d !== 0 ? d : a._fileKindOrigIndex - b._fileKindOrigIndex;
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
for (const r of reranked) delete r._fileKindOrigIndex;
|
|
257
|
+
|
|
258
|
+
// Concatenate unchanged tail. The cascade's CE/MaxSim score-scale
|
|
259
|
+
// boundary typically lives near rank `ceTopK`, so leaving rank
|
|
260
|
+
// `windowSize`+ untouched contains the damage from any cross-scale
|
|
261
|
+
// re-sort that might happen inside the window.
|
|
262
|
+
if (windowSize === results.length) return reranked;
|
|
263
|
+
return reranked.concat(results.slice(windowSize));
|
|
264
|
+
}
|
|
@@ -1414,10 +1414,16 @@ export class LateInteractionIndex {
|
|
|
1414
1414
|
// don't support importance weighting, so we must use the JS-tier weighted path.
|
|
1415
1415
|
const nativeScored = new Set();
|
|
1416
1416
|
|
|
1417
|
+
// Resolve a doc-lookup ID for each candidate. Graph-expanded candidates
|
|
1418
|
+
// carry `_liChunkId` (a chunk id pointing into the LI index) while their
|
|
1419
|
+
// public `id` is the entity id from the code graph. Honouring _liChunkId
|
|
1420
|
+
// lets expanded candidates participate in MaxSim rerank.
|
|
1421
|
+
const docIdOf = (c) => c._liChunkId || c.id;
|
|
1422
|
+
|
|
1417
1423
|
if (useFlatPath && !this.useTokenWeights) {
|
|
1418
1424
|
const groups = { bit4: [], perToken: [], perDoc: [] };
|
|
1419
1425
|
for (const candidate of toScore) {
|
|
1420
|
-
const doc = this.documents.get(candidate
|
|
1426
|
+
const doc = this.documents.get(docIdOf(candidate));
|
|
1421
1427
|
if (!doc) continue;
|
|
1422
1428
|
if (doc.quantBits === 4 && doc.minArray && doc.tokenNorms) {
|
|
1423
1429
|
groups.bit4.push({ candidate, doc });
|
|
@@ -1453,7 +1459,7 @@ export class LateInteractionIndex {
|
|
|
1453
1459
|
// Try WASM fused kernels first (avoids JS-side dequant), fall back to JS dequant + wasmMaxSimF32.
|
|
1454
1460
|
for (const candidate of toScore) {
|
|
1455
1461
|
if (nativeScored.has(candidate.id)) continue;
|
|
1456
|
-
const doc = this.documents.get(candidate
|
|
1462
|
+
const doc = this.documents.get(docIdOf(candidate));
|
|
1457
1463
|
if (!doc) { pushFallback(candidate); continue; }
|
|
1458
1464
|
|
|
1459
1465
|
if (useFlatPath) {
|
|
@@ -1488,7 +1494,7 @@ export class LateInteractionIndex {
|
|
|
1488
1494
|
}
|
|
1489
1495
|
|
|
1490
1496
|
// JS dequant → WASM f32 or JS fallback
|
|
1491
|
-
const flatData = this.getTokensFlat(candidate
|
|
1497
|
+
const flatData = this.getTokensFlat(docIdOf(candidate));
|
|
1492
1498
|
if (flatData) {
|
|
1493
1499
|
pushScored(candidate, this.maxSimScoreFlat(
|
|
1494
1500
|
effectiveQueryTokens, flatData.flat, flatData.numTokens, flatData.dim,
|
|
@@ -1498,7 +1504,7 @@ export class LateInteractionIndex {
|
|
|
1498
1504
|
pushFallback(candidate);
|
|
1499
1505
|
}
|
|
1500
1506
|
} else {
|
|
1501
|
-
const docTokens = this.getTokens(candidate
|
|
1507
|
+
const docTokens = this.getTokens(docIdOf(candidate));
|
|
1502
1508
|
if (docTokens) {
|
|
1503
1509
|
pushScored(candidate, this.maxSimScore(effectiveQueryTokens, docTokens, pruneOpts));
|
|
1504
1510
|
} else {
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Late-interaction search-rerank policy resolver.
|
|
3
|
+
*
|
|
4
|
+
* Pure function — no I/O. The two product concepts (LI indexing model vs
|
|
5
|
+
* search-side LI rerank policy) are separate. This resolver computes the
|
|
6
|
+
* effective search-side state from explicit user choices, env vars,
|
|
7
|
+
* persisted init config, and the on-disk LI index manifest.
|
|
8
|
+
*
|
|
9
|
+
* The on-disk manifest is the source of truth: if the user (or auto)
|
|
10
|
+
* asks for rerank ON but the loaded index is missing, mismatched, or
|
|
11
|
+
* built with an edge model that's known to underperform as a reranker
|
|
12
|
+
* on benchmarked corpora, the resolver downgrades or warns accordingly.
|
|
13
|
+
*
|
|
14
|
+
* Bench evidence backing the auto rules (gencodesearchnet, 2026-05-03):
|
|
15
|
+
* standard `lateon-code` + LI on : 85.57 % MRR ← auto resolves ON
|
|
16
|
+
* edge `lateon-code-edge` + LI on : 80.65 % MRR ← auto resolves OFF
|
|
17
|
+
* edge + LI off : 82.91 % MRR (best edge config)
|
|
18
|
+
* standard + LI off : 82.91 % MRR (index-independent floor)
|
|
19
|
+
* See docs/BENCH_TODO.md "Phase 3 — Honest sweep before v2.5.0 (post-fix re-run)".
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
// Public model identifiers — kept in lockstep with
|
|
24
|
+
// `core/infrastructure/config/ranking.js::LATE_INTERACTION_CONFIG.models`.
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
export const LI_MODEL_STANDARD = 'lateon-code';
|
|
28
|
+
export const LI_MODEL_EDGE = 'lateon-code-edge';
|
|
29
|
+
export const LI_MODEL_NONE = 'none';
|
|
30
|
+
|
|
31
|
+
export const VALID_LI_MODELS = Object.freeze([
|
|
32
|
+
LI_MODEL_STANDARD,
|
|
33
|
+
LI_MODEL_EDGE,
|
|
34
|
+
LI_MODEL_NONE,
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
export const VALID_RERANK_POLICIES = Object.freeze(['auto', 'on', 'off']);
|
|
38
|
+
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// Env-var coercion (same shape as other env opt-outs in the repo —
|
|
41
|
+
// SWEET_SEARCH_COREML_CASCADE, SWEET_SEARCH_NATIVE_INFERENCE, etc.)
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
function isEnvOff(value) {
|
|
45
|
+
if (value == null) return false;
|
|
46
|
+
const v = String(value).trim().toLowerCase();
|
|
47
|
+
return v === '0' || v === 'false' || v === 'off' || v === 'no';
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function isEnvOn(value) {
|
|
51
|
+
if (value == null) return false;
|
|
52
|
+
const v = String(value).trim().toLowerCase();
|
|
53
|
+
return v === '1' || v === 'true' || v === 'on' || v === 'yes';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
// Resolver
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Resolve effective search-side LI rerank state.
|
|
62
|
+
*
|
|
63
|
+
* @param {object} input
|
|
64
|
+
* @param {object} [input.persisted] - parsed `.sweet-search/config.json`
|
|
65
|
+
* @param {string} [input.persisted.liModel] - 'lateon-code' | 'lateon-code-edge' | 'none'
|
|
66
|
+
* @param {string} [input.persisted.searchReranking] - 'auto' | 'on' | 'off'
|
|
67
|
+
* @param {object} [input.indexManifest] - loaded LI index header
|
|
68
|
+
* @param {string} [input.indexManifest.modelId] - id baked into the SSLX header
|
|
69
|
+
* @param {number} [input.indexManifest.tokenDim] - per-token dimension
|
|
70
|
+
* @param {boolean} [input.indexManifest.modelMismatch] - true when loaded
|
|
71
|
+
* modelId disagrees with the active config model
|
|
72
|
+
* @param {boolean} [input.indexManifest.exists] - false when no index file on disk
|
|
73
|
+
* @param {object} [input.env] - env-var snapshot (defaults to process.env)
|
|
74
|
+
* @param {boolean} [input.optionOverride] - explicit per-call override from caller
|
|
75
|
+
* (search-time options.useLateInteraction). If a boolean, it short-circuits the
|
|
76
|
+
* resolver and wins over everything else.
|
|
77
|
+
* @param {string} [input.activeConfigModel] - LATE_INTERACTION_CONFIG.model fallback
|
|
78
|
+
* (used when the index hasn't been loaded yet so we can still emit a sensible
|
|
79
|
+
* default — preserves back-compat with the pre-Phase-4 behaviour).
|
|
80
|
+
*
|
|
81
|
+
* @returns {{
|
|
82
|
+
* effective: boolean,
|
|
83
|
+
* policy: 'auto'|'on'|'off',
|
|
84
|
+
* reason: string,
|
|
85
|
+
* warning?: string
|
|
86
|
+
* }}
|
|
87
|
+
*/
|
|
88
|
+
export function resolveSearchRerankPolicy(input = {}) {
|
|
89
|
+
const env = input.env ?? process.env;
|
|
90
|
+
const persisted = input.persisted ?? {};
|
|
91
|
+
const manifest = input.indexManifest ?? null;
|
|
92
|
+
const declaredPolicy = normalizePolicy(persisted.searchReranking);
|
|
93
|
+
|
|
94
|
+
// 1. Per-call explicit override (existing API, highest precedence).
|
|
95
|
+
if (typeof input.optionOverride === 'boolean') {
|
|
96
|
+
return {
|
|
97
|
+
effective: input.optionOverride,
|
|
98
|
+
policy: declaredPolicy,
|
|
99
|
+
reason: `per-call override (${input.optionOverride ? 'on' : 'off'})`,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// 2. Env-var hard kill switch — opt-out for benchmarks / scripts that
|
|
104
|
+
// must defeat any persisted policy. Mirrors SWEET_SEARCH_COREML_CASCADE=0.
|
|
105
|
+
if (isEnvOff(env.SWEET_SEARCH_LI_RERANK)) {
|
|
106
|
+
return {
|
|
107
|
+
effective: false,
|
|
108
|
+
policy: declaredPolicy,
|
|
109
|
+
reason: 'SWEET_SEARCH_LI_RERANK=0 (env opt-out)',
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
if (isEnvOn(env.SWEET_SEARCH_LI_RERANK)) {
|
|
113
|
+
// Env-on still goes through the safety check below — we never silently
|
|
114
|
+
// rerank with a missing or mismatched index.
|
|
115
|
+
if (!manifestUsable(manifest)) {
|
|
116
|
+
return {
|
|
117
|
+
effective: false,
|
|
118
|
+
policy: declaredPolicy,
|
|
119
|
+
reason: 'SWEET_SEARCH_LI_RERANK=1 but no usable LI index',
|
|
120
|
+
warning: manifestUnusableReason(manifest),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
effective: true,
|
|
125
|
+
policy: declaredPolicy,
|
|
126
|
+
reason: 'SWEET_SEARCH_LI_RERANK=1 (env opt-in)',
|
|
127
|
+
...(isEdgeManifest(manifest) ? { warning: edgeOnWarning() } : {}),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// 3. Persisted explicit ON / OFF (init wizard / --search-reranking).
|
|
132
|
+
if (declaredPolicy === 'off') {
|
|
133
|
+
return {
|
|
134
|
+
effective: false,
|
|
135
|
+
policy: 'off',
|
|
136
|
+
reason: 'persisted searchReranking=off',
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
if (declaredPolicy === 'on') {
|
|
140
|
+
if (!manifestUsable(manifest)) {
|
|
141
|
+
return {
|
|
142
|
+
effective: false,
|
|
143
|
+
policy: 'on',
|
|
144
|
+
reason: 'persisted searchReranking=on but no usable LI index',
|
|
145
|
+
warning: manifestUnusableReason(manifest),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
effective: true,
|
|
150
|
+
policy: 'on',
|
|
151
|
+
reason: 'persisted searchReranking=on',
|
|
152
|
+
...(isEdgeManifest(manifest) ? { warning: edgeOnWarning() } : {}),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// 4. Auto (the default): consult the index manifest.
|
|
157
|
+
// - missing/mismatched → off (with diagnostic)
|
|
158
|
+
// - edge modelId → off (Phase 3 shows edge LI rerank is net-negative)
|
|
159
|
+
// - any standard model → on
|
|
160
|
+
if (manifest != null && (manifest.exists === false)) {
|
|
161
|
+
return {
|
|
162
|
+
effective: false,
|
|
163
|
+
policy: 'auto',
|
|
164
|
+
reason: 'auto: no LI index on disk',
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
if (manifest != null && manifest.modelMismatch === true) {
|
|
168
|
+
return {
|
|
169
|
+
effective: false,
|
|
170
|
+
policy: 'auto',
|
|
171
|
+
reason: `auto: LI index model mismatch (header=${manifest.modelId ?? '?'})`,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
if (manifest != null && isEdgeManifest(manifest)) {
|
|
175
|
+
return {
|
|
176
|
+
effective: false,
|
|
177
|
+
policy: 'auto',
|
|
178
|
+
reason: `auto: edge LI index (${manifest.modelId}) — search rerank disabled by default (see Phase 3 bench)`,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
if (manifest != null && manifest.modelId) {
|
|
182
|
+
return {
|
|
183
|
+
effective: true,
|
|
184
|
+
policy: 'auto',
|
|
185
|
+
reason: `auto: standard LI index (${manifest.modelId})`,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// 5. Fallback — manifest not yet loaded (early call path) OR no manifest
|
|
190
|
+
// info at all. Defer to the active config model so existing flows that
|
|
191
|
+
// constructed SweetSearch before init() still get the historical
|
|
192
|
+
// behaviour. Edge model active → off; anything else → on.
|
|
193
|
+
if (input.activeConfigModel === LI_MODEL_EDGE) {
|
|
194
|
+
return {
|
|
195
|
+
effective: false,
|
|
196
|
+
policy: 'auto',
|
|
197
|
+
reason: 'auto: edge LI active in config (manifest not yet loaded)',
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
effective: true,
|
|
202
|
+
policy: 'auto',
|
|
203
|
+
reason: 'auto: standard LI active in config (manifest not yet loaded)',
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ---------------------------------------------------------------------------
|
|
208
|
+
// Helpers (also exported for test reuse)
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
|
|
211
|
+
export function normalizePolicy(value) {
|
|
212
|
+
if (typeof value !== 'string') return 'auto';
|
|
213
|
+
const v = value.trim().toLowerCase();
|
|
214
|
+
return VALID_RERANK_POLICIES.includes(v) ? v : 'auto';
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export function normalizeLiModel(value) {
|
|
218
|
+
if (typeof value !== 'string') return null;
|
|
219
|
+
const v = value.trim();
|
|
220
|
+
return VALID_LI_MODELS.includes(v) ? v : null;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function isEdgeManifest(manifest) {
|
|
224
|
+
return !!manifest && manifest.modelId === LI_MODEL_EDGE;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function manifestUsable(manifest) {
|
|
228
|
+
if (manifest == null) return false;
|
|
229
|
+
if (manifest.exists === false) return false;
|
|
230
|
+
if (manifest.modelMismatch === true) return false;
|
|
231
|
+
if (!manifest.modelId) return false;
|
|
232
|
+
return true;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function manifestUnusableReason(manifest) {
|
|
236
|
+
if (manifest == null) return 'no LI index manifest';
|
|
237
|
+
if (manifest.exists === false) return 'no LI index file on disk';
|
|
238
|
+
if (manifest.modelMismatch === true) {
|
|
239
|
+
return `LI index built with ${manifest.modelId ?? '?'} but config says otherwise — re-index to fix`;
|
|
240
|
+
}
|
|
241
|
+
if (!manifest.modelId) return 'LI index manifest missing modelId';
|
|
242
|
+
return 'unusable LI index manifest';
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function edgeOnWarning() {
|
|
246
|
+
return (
|
|
247
|
+
'Edge LI search reranking benchmarked below no-rerank search on '
|
|
248
|
+
+ 'gencodesearchnet (80.65% vs 82.91% MRR). The recommended edge '
|
|
249
|
+
+ 'setup is search reranking off — edge LI tokens still power '
|
|
250
|
+
+ 'read-semantic and ColGrep without participating in search rerank.'
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
// Init-side hardware-aware default
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Recommend an LI model + rerank policy from a hardware capability snapshot.
|
|
260
|
+
* Pure function — caller passes in detectHardwareCapability() output.
|
|
261
|
+
*
|
|
262
|
+
* Conservative: accuracy-first by default. Only recommends edge when the
|
|
263
|
+
* machine is clearly RAM- or disk-constrained.
|
|
264
|
+
*
|
|
265
|
+
* @param {object} hw - shape returned by detectHardwareCapability()
|
|
266
|
+
* @returns {{ liModel: string, searchReranking: 'auto', reason: string }}
|
|
267
|
+
*/
|
|
268
|
+
export function recommendInitDefaults(hw = {}) {
|
|
269
|
+
const ramGB = Number(hw.totalMemGB ?? 0);
|
|
270
|
+
// Constrained heuristic (intentionally narrow — "accuracy-first unless
|
|
271
|
+
// hardware/disk clearly indicates constrained mode" per Phase 4 brief):
|
|
272
|
+
// - RAM ≤ 8 GB → edge
|
|
273
|
+
// - Apple Silicon M1/M2 (older ANE) → edge candidate
|
|
274
|
+
// - everything else → standard
|
|
275
|
+
// The bench shows standard LI is the accuracy default; only flip when
|
|
276
|
+
// the constrained signal is unambiguous.
|
|
277
|
+
const isLowRam = ramGB > 0 && ramGB <= 8;
|
|
278
|
+
const isOlderApple =
|
|
279
|
+
hw.appleSilicon && typeof hw.appleSilicon === 'object'
|
|
280
|
+
? Number(hw.appleSilicon.generation ?? hw.appleSilicon.gen ?? 99) <= 2
|
|
281
|
+
: false;
|
|
282
|
+
|
|
283
|
+
if (isLowRam) {
|
|
284
|
+
return {
|
|
285
|
+
liModel: LI_MODEL_EDGE,
|
|
286
|
+
searchReranking: 'auto',
|
|
287
|
+
reason: `constrained: RAM=${ramGB} GB (≤8) — edge LI for indexing, search rerank auto-disables on edge per Phase 3 bench`,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
if (isOlderApple && ramGB > 0 && ramGB <= 16) {
|
|
291
|
+
return {
|
|
292
|
+
liModel: LI_MODEL_EDGE,
|
|
293
|
+
searchReranking: 'auto',
|
|
294
|
+
reason: `constrained: M1/M2 + RAM=${ramGB} GB — edge LI recommended for indexing speed + disk savings`,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
return {
|
|
298
|
+
liModel: LI_MODEL_STANDARD,
|
|
299
|
+
searchReranking: 'auto',
|
|
300
|
+
reason: ramGB > 0
|
|
301
|
+
? `capable: RAM=${ramGB} GB — standard LI (accuracy default at 85.57% MRR on gencodesearchnet)`
|
|
302
|
+
: 'capable (default): standard LI (accuracy default)',
|
|
303
|
+
};
|
|
304
|
+
}
|