sweet-search 2.4.2 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/core/cli.js +19 -5
  2. package/core/embedding/embedding-cache.js +177 -15
  3. package/core/embedding/embedding-service.js +18 -4
  4. package/core/graph/graph-expansion.js +52 -12
  5. package/core/graph/graph-extractor.js +30 -1
  6. package/core/indexing/ast-chunker.js +331 -16
  7. package/core/indexing/chunking/chunk-builder.js +34 -1
  8. package/core/indexing/index.js +6 -3
  9. package/core/indexing/indexer-ann.js +45 -6
  10. package/core/indexing/indexer-build.js +9 -1
  11. package/core/indexing/indexer-phases.js +6 -4
  12. package/core/indexing/indexing-file-policy.js +140 -0
  13. package/core/indexing/li-skip-policy.js +11 -220
  14. package/core/infrastructure/codebase-repository.js +21 -0
  15. package/core/infrastructure/config/embedding.js +20 -1
  16. package/core/infrastructure/config/graph.js +2 -2
  17. package/core/infrastructure/config/ranking.js +10 -0
  18. package/core/infrastructure/config/vector-store.js +1 -1
  19. package/core/infrastructure/coreml-cascade.js +236 -30
  20. package/core/infrastructure/coreml-cascade.json +25 -0
  21. package/core/infrastructure/index.js +15 -0
  22. package/core/infrastructure/init-config.js +78 -0
  23. package/core/infrastructure/language-patterns/registry-core.js +18 -0
  24. package/core/infrastructure/model-registry.js +12 -0
  25. package/core/infrastructure/native-inference.js +143 -51
  26. package/core/infrastructure/tree-sitter-provider.js +92 -2
  27. package/core/ranking/cascaded-scorer.js +6 -2
  28. package/core/ranking/file-kind-ranking.js +264 -0
  29. package/core/ranking/late-interaction-index.js +10 -4
  30. package/core/ranking/late-interaction-policy.js +304 -0
  31. package/core/search/context-expander.js +267 -28
  32. package/core/search/index.js +4 -0
  33. package/core/search/search-cli.js +3 -1
  34. package/core/search/search-pattern.js +4 -3
  35. package/core/search/search-postprocess.js +189 -8
  36. package/core/search/search-read-semantic.js +717 -0
  37. package/core/search/search-read.js +481 -0
  38. package/core/search/search-server.js +6 -4
  39. package/core/search/sweet-search.js +119 -15
  40. package/mcp/server.js +41 -0
  41. package/mcp/tool-handlers.js +117 -6
  42. package/package.json +9 -7
  43. package/scripts/init.js +386 -5
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Intent-aware file-kind ranking (conservative variant).
3
+ *
4
+ * Background: real-codebase miss analysis found that documentation, test, and
5
+ * TypeScript-declaration files often outrank the implementation file users
6
+ * were actually looking for on multi-file codebases. The first version of
7
+ * this rule (commit f6fcfd1) lifted graph-2hop R@1 from 47.46 % → 64.41 %
8
+ * but catastrophically regressed GenCodeSearchNet under the dense profile
9
+ * (full-6 000 dense run: MRR@10 84.4 % → 47.4 %, Recall@5 92.0 % → 48.4 %).
10
+ * Root cause: the legacy LI-rerank pipeline assembles
11
+ * `results = [...liScored, ...tail]`, where `liScored` carries MaxSim
12
+ * scores that are sometimes *lower* (in absolute value) than the int8
13
+ * cosine scores already on the un-reranked tail. The concatenated list is
14
+ * therefore not globally score-monotonic. The old helper unconditionally
15
+ * spread and re-sorted *all* results by `score`, which floated the
16
+ * int8-only tail above the LI-reranked head and undid the rerank — even
17
+ * when every multiplier was 1 (GenCodeSearchNet is a single-source
18
+ * corpus, so no docs/tests/types kind ever matches there).
19
+ *
20
+ * Conservative variant fixes both regressions with three guards:
21
+ *
22
+ * 1. Confident-intent gating. `classifyFileKindIntent` now returns
23
+ * `'unknown'` for queries with no implementation-seeking signal. Only
24
+ * explicit `'implementation'` intent triggers demotion. `'unknown'`,
25
+ * `'docs'`, `'tests'`, `'types'` are no-ops.
26
+ *
27
+ * 2. Structural skip. The rule looks at the top-N candidates (default 30).
28
+ * If the window has zero docs/tests/types files (single-source corpus
29
+ * like GCSN) or zero implementation files (nothing to promote), the
30
+ * input is returned untouched. No re-sort, no new objects.
31
+ *
32
+ * 3. Window-bounded re-sort. When the rule does fire, only the top-N
33
+ * window is re-ranked. The tail — where the rerank/non-rerank score-
34
+ * scale boundary usually lives — is concatenated unchanged. This
35
+ * keeps mixed-scale damage contained.
36
+ *
37
+ * Disable at runtime with `SWEET_SEARCH_FILE_KIND_RANKING=0`. Tune the soft
38
+ * factor with `SWEET_SEARCH_FILE_KIND_FACTOR` (default 0.85; range (0, 1]).
39
+ * Tune the window with `SWEET_SEARCH_FILE_KIND_WINDOW` (default 30).
40
+ */
41
+
42
+ const DOCS_RE = /\.md$|\.mdx$|\.rst$|(?:^|\/)docs?\//i;
43
+ const TESTS_RE = /(?:^|\/)tests?\/|(?:^|\/)spec\/|\.test\.[a-z0-9]+$|_test\.[a-z0-9]+$|\.spec\.[a-z0-9]+$|_spec\.[a-z0-9]+$/i;
44
+ const TYPES_RE = /\.d\.ts$|(?:^|\/)types\//i;
45
+
46
+ // Strong implementation-seeking signals. A query that fires one of these is
47
+ // confidently asking for source code; anything else is treated as `'unknown'`.
48
+ // Curated to cover the validated guard-set queries plus common phrasings,
49
+ // without matching pure descriptive corpus prose like "Convert XML to URL List".
50
+ const IMPL_INTENT_RE = new RegExp(
51
+ '\\b(' + [
52
+ // English wh-questions about location/behaviour
53
+ 'where', 'how does', 'how do',
54
+ // Definition / implementation phrasing
55
+ 'implements?', 'implementation', 'defines?', 'definition', 'declared?',
56
+ // Code-structure nouns
57
+ 'function', 'functions', 'method', 'methods', 'class', 'classes',
58
+ 'constructor', 'module', 'library', 'crate', 'package',
59
+ // Verbs that strongly signal a code unit
60
+ 'dispatch(?:es|er)?', 'handles?', 'handler', 'handlers',
61
+ 'parses?', 'parser', 'parsers',
62
+ 'router?', 'routes?', 'routing',
63
+ 'register(?:s|ed|ing)?',
64
+ 'builds?', 'builder', 'builders',
65
+ 'generat(?:es?|or|ors|ed|ing)',
66
+ 'creat(?:es?|or|ed|ion|ing)',
67
+ 'loads?', 'loader',
68
+ 'writes?', 'writer',
69
+ 'reads?', 'reader',
70
+ 'sends?', 'receives?',
71
+ 'computes?', 'computed',
72
+ 'encodes?', 'encoder', 'decodes?', 'decoder',
73
+ 'transforms?', 'transformer',
74
+ 'invokes?', 'calls?', 'returns?',
75
+ 'valid(?:ate|ates|ator|ation)',
76
+ 'serial(?:ize|izes|izer)', 'deserial(?:ize|izes|izer)',
77
+ 'wrap(?:s|per|ped|ping)?',
78
+ 'matchers?', 'matches?',
79
+ 'printers?', 'prints?',
80
+ 'searchers?', 'searches?',
81
+ // Specific terms common in real-repo guard queries
82
+ 'callback', 'callbacks',
83
+ 'factory', 'factories',
84
+ 'controller', 'controllers',
85
+ 'middleware',
86
+ 'fallback', 'fallbacks',
87
+ 'entrypoint', 'entry-point', 'main',
88
+ 'init', 'initialise', 'initialize', 'initialiser', 'initializer',
89
+ 'kernel', 'engine',
90
+ 'wrapper', 'wrappers',
91
+ 'singleton',
92
+ 'factory',
93
+ 'decorator', 'decorators',
94
+ 'closure', 'closures',
95
+ ].join('|') + ')\\b',
96
+ 'i',
97
+ );
98
+
99
+ const DOCS_INTENT_RE = /\b(doc|docs|documentation|readme|guide|tutorial|reference|example)\b/i;
100
+ const TESTS_INTENT_RE = /\b(test|tests|spec|specs|fixture|fixtures|mock|mocks)\b/i;
101
+ const TYPES_INTENT_RE = /\b(type|types|interface|declaration|signature|typings|typedef)\b/i;
102
+
103
+ /**
104
+ * Detect the file kind from a result path.
105
+ * @returns {'docs'|'tests'|'types'|'implementation'}
106
+ */
107
+ export function detectFileKind(filePath) {
108
+ if (!filePath || typeof filePath !== 'string') return 'implementation';
109
+ if (DOCS_RE.test(filePath)) return 'docs';
110
+ if (TESTS_RE.test(filePath)) return 'tests';
111
+ if (TYPES_RE.test(filePath)) return 'types';
112
+ return 'implementation';
113
+ }
114
+
115
+ /**
116
+ * Detect file-kind intent of a query along the docs/tests/types/implementation
117
+ * axis. Conservative: a query with no implementation-seeking signal returns
118
+ * `'unknown'`, and the helper treats `'unknown'` as a no-op (just like the
119
+ * docs/tests/types intents).
120
+ *
121
+ * @returns {'docs'|'tests'|'types'|'implementation'|'unknown'}
122
+ */
123
+ export function classifyFileKindIntent(query) {
124
+ const q = (query || '').toLowerCase();
125
+ if (!q) return 'unknown';
126
+ // Type-seeking trumps test-seeking when both fire (existing convention).
127
+ if (TYPES_INTENT_RE.test(q)) return 'types';
128
+ if (DOCS_INTENT_RE.test(q)) return 'docs';
129
+ if (TESTS_INTENT_RE.test(q)) return 'tests';
130
+ if (IMPL_INTENT_RE.test(q)) return 'implementation';
131
+ return 'unknown';
132
+ }
133
+
134
+ function resolveFilePath(r) {
135
+ return r?.file
136
+ || r?.file_path
137
+ || r?.path
138
+ || r?.metadata?.file
139
+ || r?.metadata?.file_path
140
+ || r?.metadata?.path
141
+ || '';
142
+ }
143
+
144
+ function envOff() {
145
+ return process.env.SWEET_SEARCH_FILE_KIND_RANKING === '0'
146
+ || process.env.SWEET_SEARCH_FILE_KIND_RANKING === 'false';
147
+ }
148
+
149
+ function envFactor(name, fallback) {
150
+ const v = process.env[name];
151
+ if (!v) return fallback;
152
+ const n = Number.parseFloat(v);
153
+ return Number.isFinite(n) && n > 0 && n <= 1 ? n : fallback;
154
+ }
155
+
156
+ function envWindow(name, fallback) {
157
+ const v = process.env[name];
158
+ if (!v) return fallback;
159
+ const n = Number.parseInt(v, 10);
160
+ return Number.isFinite(n) && n > 0 ? n : fallback;
161
+ }
162
+
163
+ const DEFAULT_FACTOR = 0.85;
164
+ const DEFAULT_WINDOW = 30;
165
+
166
+ /**
167
+ * Apply intent-aware file-kind score multipliers, then re-sort the top-N
168
+ * window descending. The original array is not mutated.
169
+ *
170
+ * Demotion fires only when:
171
+ * - intent === 'implementation' (confident, NOT 'unknown'), AND
172
+ * - the top-N window contains at least one docs/tests/types candidate, AND
173
+ * - the top-N window contains at least one implementation candidate.
174
+ *
175
+ * In every other case the original `results` array is returned unchanged
176
+ * (same reference, no copy, no re-sort) — this is critical so the helper is
177
+ * a structural no-op on single-source corpora (GCSN) and on cascades whose
178
+ * top-N has no demotable competition.
179
+ *
180
+ * @param {Array} results - search results carrying .score and a file-path
181
+ * field (.file / .file_path / .path / .metadata.*).
182
+ * @param {Object} [opts]
183
+ * @param {string} [opts.query] - raw query (used to infer intent
184
+ * if opts.intent isn't supplied)
185
+ * @param {'docs'|'tests'|'types'|'implementation'|'unknown'} [opts.intent]
186
+ * - explicit intent override
187
+ * @param {number} [opts.docFactor] - default from env / 0.85
188
+ * @param {number} [opts.testFactor] - default from env / 0.85
189
+ * @param {number} [opts.typeFactor] - default from env / 0.85
190
+ * @param {number} [opts.window] - top-N window for analysis +
191
+ * bounded re-sort (default 30)
192
+ * @returns {Array} either the original `results` (no-op) or a new array
193
+ * whose head is sorted by adjusted score and whose tail is
194
+ * the unchanged input tail. Stable on ties.
195
+ */
196
+ export function applyFileKindRanking(results, opts = {}) {
197
+ if (envOff()) return results;
198
+ if (!Array.isArray(results) || results.length === 0) return results;
199
+
200
+ const intent = opts.intent != null
201
+ ? opts.intent
202
+ : classifyFileKindIntent(opts.query || '');
203
+
204
+ // Conservative gate: only confident 'implementation' intent fires.
205
+ if (intent !== 'implementation') return results;
206
+
207
+ const window = opts.window != null
208
+ ? opts.window
209
+ : envWindow('SWEET_SEARCH_FILE_KIND_WINDOW', DEFAULT_WINDOW);
210
+ const windowSize = Math.min(window, results.length);
211
+
212
+ // Walk the window once: classify kinds and check for competition.
213
+ const kinds = new Array(windowSize);
214
+ let demotableCount = 0;
215
+ let implCount = 0;
216
+ for (let i = 0; i < windowSize; i++) {
217
+ const k = detectFileKind(resolveFilePath(results[i]));
218
+ kinds[i] = k;
219
+ if (k === 'docs' || k === 'tests' || k === 'types') demotableCount++;
220
+ else if (k === 'implementation') implCount++;
221
+ }
222
+
223
+ // Structural skip: nothing to demote, or nothing to promote.
224
+ if (demotableCount === 0 || implCount === 0) return results;
225
+
226
+ const factor = envFactor('SWEET_SEARCH_FILE_KIND_FACTOR', DEFAULT_FACTOR);
227
+ const docFactor = opts.docFactor != null ? opts.docFactor : factor;
228
+ const testFactor = opts.testFactor != null ? opts.testFactor : factor;
229
+ const typeFactor = opts.typeFactor != null ? opts.typeFactor : factor;
230
+
231
+ const reranked = new Array(windowSize);
232
+ for (let i = 0; i < windowSize; i++) {
233
+ const r = results[i];
234
+ const kind = kinds[i];
235
+ let mult = 1;
236
+ if (kind === 'docs') mult = docFactor;
237
+ else if (kind === 'tests') mult = testFactor;
238
+ else if (kind === 'types') mult = typeFactor;
239
+ const baseScore = (typeof r.score === 'number') ? r.score : 0;
240
+ reranked[i] = {
241
+ ...r,
242
+ _fileKindOrigScore: baseScore,
243
+ _fileKindMult: mult,
244
+ _fileKindKind: kind,
245
+ _fileKindOrigIndex: i,
246
+ score: baseScore * mult,
247
+ };
248
+ }
249
+
250
+ // Stable sort: descending score, tie-break on original index.
251
+ reranked.sort((a, b) => {
252
+ const d = (b.score || 0) - (a.score || 0);
253
+ return d !== 0 ? d : a._fileKindOrigIndex - b._fileKindOrigIndex;
254
+ });
255
+
256
+ for (const r of reranked) delete r._fileKindOrigIndex;
257
+
258
+ // Concatenate unchanged tail. The cascade's CE/MaxSim score-scale
259
+ // boundary typically lives near rank `ceTopK`, so leaving rank
260
+ // `windowSize`+ untouched contains the damage from any cross-scale
261
+ // re-sort that might happen inside the window.
262
+ if (windowSize === results.length) return reranked;
263
+ return reranked.concat(results.slice(windowSize));
264
+ }
@@ -1414,10 +1414,16 @@ export class LateInteractionIndex {
1414
1414
  // don't support importance weighting, so we must use the JS-tier weighted path.
1415
1415
  const nativeScored = new Set();
1416
1416
 
1417
+ // Resolve a doc-lookup ID for each candidate. Graph-expanded candidates
1418
+ // carry `_liChunkId` (a chunk id pointing into the LI index) while their
1419
+ // public `id` is the entity id from the code graph. Honouring _liChunkId
1420
+ // lets expanded candidates participate in MaxSim rerank.
1421
+ const docIdOf = (c) => c._liChunkId || c.id;
1422
+
1417
1423
  if (useFlatPath && !this.useTokenWeights) {
1418
1424
  const groups = { bit4: [], perToken: [], perDoc: [] };
1419
1425
  for (const candidate of toScore) {
1420
- const doc = this.documents.get(candidate.id);
1426
+ const doc = this.documents.get(docIdOf(candidate));
1421
1427
  if (!doc) continue;
1422
1428
  if (doc.quantBits === 4 && doc.minArray && doc.tokenNorms) {
1423
1429
  groups.bit4.push({ candidate, doc });
@@ -1453,7 +1459,7 @@ export class LateInteractionIndex {
1453
1459
  // Try WASM fused kernels first (avoids JS-side dequant), fall back to JS dequant + wasmMaxSimF32.
1454
1460
  for (const candidate of toScore) {
1455
1461
  if (nativeScored.has(candidate.id)) continue;
1456
- const doc = this.documents.get(candidate.id);
1462
+ const doc = this.documents.get(docIdOf(candidate));
1457
1463
  if (!doc) { pushFallback(candidate); continue; }
1458
1464
 
1459
1465
  if (useFlatPath) {
@@ -1488,7 +1494,7 @@ export class LateInteractionIndex {
1488
1494
  }
1489
1495
 
1490
1496
  // JS dequant → WASM f32 or JS fallback
1491
- const flatData = this.getTokensFlat(candidate.id);
1497
+ const flatData = this.getTokensFlat(docIdOf(candidate));
1492
1498
  if (flatData) {
1493
1499
  pushScored(candidate, this.maxSimScoreFlat(
1494
1500
  effectiveQueryTokens, flatData.flat, flatData.numTokens, flatData.dim,
@@ -1498,7 +1504,7 @@ export class LateInteractionIndex {
1498
1504
  pushFallback(candidate);
1499
1505
  }
1500
1506
  } else {
1501
- const docTokens = this.getTokens(candidate.id);
1507
+ const docTokens = this.getTokens(docIdOf(candidate));
1502
1508
  if (docTokens) {
1503
1509
  pushScored(candidate, this.maxSimScore(effectiveQueryTokens, docTokens, pruneOpts));
1504
1510
  } else {
@@ -0,0 +1,304 @@
1
+ /**
2
+ * Late-interaction search-rerank policy resolver.
3
+ *
4
+ * Pure function — no I/O. The two product concepts (LI indexing model vs
5
+ * search-side LI rerank policy) are separate. This resolver computes the
6
+ * effective search-side state from explicit user choices, env vars,
7
+ * persisted init config, and the on-disk LI index manifest.
8
+ *
9
+ * The on-disk manifest is the source of truth: if the user (or auto)
10
+ * asks for rerank ON but the loaded index is missing, mismatched, or
11
+ * built with an edge model that's known to underperform as a reranker
12
+ * on benchmarked corpora, the resolver downgrades or warns accordingly.
13
+ *
14
+ * Bench evidence backing the auto rules (gencodesearchnet, 2026-05-03):
15
+ * standard `lateon-code` + LI on : 85.57 % MRR ← auto resolves ON
16
+ * edge `lateon-code-edge` + LI on : 80.65 % MRR ← auto resolves OFF
17
+ * edge + LI off : 82.91 % MRR (best edge config)
18
+ * standard + LI off : 82.91 % MRR (index-independent floor)
19
+ * See docs/BENCH_TODO.md "Phase 3 — Honest sweep before v2.5.0 (post-fix re-run)".
20
+ */
21
+
22
+ // ---------------------------------------------------------------------------
23
+ // Public model identifiers — kept in lockstep with
24
+ // `core/infrastructure/config/ranking.js::LATE_INTERACTION_CONFIG.models`.
25
+ // ---------------------------------------------------------------------------
26
+
27
+ export const LI_MODEL_STANDARD = 'lateon-code';
28
+ export const LI_MODEL_EDGE = 'lateon-code-edge';
29
+ export const LI_MODEL_NONE = 'none';
30
+
31
+ export const VALID_LI_MODELS = Object.freeze([
32
+ LI_MODEL_STANDARD,
33
+ LI_MODEL_EDGE,
34
+ LI_MODEL_NONE,
35
+ ]);
36
+
37
+ export const VALID_RERANK_POLICIES = Object.freeze(['auto', 'on', 'off']);
38
+
39
+ // ---------------------------------------------------------------------------
40
+ // Env-var coercion (same shape as other env opt-outs in the repo —
41
+ // SWEET_SEARCH_COREML_CASCADE, SWEET_SEARCH_NATIVE_INFERENCE, etc.)
42
+ // ---------------------------------------------------------------------------
43
+
44
+ function isEnvOff(value) {
45
+ if (value == null) return false;
46
+ const v = String(value).trim().toLowerCase();
47
+ return v === '0' || v === 'false' || v === 'off' || v === 'no';
48
+ }
49
+
50
+ function isEnvOn(value) {
51
+ if (value == null) return false;
52
+ const v = String(value).trim().toLowerCase();
53
+ return v === '1' || v === 'true' || v === 'on' || v === 'yes';
54
+ }
55
+
56
+ // ---------------------------------------------------------------------------
57
+ // Resolver
58
+ // ---------------------------------------------------------------------------
59
+
60
+ /**
61
+ * Resolve effective search-side LI rerank state.
62
+ *
63
+ * @param {object} input
64
+ * @param {object} [input.persisted] - parsed `.sweet-search/config.json`
65
+ * @param {string} [input.persisted.liModel] - 'lateon-code' | 'lateon-code-edge' | 'none'
66
+ * @param {string} [input.persisted.searchReranking] - 'auto' | 'on' | 'off'
67
+ * @param {object} [input.indexManifest] - loaded LI index header
68
+ * @param {string} [input.indexManifest.modelId] - id baked into the SSLX header
69
+ * @param {number} [input.indexManifest.tokenDim] - per-token dimension
70
+ * @param {boolean} [input.indexManifest.modelMismatch] - true when loaded
71
+ * modelId disagrees with the active config model
72
+ * @param {boolean} [input.indexManifest.exists] - false when no index file on disk
73
+ * @param {object} [input.env] - env-var snapshot (defaults to process.env)
74
+ * @param {boolean} [input.optionOverride] - explicit per-call override from caller
75
+ * (search-time options.useLateInteraction). If a boolean, it short-circuits the
76
+ * resolver and wins over everything else.
77
+ * @param {string} [input.activeConfigModel] - LATE_INTERACTION_CONFIG.model fallback
78
+ * (used when the index hasn't been loaded yet so we can still emit a sensible
79
+ * default — preserves back-compat with the pre-Phase-4 behaviour).
80
+ *
81
+ * @returns {{
82
+ * effective: boolean,
83
+ * policy: 'auto'|'on'|'off',
84
+ * reason: string,
85
+ * warning?: string
86
+ * }}
87
+ */
88
+ export function resolveSearchRerankPolicy(input = {}) {
89
+ const env = input.env ?? process.env;
90
+ const persisted = input.persisted ?? {};
91
+ const manifest = input.indexManifest ?? null;
92
+ const declaredPolicy = normalizePolicy(persisted.searchReranking);
93
+
94
+ // 1. Per-call explicit override (existing API, highest precedence).
95
+ if (typeof input.optionOverride === 'boolean') {
96
+ return {
97
+ effective: input.optionOverride,
98
+ policy: declaredPolicy,
99
+ reason: `per-call override (${input.optionOverride ? 'on' : 'off'})`,
100
+ };
101
+ }
102
+
103
+ // 2. Env-var hard kill switch — opt-out for benchmarks / scripts that
104
+ // must defeat any persisted policy. Mirrors SWEET_SEARCH_COREML_CASCADE=0.
105
+ if (isEnvOff(env.SWEET_SEARCH_LI_RERANK)) {
106
+ return {
107
+ effective: false,
108
+ policy: declaredPolicy,
109
+ reason: 'SWEET_SEARCH_LI_RERANK=0 (env opt-out)',
110
+ };
111
+ }
112
+ if (isEnvOn(env.SWEET_SEARCH_LI_RERANK)) {
113
+ // Env-on still goes through the safety check below — we never silently
114
+ // rerank with a missing or mismatched index.
115
+ if (!manifestUsable(manifest)) {
116
+ return {
117
+ effective: false,
118
+ policy: declaredPolicy,
119
+ reason: 'SWEET_SEARCH_LI_RERANK=1 but no usable LI index',
120
+ warning: manifestUnusableReason(manifest),
121
+ };
122
+ }
123
+ return {
124
+ effective: true,
125
+ policy: declaredPolicy,
126
+ reason: 'SWEET_SEARCH_LI_RERANK=1 (env opt-in)',
127
+ ...(isEdgeManifest(manifest) ? { warning: edgeOnWarning() } : {}),
128
+ };
129
+ }
130
+
131
+ // 3. Persisted explicit ON / OFF (init wizard / --search-reranking).
132
+ if (declaredPolicy === 'off') {
133
+ return {
134
+ effective: false,
135
+ policy: 'off',
136
+ reason: 'persisted searchReranking=off',
137
+ };
138
+ }
139
+ if (declaredPolicy === 'on') {
140
+ if (!manifestUsable(manifest)) {
141
+ return {
142
+ effective: false,
143
+ policy: 'on',
144
+ reason: 'persisted searchReranking=on but no usable LI index',
145
+ warning: manifestUnusableReason(manifest),
146
+ };
147
+ }
148
+ return {
149
+ effective: true,
150
+ policy: 'on',
151
+ reason: 'persisted searchReranking=on',
152
+ ...(isEdgeManifest(manifest) ? { warning: edgeOnWarning() } : {}),
153
+ };
154
+ }
155
+
156
+ // 4. Auto (the default): consult the index manifest.
157
+ // - missing/mismatched → off (with diagnostic)
158
+ // - edge modelId → off (Phase 3 shows edge LI rerank is net-negative)
159
+ // - any standard model → on
160
+ if (manifest != null && (manifest.exists === false)) {
161
+ return {
162
+ effective: false,
163
+ policy: 'auto',
164
+ reason: 'auto: no LI index on disk',
165
+ };
166
+ }
167
+ if (manifest != null && manifest.modelMismatch === true) {
168
+ return {
169
+ effective: false,
170
+ policy: 'auto',
171
+ reason: `auto: LI index model mismatch (header=${manifest.modelId ?? '?'})`,
172
+ };
173
+ }
174
+ if (manifest != null && isEdgeManifest(manifest)) {
175
+ return {
176
+ effective: false,
177
+ policy: 'auto',
178
+ reason: `auto: edge LI index (${manifest.modelId}) — search rerank disabled by default (see Phase 3 bench)`,
179
+ };
180
+ }
181
+ if (manifest != null && manifest.modelId) {
182
+ return {
183
+ effective: true,
184
+ policy: 'auto',
185
+ reason: `auto: standard LI index (${manifest.modelId})`,
186
+ };
187
+ }
188
+
189
+ // 5. Fallback — manifest not yet loaded (early call path) OR no manifest
190
+ // info at all. Defer to the active config model so existing flows that
191
+ // constructed SweetSearch before init() still get the historical
192
+ // behaviour. Edge model active → off; anything else → on.
193
+ if (input.activeConfigModel === LI_MODEL_EDGE) {
194
+ return {
195
+ effective: false,
196
+ policy: 'auto',
197
+ reason: 'auto: edge LI active in config (manifest not yet loaded)',
198
+ };
199
+ }
200
+ return {
201
+ effective: true,
202
+ policy: 'auto',
203
+ reason: 'auto: standard LI active in config (manifest not yet loaded)',
204
+ };
205
+ }
206
+
207
+ // ---------------------------------------------------------------------------
208
+ // Helpers (also exported for test reuse)
209
+ // ---------------------------------------------------------------------------
210
+
211
+ export function normalizePolicy(value) {
212
+ if (typeof value !== 'string') return 'auto';
213
+ const v = value.trim().toLowerCase();
214
+ return VALID_RERANK_POLICIES.includes(v) ? v : 'auto';
215
+ }
216
+
217
+ export function normalizeLiModel(value) {
218
+ if (typeof value !== 'string') return null;
219
+ const v = value.trim();
220
+ return VALID_LI_MODELS.includes(v) ? v : null;
221
+ }
222
+
223
+ export function isEdgeManifest(manifest) {
224
+ return !!manifest && manifest.modelId === LI_MODEL_EDGE;
225
+ }
226
+
227
+ export function manifestUsable(manifest) {
228
+ if (manifest == null) return false;
229
+ if (manifest.exists === false) return false;
230
+ if (manifest.modelMismatch === true) return false;
231
+ if (!manifest.modelId) return false;
232
+ return true;
233
+ }
234
+
235
+ function manifestUnusableReason(manifest) {
236
+ if (manifest == null) return 'no LI index manifest';
237
+ if (manifest.exists === false) return 'no LI index file on disk';
238
+ if (manifest.modelMismatch === true) {
239
+ return `LI index built with ${manifest.modelId ?? '?'} but config says otherwise — re-index to fix`;
240
+ }
241
+ if (!manifest.modelId) return 'LI index manifest missing modelId';
242
+ return 'unusable LI index manifest';
243
+ }
244
+
245
+ function edgeOnWarning() {
246
+ return (
247
+ 'Edge LI search reranking benchmarked below no-rerank search on '
248
+ + 'gencodesearchnet (80.65% vs 82.91% MRR). The recommended edge '
249
+ + 'setup is search reranking off — edge LI tokens still power '
250
+ + 'read-semantic and ColGrep without participating in search rerank.'
251
+ );
252
+ }
253
+
254
+ // ---------------------------------------------------------------------------
255
+ // Init-side hardware-aware default
256
+ // ---------------------------------------------------------------------------
257
+
258
+ /**
259
+ * Recommend an LI model + rerank policy from a hardware capability snapshot.
260
+ * Pure function — caller passes in detectHardwareCapability() output.
261
+ *
262
+ * Conservative: accuracy-first by default. Only recommends edge when the
263
+ * machine is clearly RAM- or disk-constrained.
264
+ *
265
+ * @param {object} hw - shape returned by detectHardwareCapability()
266
+ * @returns {{ liModel: string, searchReranking: 'auto', reason: string }}
267
+ */
268
+ export function recommendInitDefaults(hw = {}) {
269
+ const ramGB = Number(hw.totalMemGB ?? 0);
270
+ // Constrained heuristic (intentionally narrow — "accuracy-first unless
271
+ // hardware/disk clearly indicates constrained mode" per Phase 4 brief):
272
+ // - RAM ≤ 8 GB → edge
273
+ // - Apple Silicon M1/M2 (older ANE) → edge candidate
274
+ // - everything else → standard
275
+ // The bench shows standard LI is the accuracy default; only flip when
276
+ // the constrained signal is unambiguous.
277
+ const isLowRam = ramGB > 0 && ramGB <= 8;
278
+ const isOlderApple =
279
+ hw.appleSilicon && typeof hw.appleSilicon === 'object'
280
+ ? Number(hw.appleSilicon.generation ?? hw.appleSilicon.gen ?? 99) <= 2
281
+ : false;
282
+
283
+ if (isLowRam) {
284
+ return {
285
+ liModel: LI_MODEL_EDGE,
286
+ searchReranking: 'auto',
287
+ reason: `constrained: RAM=${ramGB} GB (≤8) — edge LI for indexing, search rerank auto-disables on edge per Phase 3 bench`,
288
+ };
289
+ }
290
+ if (isOlderApple && ramGB > 0 && ramGB <= 16) {
291
+ return {
292
+ liModel: LI_MODEL_EDGE,
293
+ searchReranking: 'auto',
294
+ reason: `constrained: M1/M2 + RAM=${ramGB} GB — edge LI recommended for indexing speed + disk savings`,
295
+ };
296
+ }
297
+ return {
298
+ liModel: LI_MODEL_STANDARD,
299
+ searchReranking: 'auto',
300
+ reason: ramGB > 0
301
+ ? `capable: RAM=${ramGB} GB — standard LI (accuracy default at 85.57% MRR on gencodesearchnet)`
302
+ : 'capable (default): standard LI (accuracy default)',
303
+ };
304
+ }