sweet-search 2.6.8 → 2.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/core/cli.js +7 -0
- package/core/graph/graph-extractor.js +24 -14
- package/core/graph/hcgs-generator.js +4 -3
- package/core/incremental-indexing/application/production-li-delta.mjs +28 -3
- package/core/incremental-indexing/application/production-reconciler.mjs +40 -11
- package/core/incremental-indexing/application/reconciler.mjs +1 -1
- package/core/incremental-indexing/domain/encoder-input.mjs +9 -4
- package/core/indexing/artifact-builder.js +5 -3
- package/core/indexing/ast-chunker.js +65 -62
- package/core/indexing/index-maintainer.mjs +49 -6
- package/core/indexing/indexer-build.js +18 -4
- package/core/indexing/indexer-phases.js +45 -24
- package/core/indexing/indexer-utils.js +96 -6
- package/core/indexing/indexing-file-policy.js +30 -7
- package/core/indexing/model-pool.js +26 -0
- package/core/indexing/rss-budget.mjs +65 -0
- package/core/infrastructure/code-graph-repository.js +18 -18
- package/core/infrastructure/db-utils.js +29 -0
- package/core/infrastructure/simd-distance.js +35 -21
- package/core/infrastructure/tombstone-bitmap-reader.js +3 -1
- package/core/ranking/file-kind-ranking.js +14 -2
- package/core/ranking/late-interaction-index.js +86 -15
- package/core/ranking/late-interaction-model.js +25 -4
- package/core/ranking/mmr.js +15 -10
- package/core/search/search-anchor.js +21 -3
- package/core/search/search-boost.js +25 -13
- package/core/search/search-fusion.js +13 -5
- package/core/search/search-read-semantic.js +13 -12
- package/core/search/search-server.js +2 -1
- package/core/vector-store/binary-hnsw-index.js +338 -33
- package/core/vector-store/float-vector-store.js +9 -5
- package/eval/agent-read-workflows/bin/_ss-helpers.mjs +11 -0
- package/package.json +8 -8
|
@@ -98,6 +98,23 @@ async function initWasm() {
|
|
|
98
98
|
// Eager non-blocking init
|
|
99
99
|
initWasm().catch(() => {});
|
|
100
100
|
|
|
101
|
+
// WASM memory views are cached; both modules declare fixed-size memory (no
|
|
102
|
+
// memory.grow anywhere), so the buffer identity check only fires if that
|
|
103
|
+
// ever changes. Same pattern as _wasmPerTokenLayout.
|
|
104
|
+
function wasmMemView() {
|
|
105
|
+
if (wasmMem === null || wasmMem.buffer !== wasmExports.memory.buffer) {
|
|
106
|
+
wasmMem = new Uint8Array(wasmExports.memory.buffer);
|
|
107
|
+
}
|
|
108
|
+
return wasmMem;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function maxsimMemView() {
|
|
112
|
+
if (maxsimMem === null || maxsimMem.buffer !== maxsimExports.memory.buffer) {
|
|
113
|
+
maxsimMem = new Uint8Array(maxsimExports.memory.buffer);
|
|
114
|
+
}
|
|
115
|
+
return maxsimMem;
|
|
116
|
+
}
|
|
117
|
+
|
|
101
118
|
// =============================================================================
|
|
102
119
|
// POPCOUNT LUT (JS fallback)
|
|
103
120
|
// =============================================================================
|
|
@@ -115,10 +132,9 @@ export function wasmHammingDistance(a, b) {
|
|
|
115
132
|
if (wasmExports) {
|
|
116
133
|
const aPtr = DATA_OFFSET;
|
|
117
134
|
const bPtr = DATA_OFFSET + a.length;
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
wasmMem.set(b, bPtr);
|
|
135
|
+
const mem = wasmMemView();
|
|
136
|
+
mem.set(a, aPtr);
|
|
137
|
+
mem.set(b, bPtr);
|
|
122
138
|
return wasmExports.hamming_distance(aPtr, bPtr, a.length);
|
|
123
139
|
}
|
|
124
140
|
// JS fallback
|
|
@@ -137,9 +153,9 @@ export function wasmInt8Cosine(a, b) {
|
|
|
137
153
|
if (wasmExports) {
|
|
138
154
|
const aPtr = DATA_OFFSET;
|
|
139
155
|
const bPtr = DATA_OFFSET + a.length;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
156
|
+
const mem = wasmMemView();
|
|
157
|
+
mem.set(new Uint8Array(a.buffer, a.byteOffset, a.byteLength), aPtr);
|
|
158
|
+
mem.set(new Uint8Array(b.buffer, b.byteOffset, b.byteLength), bPtr);
|
|
143
159
|
|
|
144
160
|
const dot = wasmExports.int8_dot(aPtr, bPtr, a.length);
|
|
145
161
|
const normA = wasmExports.int8_norm_sq(aPtr, a.length);
|
|
@@ -193,18 +209,18 @@ export function wasmInt8BatchDot(query, candidates) {
|
|
|
193
209
|
const alignedScoresPtr = (scoresPtr + 3) & ~3;
|
|
194
210
|
const needed = alignedScoresPtr + count * 4;
|
|
195
211
|
|
|
196
|
-
|
|
197
|
-
if (needed >
|
|
212
|
+
const mem = wasmMemView();
|
|
213
|
+
if (needed > mem.length) {
|
|
198
214
|
return _jsInt8BatchDot(query, candidates);
|
|
199
215
|
}
|
|
200
216
|
|
|
201
217
|
// Copy query once
|
|
202
|
-
|
|
218
|
+
mem.set(new Uint8Array(query.buffer, query.byteOffset, query.byteLength), queryPtr);
|
|
203
219
|
|
|
204
220
|
// Copy all candidates as contiguous slab
|
|
205
221
|
for (let i = 0; i < count; i++) {
|
|
206
222
|
const v = candidates[i];
|
|
207
|
-
|
|
223
|
+
mem.set(new Uint8Array(v.buffer, v.byteOffset, v.byteLength), slabPtr + i * dim);
|
|
208
224
|
}
|
|
209
225
|
|
|
210
226
|
// Single WASM call: scores all candidates internally, writes to output buffer
|
|
@@ -267,9 +283,9 @@ export function wasmInt8Dot(a, b) {
|
|
|
267
283
|
if (wasmExports) {
|
|
268
284
|
const aPtr = DATA_OFFSET;
|
|
269
285
|
const bPtr = DATA_OFFSET + a.length;
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
286
|
+
const mem = wasmMemView();
|
|
287
|
+
mem.set(new Uint8Array(a.buffer, a.byteOffset, a.byteLength), aPtr);
|
|
288
|
+
mem.set(new Uint8Array(b.buffer, b.byteOffset, b.byteLength), bPtr);
|
|
273
289
|
return wasmExports.int8_dot(aPtr, bPtr, a.length);
|
|
274
290
|
}
|
|
275
291
|
// JS fallback
|
|
@@ -290,9 +306,9 @@ export function wasmAsymmetricDistance(docBinary, queryInt4, queryNormScaled) {
|
|
|
290
306
|
if (wasmExports) {
|
|
291
307
|
const docPtr = DATA_OFFSET;
|
|
292
308
|
const queryPtr = DATA_OFFSET + docBinary.length;
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
309
|
+
const mem = wasmMemView();
|
|
310
|
+
mem.set(docBinary, docPtr);
|
|
311
|
+
mem.set(new Uint8Array(queryInt4.buffer, queryInt4.byteOffset, queryInt4.byteLength), queryPtr);
|
|
296
312
|
return wasmExports.asymmetric_distance(docPtr, queryPtr, queryInt4.length, queryNormScaled);
|
|
297
313
|
}
|
|
298
314
|
// JS fallback
|
|
@@ -336,9 +352,7 @@ export function wasmMaxSimF32(queryFlat, docFlat, numQ, numD, dim) {
|
|
|
336
352
|
if (!maxsimExports?.maxsim_f32) return null;
|
|
337
353
|
const exports = maxsimExports;
|
|
338
354
|
|
|
339
|
-
const mem =
|
|
340
|
-
? new Uint8Array(maxsimExports.memory.buffer)
|
|
341
|
-
: (wasmMem = new Uint8Array(wasmExports.memory.buffer));
|
|
355
|
+
const mem = maxsimMemView();
|
|
342
356
|
|
|
343
357
|
const qBytes = numQ * dim * 4;
|
|
344
358
|
const dBytes = numD * dim * 4;
|
|
@@ -375,7 +389,7 @@ export function wasmMaxSimDequant(queryFlat, docInt8, numQ, numD, dim, min, scal
|
|
|
375
389
|
const qBytes = numQ * dim * 4; // float32
|
|
376
390
|
const dBytes = numD * dim; // int8 (4x smaller!)
|
|
377
391
|
|
|
378
|
-
const mem =
|
|
392
|
+
const mem = maxsimMemView();
|
|
379
393
|
if (qBytes + dBytes + 1024 > mem.length) return null;
|
|
380
394
|
|
|
381
395
|
const qPtr = DATA_OFFSET;
|
|
@@ -49,8 +49,10 @@ export function loadBitmap(filePath) {
|
|
|
49
49
|
if (raw.length < expectedLength) {
|
|
50
50
|
throw new Error(`loadBitmap: ${filePath} truncated payload (${raw.length} bytes, expected ${expectedLength})`);
|
|
51
51
|
}
|
|
52
|
+
// View, not copy: `raw` is otherwise unreferenced, so this only keeps the
|
|
53
|
+
// 64-byte header alive alongside the payload bytes.
|
|
52
54
|
const payload = raw.subarray(HEADER_RESERVED, HEADER_RESERVED + payloadByteLength(capacity));
|
|
53
|
-
return { capacity, payload
|
|
55
|
+
return { capacity, payload };
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
export function saveBitmap(filePath, bitmap) {
|
|
@@ -533,11 +533,23 @@ function envFloat(name, dflt) {
|
|
|
533
533
|
return Number.isFinite(n) && n > 0 ? n : dflt;
|
|
534
534
|
}
|
|
535
535
|
|
|
536
|
+
// Pure function of `preferred` (static ENTITY_KIND_KEYWORDS table) — memoize
|
|
537
|
+
// the last kind so the per-result demotion loop reuses one Set.
|
|
538
|
+
let _kindKeywordSetKey;
|
|
539
|
+
let _kindKeywordSet;
|
|
540
|
+
function kindKeywordSet(preferred) {
|
|
541
|
+
if (preferred !== _kindKeywordSetKey) {
|
|
542
|
+
_kindKeywordSetKey = preferred;
|
|
543
|
+
_kindKeywordSet = new Set((ENTITY_KIND_KEYWORDS[preferred] || []).map(normalizeType));
|
|
544
|
+
}
|
|
545
|
+
return _kindKeywordSet;
|
|
546
|
+
}
|
|
547
|
+
|
|
536
548
|
function entityKindMultiplier(r, preferred, opts = {}) {
|
|
537
549
|
if (!preferred) return 1;
|
|
538
550
|
const kindBoost = envFloat('SWEET_SEARCH_KIND_BOOST', 1.10);
|
|
539
551
|
const kindDemote = envFloat('SWEET_SEARCH_KIND_DEMOTE', 0.90);
|
|
540
|
-
const wantSet =
|
|
552
|
+
const wantSet = kindKeywordSet(preferred);
|
|
541
553
|
const inferred = resolveEntityKindInfo(r, opts)?.type || '';
|
|
542
554
|
const recorded = normalizeType(resolveResultType(r));
|
|
543
555
|
const type = recorded && recorded !== 'code' && recorded !== 'chunk' ? recorded : normalizeType(inferred);
|
|
@@ -550,7 +562,7 @@ function namePrecisionMultiplier(r, preferred, nameHintsLower, opts = {}) {
|
|
|
550
562
|
if (!preferred || nameHintsLower.size === 0) return 1;
|
|
551
563
|
const exactBoost = envFloat('SWEET_SEARCH_NAME_EXACT_BOOST', 1.10);
|
|
552
564
|
const substrBoost = envFloat('SWEET_SEARCH_NAME_SUBSTR_BOOST', 1.03);
|
|
553
|
-
const wantSet =
|
|
565
|
+
const wantSet = kindKeywordSet(preferred);
|
|
554
566
|
const entityInfo = resolveEntityKindInfo(r, opts);
|
|
555
567
|
const recorded = normalizeType(resolveResultType(r));
|
|
556
568
|
const type = recorded && recorded !== 'code' && recorded !== 'chunk'
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import fs from 'fs/promises';
|
|
14
14
|
import { existsSync, createWriteStream, createReadStream, statSync } from 'fs';
|
|
15
|
+
import readline from 'readline';
|
|
15
16
|
import path from 'path';
|
|
16
17
|
import { DB_PATHS, LATE_INTERACTION_CONFIG } from '../infrastructure/config/index.js';
|
|
17
18
|
import { wasmMaxSimF32, wasmMaxSimDequantPerToken, wasmMaxSimDequant4Bit, nativeMaxSimBatch, nativeMaxSimBatchPerToken, nativeMaxSimBatch4Bit, initWasm, isNativeMaxSimAvailable, isNativePerTokenAvailable, isNative4BitAvailable } from '../infrastructure/simd-distance.js';
|
|
@@ -938,33 +939,90 @@ export class LateInteractionIndex {
|
|
|
938
939
|
}
|
|
939
940
|
|
|
940
941
|
async _saveAliasSidecar(indexPath = this.indexPath) {
|
|
942
|
+
const sidecarPath = this._aliasSidecarPath(indexPath);
|
|
941
943
|
if (this.aliasPointers.size === 0) {
|
|
942
944
|
// Remove any stale sidecar from a previous build.
|
|
943
|
-
try { await fs.unlink(
|
|
945
|
+
try { await fs.unlink(sidecarPath); } catch (_e) { /* not present */ }
|
|
944
946
|
return;
|
|
945
947
|
}
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
948
|
+
// NDJSON (version 2): one header line, then one alias pointer per line,
|
|
949
|
+
// flushed through a write stream in bounded batches. The previous
|
|
950
|
+
// monolithic JSON.stringify of the whole payload hits V8's ~512 MB string
|
|
951
|
+
// ceiling ("Invalid string length") on extreme-dedup repos, and the
|
|
952
|
+
// matching readFile on load had the same ceiling.
|
|
953
|
+
const ws = createWriteStream(sidecarPath, { encoding: 'utf8' });
|
|
954
|
+
const finished = new Promise((resolve, reject) => {
|
|
955
|
+
ws.on('error', reject);
|
|
956
|
+
ws.on('finish', resolve);
|
|
957
|
+
});
|
|
958
|
+
const flush = (str) => new Promise((resolve, reject) => {
|
|
959
|
+
ws.write(str, (err) => (err ? reject(err) : resolve()));
|
|
960
|
+
});
|
|
961
|
+
try {
|
|
962
|
+
let lines = [JSON.stringify({ version: 2, count: this.aliasPointers.size })];
|
|
963
|
+
for (const [aliasId, ptr] of this.aliasPointers) {
|
|
964
|
+
lines.push(JSON.stringify({
|
|
965
|
+
aliasId,
|
|
966
|
+
exemplarId: ptr.exemplarId,
|
|
967
|
+
clusterId: ptr.clusterId,
|
|
968
|
+
metadata: ptr.metadata || {},
|
|
969
|
+
}));
|
|
970
|
+
if (lines.length >= 20000) {
|
|
971
|
+
await flush(lines.join('\n') + '\n');
|
|
972
|
+
lines = [];
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
if (lines.length > 0) await flush(lines.join('\n') + '\n');
|
|
976
|
+
ws.end();
|
|
977
|
+
await finished;
|
|
978
|
+
} catch (err) {
|
|
979
|
+
finished.catch(() => { /* surfaced via throw below */ });
|
|
980
|
+
ws.destroy();
|
|
981
|
+
throw err;
|
|
954
982
|
}
|
|
955
|
-
const payload = { version: 1, count: entries.length, aliases: entries };
|
|
956
|
-
await fs.writeFile(this._aliasSidecarPath(indexPath), JSON.stringify(payload));
|
|
957
983
|
}
|
|
958
984
|
|
|
959
985
|
async _loadAliasSidecar(indexPath = this.indexPath) {
|
|
960
986
|
const p = this._aliasSidecarPath(indexPath);
|
|
961
987
|
if (!existsSync(p)) return;
|
|
988
|
+
// Streamed line-by-line so no single string approaches V8's ~512 MB
|
|
989
|
+
// ceiling. Two on-disk formats:
|
|
990
|
+
// v1 — one JSON.stringify'd { version: 1, aliases: [...] } line
|
|
991
|
+
// v2 — NDJSON: { version: 2, count } header, then one alias per line
|
|
992
|
+
//
|
|
993
|
+
// The input stream is destroyed in `finally`: early returns abandon the
|
|
994
|
+
// readline iterator, and rl.close() does NOT destroy its input — an
|
|
995
|
+
// fs read stream only auto-closes on 'end'/'error', so without the
|
|
996
|
+
// explicit destroy every early return leaks an fd (fatal only in
|
|
997
|
+
// long-lived processes like the daemon, but a leak everywhere).
|
|
998
|
+
const input = createReadStream(p, { encoding: 'utf8' });
|
|
999
|
+
const rl = readline.createInterface({ input, crlfDelay: Infinity });
|
|
962
1000
|
try {
|
|
963
|
-
const raw = await fs.readFile(p, 'utf-8');
|
|
964
|
-
const payload = JSON.parse(raw);
|
|
965
|
-
if (!payload || !Array.isArray(payload.aliases)) return;
|
|
966
1001
|
this.aliasPointers.clear();
|
|
967
|
-
|
|
1002
|
+
let first = true;
|
|
1003
|
+
let expected = 0;
|
|
1004
|
+
let parsed = 0;
|
|
1005
|
+
for await (const line of rl) {
|
|
1006
|
+
if (!line) continue;
|
|
1007
|
+
if (first) {
|
|
1008
|
+
first = false;
|
|
1009
|
+
const head = JSON.parse(line);
|
|
1010
|
+
if (head && Array.isArray(head.aliases)) {
|
|
1011
|
+
// v1 monolithic payload — the whole sidecar is this one line,
|
|
1012
|
+
// so JSON.parse succeeding IS the integrity check (any
|
|
1013
|
+
// truncation makes it unparseable).
|
|
1014
|
+
for (const { aliasId, exemplarId, clusterId, metadata } of head.aliases) {
|
|
1015
|
+
if (!this.documents.has(exemplarId)) continue; // orphan guard, see below
|
|
1016
|
+
this.aliasPointers.set(aliasId, { exemplarId, clusterId, metadata: metadata || {} });
|
|
1017
|
+
}
|
|
1018
|
+
return;
|
|
1019
|
+
}
|
|
1020
|
+
if (!head || head.version !== 2 || !Number.isFinite(head.count)) return;
|
|
1021
|
+
expected = head.count;
|
|
1022
|
+
continue;
|
|
1023
|
+
}
|
|
1024
|
+
const { aliasId, exemplarId, clusterId, metadata } = JSON.parse(line);
|
|
1025
|
+
parsed++;
|
|
968
1026
|
// Orphan guard: drop aliases whose exemplar is no longer in documents.
|
|
969
1027
|
// Happens if the file containing the exemplar was removed between
|
|
970
1028
|
// save and load (incremental re-index removed the exemplar file
|
|
@@ -972,8 +1030,21 @@ export class LateInteractionIndex {
|
|
|
972
1030
|
if (!this.documents.has(exemplarId)) continue;
|
|
973
1031
|
this.aliasPointers.set(aliasId, { exemplarId, clusterId, metadata: metadata || {} });
|
|
974
1032
|
}
|
|
1033
|
+
// Truncation guard: NDJSON has no whole-file parse to fail, so a file
|
|
1034
|
+
// cut exactly at a line boundary (crash mid-save, disk full) is valid
|
|
1035
|
+
// prefix NDJSON and would otherwise load a silent subset. The header's
|
|
1036
|
+
// `count` restores v1's all-or-nothing semantics. Compared against
|
|
1037
|
+
// PARSED lines, not kept aliases — the orphan guard intentionally
|
|
1038
|
+
// drops entries and must not trip this.
|
|
1039
|
+
if (parsed !== expected) {
|
|
1040
|
+
this.aliasPointers.clear();
|
|
1041
|
+
}
|
|
975
1042
|
} catch (_e) {
|
|
976
1043
|
// Malformed sidecar — treat as absent; aliases will be skipped at query time.
|
|
1044
|
+
this.aliasPointers.clear();
|
|
1045
|
+
} finally {
|
|
1046
|
+
rl.close();
|
|
1047
|
+
input.destroy();
|
|
977
1048
|
}
|
|
978
1049
|
}
|
|
979
1050
|
|
|
@@ -26,6 +26,18 @@ let lateInteractionPipeline = null;
|
|
|
26
26
|
let loadPromise = null;
|
|
27
27
|
let lateInteractionRuntimeConfig = {
|
|
28
28
|
intraOpThreads: null,
|
|
29
|
+
// G3-LI: background/maintainer ORT profile for the LI session. When truthy,
|
|
30
|
+
// the session is created with the CPU mem arena OFF and force_spinning_stop
|
|
31
|
+
// instead of allow_spinning — the same profile buildLocalSessionOptions
|
|
32
|
+
// applies to the dense session. ORT never returns arena memory once grown
|
|
33
|
+
// (#25325), and per-file reconcile encodes produce highly variable batch
|
|
34
|
+
// shapes, so an arena-ON LI session in the resident maintainer accrues
|
|
35
|
+
// monotonic RSS in 128MB arena-extension steps (measured: 354×128MB ≈ 34GB
|
|
36
|
+
// after one heavy edit day). Must be set BEFORE the first encode — the
|
|
37
|
+
// session singleton is built once; configuring after is a silent no-op.
|
|
38
|
+
// Default null/off everywhere else (search/query path keeps the foreground
|
|
39
|
+
// arena+spinning profile for latency).
|
|
40
|
+
background: null,
|
|
29
41
|
};
|
|
30
42
|
|
|
31
43
|
// Lightweight timing accumulators for profiling (Phase 6a).
|
|
@@ -90,6 +102,7 @@ export function configureLateInteractionRuntime(overrides = {}) {
|
|
|
90
102
|
export function resetLateInteractionRuntime() {
|
|
91
103
|
lateInteractionRuntimeConfig = {
|
|
92
104
|
intraOpThreads: null,
|
|
105
|
+
background: null,
|
|
93
106
|
};
|
|
94
107
|
}
|
|
95
108
|
|
|
@@ -191,18 +204,26 @@ async function loadModel() {
|
|
|
191
204
|
// 'extended' + memArena + memPattern: marginal overhead for no measurable gain.
|
|
192
205
|
// Conclusion: keep LI session lean. Only proven-beneficial options added.
|
|
193
206
|
const { getOptimizedGraphPath } = await import('../infrastructure/onnx-session-utils.js');
|
|
207
|
+
const liBackground = !!lateInteractionRuntimeConfig.background;
|
|
194
208
|
const session = await ort.InferenceSession.create(onnxPath, {
|
|
195
209
|
executionProviders: ['cpu'],
|
|
196
210
|
logSeverityLevel: 3, // ERROR — silence ORT's expected "optimized model is machine-specific" warning
|
|
197
211
|
intraOpNumThreads: lateInteractionRuntimeConfig.intraOpThreads ?? bestIntraOpThreads(),
|
|
198
212
|
interOpNumThreads: 1,
|
|
199
213
|
optimizedModelFilePath: getOptimizedGraphPath(modelConfig.hfId, 'lateon'),
|
|
214
|
+
// G3-LI: background (maintainer) profile disables the CPU mem arena — ORT
|
|
215
|
+
// never returns arena memory once grown (#25325), and variable-shaped
|
|
216
|
+
// per-file reconcile batches make the arena extend monotonically (128MB
|
|
217
|
+
// steps) in a resident daemon. Foreground keeps the arena for throughput.
|
|
218
|
+
enableCpuMemArena: !liBackground,
|
|
200
219
|
// Thread spinning keeps ORT worker threads hot between batches — trades idle
|
|
201
|
-
// CPU for lower per-batch latency during sustained indexing runs.
|
|
220
|
+
// CPU for lower per-batch latency during sustained indexing runs. The
|
|
221
|
+
// background profile parks workers after the last Run() instead (the
|
|
222
|
+
// maintainer sits idle 20-60s between ticks; ~14% re-spin latency cost).
|
|
202
223
|
extra: {
|
|
203
|
-
session:
|
|
204
|
-
|
|
205
|
-
|
|
224
|
+
session: liBackground
|
|
225
|
+
? { force_spinning_stop: '1' }
|
|
226
|
+
: { intra_op: { allow_spinning: '1' } },
|
|
206
227
|
},
|
|
207
228
|
});
|
|
208
229
|
const coremlActive = false;
|
package/core/ranking/mmr.js
CHANGED
|
@@ -52,6 +52,11 @@ export const MMR_CONFIG = {
|
|
|
52
52
|
* @param {Object} weights - Feature weights
|
|
53
53
|
* @returns {number} Similarity score [0, 1]
|
|
54
54
|
*/
|
|
55
|
+
// Static type groupings used by computeSimilarity — hoisted out of the
|
|
56
|
+
// O(candidates × selected) MMR inner loop.
|
|
57
|
+
const SIMILARITY_DEF_TYPES = new Set(['class', 'interface', 'struct', 'enum', 'trait']);
|
|
58
|
+
const SIMILARITY_METHOD_TYPES = new Set(['method', 'function', 'constructor']);
|
|
59
|
+
|
|
55
60
|
export function computeSimilarity(a, b, weights = MMR_CONFIG.weights) {
|
|
56
61
|
let totalSim = 0;
|
|
57
62
|
let totalWeight = 0;
|
|
@@ -83,11 +88,8 @@ export function computeSimilarity(a, b, weights = MMR_CONFIG.weights) {
|
|
|
83
88
|
totalSim += weights.type * 1.0;
|
|
84
89
|
} else {
|
|
85
90
|
// Partial similarity for related types
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
if ((defTypes.has(typeA) && defTypes.has(typeB)) ||
|
|
90
|
-
(methodTypes.has(typeA) && methodTypes.has(typeB))) {
|
|
91
|
+
if ((SIMILARITY_DEF_TYPES.has(typeA) && SIMILARITY_DEF_TYPES.has(typeB)) ||
|
|
92
|
+
(SIMILARITY_METHOD_TYPES.has(typeA) && SIMILARITY_METHOD_TYPES.has(typeB))) {
|
|
91
93
|
totalSim += weights.type * 0.5;
|
|
92
94
|
}
|
|
93
95
|
}
|
|
@@ -237,11 +239,19 @@ export function applyMMR(results, options = {}) {
|
|
|
237
239
|
while (selected.length < k && remaining.size > 0) {
|
|
238
240
|
let bestIdx = -1;
|
|
239
241
|
let bestMMR = -Infinity;
|
|
242
|
+
// Highest-relevance tracking shares the same pass (same iteration order
|
|
243
|
+
// and strict-> comparison as the reduce it replaces).
|
|
244
|
+
let highestRelevanceIdx = -1;
|
|
240
245
|
|
|
241
246
|
// Find the candidate with highest MMR score
|
|
242
247
|
for (const idx of remaining) {
|
|
243
248
|
const candidate = normalized[idx];
|
|
244
249
|
|
|
250
|
+
if (highestRelevanceIdx === -1
|
|
251
|
+
|| candidate._normalizedRelevance > normalized[highestRelevanceIdx]._normalizedRelevance) {
|
|
252
|
+
highestRelevanceIdx = idx;
|
|
253
|
+
}
|
|
254
|
+
|
|
245
255
|
// Compute max similarity to already selected results
|
|
246
256
|
let maxSim = 0;
|
|
247
257
|
for (const selectedResult of selected) {
|
|
@@ -261,11 +271,6 @@ export function applyMMR(results, options = {}) {
|
|
|
261
271
|
if (bestIdx >= 0) {
|
|
262
272
|
const selectedCandidate = normalized[bestIdx];
|
|
263
273
|
|
|
264
|
-
// Track if this selection was reordered (not the highest relevance remaining)
|
|
265
|
-
const highestRelevanceIdx = [...remaining].reduce((maxIdx, idx) =>
|
|
266
|
-
normalized[idx]._normalizedRelevance > normalized[maxIdx]._normalizedRelevance ? idx : maxIdx
|
|
267
|
-
);
|
|
268
|
-
|
|
269
274
|
if (bestIdx !== highestRelevanceIdx) {
|
|
270
275
|
reorderCount++;
|
|
271
276
|
totalDiversityPenalty += normalized[highestRelevanceIdx]._normalizedRelevance - selectedCandidate._normalizedRelevance;
|
|
@@ -129,7 +129,23 @@ function readUniquenessCeil(opts) {
|
|
|
129
129
|
* @param {{ filePath: string, startLine: number, endLine: number }} entity
|
|
130
130
|
* @returns {{ id: string, metadata: object, content?: string, text?: string }|null}
|
|
131
131
|
*/
|
|
132
|
-
|
|
132
|
+
// Bucket the LI documents Map by file in ONE pass, preserving per-file
|
|
133
|
+
// iteration order, so findChunkForEntity scans only entity.filePath's chunks
|
|
134
|
+
// instead of the whole index per entity.
|
|
135
|
+
function bucketDocsByFile(liIndex) {
|
|
136
|
+
const byFile = new Map();
|
|
137
|
+
if (!liIndex?.documents) return byFile;
|
|
138
|
+
for (const [id, doc] of liIndex.documents) {
|
|
139
|
+
const file = doc?.metadata?.file;
|
|
140
|
+
if (!file) continue;
|
|
141
|
+
let arr = byFile.get(file);
|
|
142
|
+
if (!arr) byFile.set(file, arr = []);
|
|
143
|
+
arr.push([id, doc]);
|
|
144
|
+
}
|
|
145
|
+
return byFile;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function findChunkForEntity(liIndex, entity, docsByFile = null) {
|
|
133
149
|
if (!liIndex || !entity) return null;
|
|
134
150
|
let best = null;
|
|
135
151
|
let bestSize = Infinity;
|
|
@@ -145,7 +161,8 @@ function findChunkForEntity(liIndex, entity) {
|
|
|
145
161
|
let headerBest = null;
|
|
146
162
|
let headerBestSize = Infinity;
|
|
147
163
|
const entityNameLc = String(entity.name || '').toLowerCase();
|
|
148
|
-
|
|
164
|
+
const candidates = docsByFile ? (docsByFile.get(entity.filePath) || []) : liIndex.documents;
|
|
165
|
+
for (const [id, doc] of candidates) {
|
|
149
166
|
const m = doc?.metadata;
|
|
150
167
|
if (!m || m.file !== entity.filePath) continue;
|
|
151
168
|
const cs = m.startLine, ce = m.endLine;
|
|
@@ -402,10 +419,11 @@ export function injectAnchorCandidates(fused, query, opts = {}) {
|
|
|
402
419
|
let existingBoosted = 0;
|
|
403
420
|
const out = fused.slice(); // copy — we'll append injections
|
|
404
421
|
const seenInjected = new Set();
|
|
422
|
+
const docsByFile = bucketDocsByFile(liIndex);
|
|
405
423
|
|
|
406
424
|
for (const entity of entities) {
|
|
407
425
|
if (!entityMatchesAnchorHint(entity, hints)) continue;
|
|
408
|
-
const chunk = findChunkForEntity(liIndex, entity);
|
|
426
|
+
const chunk = findChunkForEntity(liIndex, entity, docsByFile);
|
|
409
427
|
if (!chunk) continue;
|
|
410
428
|
const key = chunkKey({ metadata: chunk.metadata });
|
|
411
429
|
if (seenInjected.has(key)) continue;
|
|
@@ -201,6 +201,18 @@ function envFloat(name, fallback, min = 0, max = 1) {
|
|
|
201
201
|
return Number.isFinite(parsed) && parsed >= min && parsed <= max ? parsed : fallback;
|
|
202
202
|
}
|
|
203
203
|
|
|
204
|
+
// Query-term derivation is pure in `query`; memoize the last query so the
|
|
205
|
+
// per-result map in applyPostFusionBoosts doesn't re-split identical input.
|
|
206
|
+
let _lastIdTermsQuery = null;
|
|
207
|
+
let _lastIdTerms = null;
|
|
208
|
+
function identifierTermsForQuery(query) {
|
|
209
|
+
if (query !== _lastIdTermsQuery) {
|
|
210
|
+
_lastIdTermsQuery = query;
|
|
211
|
+
_lastIdTerms = new Set(splitIdentifierTerms(query));
|
|
212
|
+
}
|
|
213
|
+
return _lastIdTerms;
|
|
214
|
+
}
|
|
215
|
+
|
|
204
216
|
function splitIdentifierTerms(value) {
|
|
205
217
|
return String(value || '')
|
|
206
218
|
.replace(/_[0-9a-f]{8}(?=\.[^.]+$|$)/gi, '')
|
|
@@ -233,7 +245,7 @@ export function computeIdentifierAgreementBoost(result, query) {
|
|
|
233
245
|
const weight = envFloat('SWEET_SEARCH_IDENTIFIER_AGREEMENT_BOOST', 0.40, 0, 1);
|
|
234
246
|
if (weight === 0) return 1.0;
|
|
235
247
|
|
|
236
|
-
const queryTerms =
|
|
248
|
+
const queryTerms = identifierTermsForQuery(query);
|
|
237
249
|
if (queryTerms.size === 0) return 1.0;
|
|
238
250
|
|
|
239
251
|
const fileName = (result.file || result.path || result.metadata?.file || '')
|
|
@@ -290,22 +302,22 @@ export function computeDefinitionBoost(result, queryLower, queryTokens) {
|
|
|
290
302
|
/**
|
|
291
303
|
* Compute syntax boost (PHASE_1_FIXES helper)
|
|
292
304
|
*/
|
|
305
|
+
const DEFINITION_SIGNATURE_PATTERNS = [
|
|
306
|
+
/\b(?:public|private|protected)?\s*(?:abstract|final)?\s*class\s+(\w+)/,
|
|
307
|
+
/\b(?:public|private|protected)?\s*interface\s+(\w+)/,
|
|
308
|
+
/\b(?:public|private|protected)?\s*enum\s+(\w+)/,
|
|
309
|
+
/\bclass\s+(\w+)/,
|
|
310
|
+
/\bfunction\s+(\w+)/,
|
|
311
|
+
/\bexport\s+(?:default\s+)?(?:class|function)\s+(\w+)/,
|
|
312
|
+
/\binterface\s+(\w+)/,
|
|
313
|
+
/\btype\s+(\w+)\s*=/,
|
|
314
|
+
];
|
|
315
|
+
|
|
293
316
|
export function computeSyntaxBoost(result, queryTokens) {
|
|
294
317
|
const signature = (result.signature || '').toLowerCase();
|
|
295
318
|
if (!signature) return 1.0;
|
|
296
319
|
|
|
297
|
-
const
|
|
298
|
-
/\b(?:public|private|protected)?\s*(?:abstract|final)?\s*class\s+(\w+)/,
|
|
299
|
-
/\b(?:public|private|protected)?\s*interface\s+(\w+)/,
|
|
300
|
-
/\b(?:public|private|protected)?\s*enum\s+(\w+)/,
|
|
301
|
-
/\bclass\s+(\w+)/,
|
|
302
|
-
/\bfunction\s+(\w+)/,
|
|
303
|
-
/\bexport\s+(?:default\s+)?(?:class|function)\s+(\w+)/,
|
|
304
|
-
/\binterface\s+(\w+)/,
|
|
305
|
-
/\btype\s+(\w+)\s*=/,
|
|
306
|
-
];
|
|
307
|
-
|
|
308
|
-
for (const pattern of definitionPatterns) {
|
|
320
|
+
for (const pattern of DEFINITION_SIGNATURE_PATTERNS) {
|
|
309
321
|
const match = signature.match(pattern);
|
|
310
322
|
if (match && match[1]) {
|
|
311
323
|
const definedName = match[1].toLowerCase();
|
|
@@ -109,20 +109,24 @@ export function quantileNormalize(scores, lowQuantile = 0.05, highQuantile = 0.9
|
|
|
109
109
|
export function convexCombination(lexicalResults, semanticResults, routeType = 'mixed', options = {}) {
|
|
110
110
|
const alpha = options.alpha ?? ROUTE_ALPHAS[routeType] ?? 0.5;
|
|
111
111
|
|
|
112
|
-
// Build score maps
|
|
112
|
+
// Build score + result maps in one key derivation per result
|
|
113
113
|
const lexicalScoreMap = new Map();
|
|
114
114
|
const semanticScoreMap = new Map();
|
|
115
|
+
const lexResultMap = new Map();
|
|
116
|
+
const semResultMap = new Map();
|
|
115
117
|
const allKeys = new Set();
|
|
116
118
|
|
|
117
119
|
for (const r of lexicalResults) {
|
|
118
120
|
const key = this.getResultKey(r);
|
|
119
121
|
lexicalScoreMap.set(key, r.score);
|
|
122
|
+
lexResultMap.set(key, r);
|
|
120
123
|
allKeys.add(key);
|
|
121
124
|
}
|
|
122
125
|
|
|
123
126
|
for (const r of semanticResults) {
|
|
124
127
|
const key = this.getResultKey(r);
|
|
125
128
|
semanticScoreMap.set(key, r.score);
|
|
129
|
+
semResultMap.set(key, r);
|
|
126
130
|
allKeys.add(key);
|
|
127
131
|
}
|
|
128
132
|
|
|
@@ -148,8 +152,6 @@ export function convexCombination(lexicalResults, semanticResults, routeType = '
|
|
|
148
152
|
|
|
149
153
|
// Compute CC scores
|
|
150
154
|
const ccResults = [];
|
|
151
|
-
const lexResultMap = new Map(lexicalResults.map(r => [this.getResultKey(r), r]));
|
|
152
|
-
const semResultMap = new Map(semanticResults.map(r => [this.getResultKey(r), r]));
|
|
153
155
|
|
|
154
156
|
for (const key of allKeys) {
|
|
155
157
|
const lexScore = lexicalScoreMap.get(key);
|
|
@@ -253,7 +255,10 @@ export function rrfFusion(lexicalResults, semanticResults, k = 60) {
|
|
|
253
255
|
});
|
|
254
256
|
|
|
255
257
|
return [...results.values()]
|
|
256
|
-
.map(r =>
|
|
258
|
+
.map(r => {
|
|
259
|
+
const score = scores.get(this.getResultKey(r));
|
|
260
|
+
return { ...r, score, ccScore: score };
|
|
261
|
+
})
|
|
257
262
|
.sort((a, b) => b.score - a.score);
|
|
258
263
|
}
|
|
259
264
|
|
|
@@ -310,7 +315,10 @@ export function robustCCFusion(lexicalResults, semanticResults, routeType = 'mix
|
|
|
310
315
|
|
|
311
316
|
return {
|
|
312
317
|
results: [...results.values()]
|
|
313
|
-
.map(r =>
|
|
318
|
+
.map(r => {
|
|
319
|
+
const score = combinedScores.get(this.getResultKey(r));
|
|
320
|
+
return { ...r, score, ccScore: score, alpha };
|
|
321
|
+
})
|
|
314
322
|
.sort((a, b) => b.score - a.score),
|
|
315
323
|
method: 'cc_robust',
|
|
316
324
|
fallbackReason: null,
|