sweet-search 2.6.8 → 2.6.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +15 -0
  2. package/core/cli.js +7 -0
  3. package/core/graph/graph-extractor.js +24 -14
  4. package/core/graph/hcgs-generator.js +4 -3
  5. package/core/incremental-indexing/application/production-li-delta.mjs +28 -3
  6. package/core/incremental-indexing/application/production-reconciler.mjs +40 -11
  7. package/core/incremental-indexing/application/reconciler.mjs +1 -1
  8. package/core/incremental-indexing/domain/encoder-input.mjs +9 -4
  9. package/core/indexing/artifact-builder.js +5 -3
  10. package/core/indexing/ast-chunker.js +65 -62
  11. package/core/indexing/index-maintainer.mjs +49 -6
  12. package/core/indexing/indexer-build.js +18 -4
  13. package/core/indexing/indexer-phases.js +45 -24
  14. package/core/indexing/indexer-utils.js +96 -6
  15. package/core/indexing/indexing-file-policy.js +30 -7
  16. package/core/indexing/model-pool.js +26 -0
  17. package/core/indexing/rss-budget.mjs +65 -0
  18. package/core/infrastructure/code-graph-repository.js +18 -18
  19. package/core/infrastructure/db-utils.js +29 -0
  20. package/core/infrastructure/simd-distance.js +35 -21
  21. package/core/infrastructure/tombstone-bitmap-reader.js +3 -1
  22. package/core/ranking/file-kind-ranking.js +14 -2
  23. package/core/ranking/late-interaction-index.js +86 -15
  24. package/core/ranking/late-interaction-model.js +25 -4
  25. package/core/ranking/mmr.js +15 -10
  26. package/core/search/search-anchor.js +21 -3
  27. package/core/search/search-boost.js +25 -13
  28. package/core/search/search-fusion.js +13 -5
  29. package/core/search/search-read-semantic.js +13 -12
  30. package/core/search/search-server.js +2 -1
  31. package/core/vector-store/binary-hnsw-index.js +338 -33
  32. package/core/vector-store/float-vector-store.js +9 -5
  33. package/eval/agent-read-workflows/bin/_ss-helpers.mjs +11 -0
  34. package/package.json +8 -8
package/README.md CHANGED
@@ -711,6 +711,21 @@ without you ever running a command.
711
711
  - **Resource-polite:** ticks are budgeted (≤50 files / ≤2 s CPU per tick), run CPU-only (the GPU is reserved for cold full indexing), and the interval auto-tunes from load average, churn, and backlog.
712
712
  - `sweet-search reconcile status` / `reconcile inspect <path>` explain exactly what the daemon thinks and why. Opt out any time with `SWEET_SEARCH_RECONCILE_V2=0`.
713
713
 
714
+ **Memory controls.** The resident daemons show up in `ps` / Activity Monitor as
715
+ `sweet-search-maintainer` and `sweet-search-daemon`. A maintainer's steady state is
716
+ roughly 2–3 GB (embedding + late-interaction models stay loaded so ticks are fast),
717
+ and four independent mechanisms keep that bounded:
718
+
719
+ | Mechanism | Default | Override |
720
+ |-----------|---------|----------|
721
+ | Background ORT profile (arena-off + parked threads) in the maintainer | on | `SWEET_SEARCH_ORT_BACKGROUND=0` |
722
+ | Per-process recycle ceiling — the maintainer finishes its tick, exits cleanly, and respawns fresh on the next edit when its RSS crosses the line | clamp(25 % of RAM, 4 GiB, 8 GiB) | `SWEET_SEARCH_MAINTAINER_RSS_MAX_MB` (0 disables) |
723
+ | Idle TTL — unattended daemons shut down and respawn on demand | tier-aware | `SWEET_SEARCH_MAINTAINER_IDLE_TTL_MS` / `SWEET_SEARCH_DAEMON_IDLE_TTL_MS` |
724
+ | Fleet RSS budget — across all repos' daemons, the longest-idle one is evicted when the sum crosses a RAM-scaled budget | tier-aware | `SWEET_SEARCH_RSS_BUDGET_FRACTION` |
725
+
726
+ A recycle or eviction never touches index state: every tick publishes atomically
727
+ before the process exits, and the next edit (or query) respawns a fresh daemon.
728
+
714
729
  </details>
715
730
 
716
731
  ## 🦀 The Native Engine Room
package/core/cli.js CHANGED
@@ -94,6 +94,13 @@ if (args[0] === 'init') {
94
94
  await runIndexer();
95
95
  }
96
96
  } else if (args[0] === '--serve' || args[0] === '--stop') {
97
+ // Name the resident daemon so ps/Activity Monitor shows what it is instead
98
+ // of an anonymous `node`. Entrypoint-scoped: library consumers of the
99
+ // search modules are never retitled. Liveness checks are lock/pid-based,
100
+ // never command-line matching, so this is cosmetic-only.
101
+ if (args[0] === '--serve') {
102
+ try { process.title = 'sweet-search-daemon'; } catch { /* best-effort */ }
103
+ }
97
104
  // Warm search server lifecycle is implemented in JS.
98
105
  const { runCli } = await import('./search/index.js');
99
106
  await runCli(args);
@@ -2351,7 +2351,28 @@ function resolveThrows(exceptionName, entityByName) {
2351
2351
  * Insert entities and relationships into database
2352
2352
  * Uses better-sqlite3 (sync API, no .free() needed)
2353
2353
  */
2354
- export function insertGraph(db, entities, relationships, hasFts5 = false) {
2354
+ /**
2355
+ * Rebuild + optimize the external-content FTS5 mirrors from the entities
2356
+ * table. 'rebuild' reconstructs the whole FTS index from current content, so
2357
+ * only the LAST rebuild before any FTS read matters — batch loops should
2358
+ * pass { syncFts: false } to insertGraph and call this once at the end.
2359
+ */
2360
+ export function rebuildGraphFts(db) {
2361
+ try {
2362
+ db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('rebuild')`);
2363
+ db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('rebuild')`);
2364
+ console.log(' FTS5 indexes rebuilt (porter + trigram)');
2365
+
2366
+ // Best-effort post-build compaction for faster reads.
2367
+ db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('optimize')`);
2368
+ db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('optimize')`);
2369
+ console.log(' FTS5 indexes optimized (segments merged)');
2370
+ } catch (err) {
2371
+ // FTS5 rebuild/optimize failed, ignore
2372
+ }
2373
+ }
2374
+
2375
+ export function insertGraph(db, entities, relationships, hasFts5 = false, { syncFts = true } = {}) {
2355
2376
  // Insert entities with HCGS hierarchy support
2356
2377
  // Includes signature_hash for collision-proof backup/restore
2357
2378
  const entityStmt = db.prepare(`
@@ -2481,19 +2502,8 @@ export function insertGraph(db, entities, relationships, hasFts5 = false) {
2481
2502
  // }
2482
2503
 
2483
2504
  // Rebuild FTS indexes if available
2484
- if (hasFts5) {
2485
- try {
2486
- db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('rebuild')`);
2487
- db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('rebuild')`);
2488
- console.log(' FTS5 indexes rebuilt (porter + trigram)');
2489
-
2490
- // Best-effort post-build compaction for faster reads.
2491
- db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('optimize')`);
2492
- db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('optimize')`);
2493
- console.log(' FTS5 indexes optimized (segments merged)');
2494
- } catch (err) {
2495
- // FTS5 rebuild/optimize failed, ignore
2496
- }
2505
+ if (hasFts5 && syncFts) {
2506
+ rebuildGraphFts(db);
2497
2507
  }
2498
2508
  }
2499
2509
 
@@ -428,9 +428,10 @@ async function generateSummariesForEntities(entityIds, options = {}) {
428
428
  const dbWrite = new Database(dbPath);
429
429
  const stmt = dbWrite.prepare(`UPDATE entities SET summary = NULL WHERE id = ?`);
430
430
 
431
- for (const entity of entities) {
432
- stmt.run(entity.id);
433
- }
431
+ const clearSummaries = dbWrite.transaction((rows) => {
432
+ for (const entity of rows) stmt.run(entity.id);
433
+ });
434
+ clearSummaries(entities);
434
435
 
435
436
  dbWrite.close();
436
437
 
@@ -117,6 +117,7 @@ export async function applyLateInteractionDelta({
117
117
  liEncoder,
118
118
  pickLiInput,
119
119
  onProgress = null,
120
+ readerCache = null,
120
121
  }) {
121
122
  const progress = typeof onProgress === 'function'
122
123
  ? (phase) => { onProgress(phase); }
@@ -128,11 +129,35 @@ export async function applyLateInteractionDelta({
128
129
  const encode = liEncoder || ((texts) => encodeDocumentsCpu(texts));
129
130
  const existing = fs.existsSync(indexPath);
130
131
  const segmented = existing ? segmentedState(indexPath) : null;
131
- const index = new LateInteractionIndex({ indexPath, loadExisting: true });
132
- await index.init();
133
- progress('li:init');
132
+
133
+ // E.1-LI reader cache: reuse the loaded read view across the tick's files.
134
+ // Only the SEGMENTED path is cacheable — everything the reader serves there
135
+ // (config fields, positions/counts of pre-tick docs) is immutable within a
136
+ // tick: per-file ops only reference the file's own pre-tick docs, appends
137
+ // go through appendGrowingSegment (which re-reads the manifest from disk),
138
+ // and tombstone sidecar state is opened fresh per call. The legacy path
139
+ // MUTATES the loaded index (rewriteLegacyIndex), so it always loads fresh
140
+ // and drops any cached reader.
141
+ const cacheable = !!(readerCache && segmented);
142
+ let index;
143
+ if (cacheable && readerCache.index && readerCache.key === indexPath) {
144
+ index = readerCache.index;
145
+ progress('li:init-cached');
146
+ } else {
147
+ index = new LateInteractionIndex({ indexPath, loadExisting: true });
148
+ await index.init();
149
+ progress('li:init');
150
+ if (cacheable) {
151
+ readerCache.key = indexPath;
152
+ readerCache.index = index;
153
+ }
154
+ }
134
155
 
135
156
  if (existing && !segmented) {
157
+ if (readerCache) {
158
+ readerCache.key = null;
159
+ readerCache.index = null;
160
+ }
136
161
  return rewriteLegacyIndex(index, ops, encode, pickLiInput, progress);
137
162
  }
138
163
 
@@ -34,6 +34,7 @@ import { floatToBinary, normalizedFloatToInt8, truncateForHNSW } from '../../inf
34
34
  import { extractSparseGramDeltaRecord } from '../../infrastructure/native-sparse-gram.js';
35
35
  import { migrateEntitiesSchema, migrateRelationshipsSchema } from '../infrastructure/schema-migrations.mjs';
36
36
  import { readMaintenanceState as readMaintenanceStateFromArtifacts } from '../infrastructure/maintenance-state-reader.mjs';
37
+ import { prepareCached } from '../../infrastructure/db-utils.js';
37
38
 
38
39
  const DIRTY_QUEUE = 'index-maintainer-queue.jsonl';
39
40
  const PROCESSING_QUEUE = 'index-maintainer-queue.processing.jsonl';
@@ -270,9 +271,9 @@ async function enrichChunksFromGraph(chunks, stateDir, tickCtx = null) {
270
271
  ownConn = true;
271
272
  }
272
273
  try {
273
- const entityStmt = db.prepare('SELECT type, name, start_line, end_line FROM entities WHERE file_path = ? AND epoch_retired IS NULL ORDER BY start_line ASC');
274
- const fileEntityStmt = db.prepare('SELECT id FROM entities WHERE file_path = ? AND logical_entity_id = ? AND epoch_retired IS NULL ORDER BY epoch_written DESC LIMIT 1');
275
- const importStmt = db.prepare("SELECT DISTINCT target_name FROM relationships WHERE source_id = ? AND type IN ('imports', 'plainImport') AND epoch_retired IS NULL ORDER BY target_name");
274
+ const entityStmt = prepareCached(db, 'SELECT type, name, start_line, end_line FROM entities WHERE file_path = ? AND epoch_retired IS NULL ORDER BY start_line ASC');
275
+ const fileEntityStmt = prepareCached(db, 'SELECT id FROM entities WHERE file_path = ? AND logical_entity_id = ? AND epoch_retired IS NULL ORDER BY epoch_written DESC LIMIT 1');
276
+ const importStmt = prepareCached(db, "SELECT DISTINCT target_name FROM relationships WHERE source_id = ? AND type IN ('imports', 'plainImport') AND epoch_retired IS NULL ORDER BY target_name");
276
277
  for (const chunk of chunks) {
277
278
  const file = chunk.file || chunk.metadata?.relative_path;
278
279
  const symbol = chunk.metadata?.symbol;
@@ -376,6 +377,23 @@ class ProductionReconcileAdapter {
376
377
  // when the flag is on; null when disabled.
377
378
  this._cutoffCache = null;
378
379
  this._cutoffDirty = false;
380
+ // merkle-state.json is written exactly once per tick (persistManifest);
381
+ // every other reader gets one shared parse. `undefined` = not read yet,
382
+ // `null` = missing/corrupt file.
383
+ this._merkleCache = undefined;
384
+ }
385
+
386
+ /**
387
+ * Shared read of merkle-state.json — parsed once per adapter lifetime and
388
+ * invalidated by persistManifest (the only writer). All consumers treat the
389
+ * object as read-only.
390
+ * @returns {object|null}
391
+ */
392
+ _merkleRead() {
393
+ if (this._merkleCache === undefined) {
394
+ this._merkleCache = readJson(path.join(this.stateDir, MERKLE_STATE), null);
395
+ }
396
+ return this._merkleCache;
379
397
  }
380
398
 
381
399
  progress(phase) {
@@ -401,7 +419,7 @@ class ProductionReconcileAdapter {
401
419
  applyGraphDelta: (file, hashes, epoch, ctx) => this.applyGraphDelta(file, hashes, epoch, ctx),
402
420
  applyVectorDelta: (file, chunks, hashes, epoch, ctx) => this.applyVectorDelta(file, chunks, hashes, epoch, ctx),
403
421
  applyBinaryHNSWDelta: (file, ops, epoch, ctx) => this.applyBinaryHNSWDelta(file, ops, epoch, ctx),
404
- applyLIDelta: (file, ops, epoch) => this.applyLIDelta(file, ops, epoch),
422
+ applyLIDelta: (file, ops, epoch, ctx) => this.applyLIDelta(file, ops, ctx),
405
423
  applySparseGramDelta: (file, ops, epoch) => this.applySparseGramDelta(file, ops, epoch),
406
424
  readMaintenanceState: () => this.readMaintenanceState(),
407
425
  scheduleMaintenance: (job) => enqueueMaintenanceJob(this.stateDir, job),
@@ -697,7 +715,7 @@ class ProductionReconcileAdapter {
697
715
  // dropped if it was never indexed, and retired if it was (so the index
698
716
  // converges to a fresh full rebuild). Existence + shape + size are sync;
699
717
  // gitignore is ONE batched check over the admissible candidates.
700
- const merkle = readJson(path.join(this.stateDir, MERKLE_STATE), { files: {} }).files || {};
718
+ const merkle = (this._merkleRead() ?? { files: {} }).files || {};
701
719
  const info = rels.map((rel) => {
702
720
  const abs = path.join(this.projectRoot, rel);
703
721
  const exists = fs.existsSync(abs);
@@ -736,7 +754,7 @@ class ProductionReconcileAdapter {
736
754
  async hashFile(file) {
737
755
  const rel = typeof file === 'string' ? file : file.path;
738
756
  const abs = path.join(this.projectRoot, rel);
739
- const merkle = readJson(path.join(this.stateDir, MERKLE_STATE), { files: {} });
757
+ const merkle = this._merkleRead() ?? { files: {} };
740
758
  // Deleted on disk, or flagged inadmissible by readDirtySet (a previously
741
759
  // indexed file that became excluded/oversized/gitignored): retire it so all
742
760
  // tiers tombstone and the merkle entry is dropped.
@@ -784,7 +802,7 @@ class ProductionReconcileAdapter {
784
802
  ownConn = true;
785
803
  }
786
804
  try {
787
- const oldRows = db.prepare('SELECT rowid, id, logical_entity_id, signature_hash FROM entities WHERE file_path = ? AND epoch_retired IS NULL').all(rel);
805
+ const oldRows = prepareCached(db, 'SELECT rowid, id, logical_entity_id, signature_hash FROM entities WHERE file_path = ? AND epoch_retired IS NULL').all(rel);
788
806
  const oldByLogical = new Map(oldRows.map((r) => [r.logical_entity_id || r.id, r]));
789
807
  const oldIds = oldRows.map((r) => r.id);
790
808
  const extractor = new GraphExtractor();
@@ -811,7 +829,7 @@ class ProductionReconcileAdapter {
811
829
  let tombstone = 0;
812
830
  const liveIdFor = new Map();
813
831
  const tx = db.transaction(() => {
814
- const retireEntity = db.prepare('UPDATE entities SET epoch_retired = ?, stale_since = COALESCE(stale_since, ?) WHERE id = ? AND epoch_retired IS NULL');
832
+ const retireEntity = prepareCached(db, 'UPDATE entities SET epoch_retired = ?, stale_since = COALESCE(stale_since, ?) WHERE id = ? AND epoch_retired IS NULL');
815
833
  const retireRel = oldIds.length > 0
816
834
  ? db.prepare(`UPDATE relationships SET epoch_retired = ? WHERE source_id IN (${oldIds.map(() => '?').join(',')}) AND epoch_retired IS NULL`)
817
835
  : null;
@@ -913,7 +931,7 @@ class ProductionReconcileAdapter {
913
931
  // tier write. The merkle still records the new content hash so the
914
932
  // file is not re-queued forever.
915
933
  const prevTouched = this.touched.get(rel) || {};
916
- const prevChunkIds = readJson(path.join(this.stateDir, MERKLE_STATE), { files: {} }).files?.[rel]?.chunkIds || prevTouched.chunkIds || [];
934
+ const prevChunkIds = (this._merkleRead() ?? { files: {} }).files?.[rel]?.chunkIds || prevTouched.chunkIds || [];
917
935
  this.touched.set(rel, { ...prevTouched, hash: hashes, chunkIds: prevChunkIds, content: hashes.content });
918
936
  if (ctx) ctx.persistedFiles.add(rel);
919
937
  return {
@@ -1139,7 +1157,7 @@ class ProductionReconcileAdapter {
1139
1157
  return { ops: { binary_hnsw_append: append, binary_hnsw_tombstone: tombstone }, manifest: { path: 'codebase-binary-hnsw.idx' } };
1140
1158
  }
1141
1159
 
1142
- async applyLIDelta(file, ops) {
1160
+ async applyLIDelta(file, ops, ctx = null) {
1143
1161
  if (!Array.isArray(ops) || ops.length === 0) return { ops: { li_segment_append: 0, li_tombstone: 0 } };
1144
1162
  // LI generated-content parity: full indexing's buildLateInteractionIndex runs
1145
1163
  // applyIndexingChunkPolicy so @generated / config-excluded files never reach
@@ -1152,12 +1170,21 @@ class ProductionReconcileAdapter {
1152
1170
  ? ops.filter((op) => !(op.addId && op.chunk))
1153
1171
  : ops;
1154
1172
  const { applyLateInteractionDelta } = await import('./production-li-delta.mjs');
1173
+ // E.1-LI: tick-scoped READER cache. The per-file delta previously paid a
1174
+ // full loadExisting init (segment manifest + doc positions over a multi-
1175
+ // hundred-MB index) once per file; within one tick the reader view it
1176
+ // needs is stable (per-file ops only reference the file's own pre-tick
1177
+ // docs, appendGrowingSegment re-reads the manifest from disk, and segment
1178
+ // compaction only runs in maintenance after the tick). Null ctx (per-file
1179
+ // path) ⇒ fresh load per file, exactly as before.
1180
+ const readerCache = ctx ? (ctx._liReaderCache ??= { key: null, index: null }) : null;
1155
1181
  const { appended, tombstone } = await applyLateInteractionDelta({
1156
1182
  indexPath: path.join(this.stateDir, 'codebase-late-interaction.db'),
1157
1183
  ops: filteredOps,
1158
1184
  liEncoder: this.liEncoder,
1159
1185
  pickLiInput,
1160
1186
  onProgress: () => this.progress('production:li-delta'),
1187
+ readerCache,
1161
1188
  });
1162
1189
  return { ops: { li_segment_append: appended, li_tombstone: tombstone }, manifest: { path: 'codebase-late-interaction.db', segments: 'codebase-late-interaction.db.segments/manifest.json' } };
1163
1190
  }
@@ -1170,7 +1197,7 @@ class ProductionReconcileAdapter {
1170
1197
  * @returns {number}
1171
1198
  */
1172
1199
  _publishedEpoch() {
1173
- const merkle = readJson(path.join(this.stateDir, MERKLE_STATE), {});
1200
+ const merkle = this._merkleRead() ?? {};
1174
1201
  return Number.isInteger(merkle?.epoch) ? merkle.epoch : -1;
1175
1202
  }
1176
1203
 
@@ -1225,6 +1252,8 @@ class ProductionReconcileAdapter {
1225
1252
  merkle.epoch = manifest.epoch;
1226
1253
  merkle.stats = { ...(merkle.stats || {}), totalFiles: Object.keys(merkle.files).length };
1227
1254
  safeWriteJson(merklePath, merkle);
1255
+ // Invalidate the shared read cache — this is the one merkle writer.
1256
+ this._merkleCache = undefined;
1228
1257
  // E.6: persist the updated chunk-cutoff cache once per tick (after the
1229
1258
  // merkle advances). Best-effort; a failure only costs a redundant re-embed.
1230
1259
  if (this._cutoffCache && this._cutoffDirty) {
@@ -461,7 +461,7 @@ export class Reconciler {
461
461
  if (bin?.ops?.binary_hnsw_append != null) ops.binary_hnsw_append = bin.ops.binary_hnsw_append;
462
462
  if (bin?.ops?.binary_hnsw_tombstone != null) ops.binary_hnsw_tombstone = bin.ops.binary_hnsw_tombstone;
463
463
  this.progress('reconciler:li:start');
464
- const li = await this.adapters.applyLIDelta?.(file, vec?.tokenOps ?? [], epoch);
464
+ const li = await this.adapters.applyLIDelta?.(file, vec?.tokenOps ?? [], epoch, tickCtx);
465
465
  this.progress('reconciler:li:done');
466
466
  collectManifestTier(manifestTiers, 'lateInteraction', li);
467
467
  if (li?.ops?.li_segment_append != null) ops.li_segment_append = li.ops.li_segment_append;
@@ -192,13 +192,17 @@ function firstSafeRelativePath(...candidates) {
192
192
  * carried inside the encoder-input dependency registry; see
193
193
  * `core/incremental-indexing/domain/encoder-deps.mjs`).
194
194
  *
195
+ * Pass `{ includeDedup: false }` when the caller never persists the dedup
196
+ * fingerprint (the full-indexing insert path) to skip that hash entirely.
197
+ *
195
198
  * @param {object} chunk
196
- * @returns {{chunk_text_hash:string, embedding_input_hash:string, li_input_hash:string, metadata_fingerprint:string, dedup_fingerprint:string}}
199
+ * @param {{includeDedup?: boolean}} [opts]
200
+ * @returns {{chunk_text_hash:string, embedding_input_hash:string, li_input_hash:string, metadata_fingerprint:string, dedup_fingerprint?:string}}
197
201
  */
198
- export function chunkInputHashes(chunk) {
202
+ export function chunkInputHashes(chunk, { includeDedup = true } = {}) {
199
203
  const text = chunk?.content || chunk?.text || '';
200
204
  const meta = chunk?.metadata || {};
201
- return {
205
+ const hashes = {
202
206
  chunk_text_hash: contentHashSync(text),
203
207
  embedding_input_hash: denseInputHash(chunk),
204
208
  li_input_hash: liInputHash(chunk),
@@ -214,8 +218,9 @@ export function chunkInputHashes(chunk) {
214
218
  policy_embed: EMBED_TEXT_POLICY_VERSION,
215
219
  policy_li: LI_INPUT_POLICY_VERSION,
216
220
  })),
217
- dedup_fingerprint: dedupFingerprint(chunk),
218
221
  };
222
+ if (includeDedup) hashes.dedup_fingerprint = dedupFingerprint(chunk);
223
+ return hashes;
219
224
  }
220
225
 
221
226
  /**
@@ -52,7 +52,7 @@ export const ARTIFACT_THRESHOLDS = {
52
52
  stateFile: '.sweet-search/artifact-rebuild-state.json',
53
53
  };
54
54
 
55
- import { BinaryHNSWIndex } from '../vector-store/binary-hnsw-index.js';
55
+ import { BinaryHNSWIndex, int8SidecarCount } from '../vector-store/binary-hnsw-index.js';
56
56
  import { truncateForHNSW, fisherYatesShuffle, normalizedFloatToInt8, floatToBinary } from '../infrastructure/quantization.js';
57
57
  import { FloatVectorStore, getFloatStorePath } from '../vector-store/float-vector-store.js';
58
58
 
@@ -828,8 +828,10 @@ export async function getArtifactStats() {
828
828
  if (existsSync(int8SidecarPath)) {
829
829
  try {
830
830
  const fileStats = await fs.stat(int8SidecarPath);
831
- const int8Data = JSON.parse(await fs.readFile(int8SidecarPath, 'utf-8'));
832
- const count = Object.keys(int8Data).length;
831
+ // Header-only count (NDJSON v2 sidecar; v1 back-compat) — never
832
+ // JSON.parse the whole file, which exceeds V8's string ceiling on
833
+ // large indexes.
834
+ const count = await int8SidecarCount(int8SidecarPath);
833
835
 
834
836
  stats.int8Sidecar = {
835
837
  exists: true,
@@ -151,12 +151,12 @@ function buildEmbeddingText({ variant: variantOverride, content, relativePath, l
151
151
  const pathLine = relativePath ? `# ${relativePath}` : null;
152
152
  const parentLine = hierarchyInfo?.parentSymbol
153
153
  ? `# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}` : null;
154
- // Ruby method chunks keep metadata anchors, but omit method-name header
155
- // text so top-level Ruby snippets stay aligned with the pre-method-boundary
156
- // embedding surface.
157
- const isRubyMethodChunk = language === 'ruby'
158
- && (chunkType === 'method' || chunkType === 'singleton_method');
159
- const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
154
+ // Ruby method chunks keep metadata anchors, but omit method-name header
155
+ // text so top-level Ruby snippets stay aligned with the pre-method-boundary
156
+ // embedding surface.
157
+ const isRubyMethodChunk = language === 'ruby'
158
+ && (chunkType === 'method' || chunkType === 'singleton_method');
159
+ const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
160
160
  ? `# ${chunkType}: ${symbol}` : null;
161
161
  const langLine = (language && language !== 'text')
162
162
  ? `# Language: ${language}` : null;
@@ -293,10 +293,10 @@ function buildLiText({ content, relativePath, language, chunkType, symbol, hiera
293
293
  if (hierarchyInfo?.parentSymbol) {
294
294
  lines.push(`# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}`);
295
295
  }
296
- // Mirror the Ruby method header carve-out used by embedding_text.
297
- const isRubyMethodChunk = language === 'ruby'
298
- && (chunkType === 'method' || chunkType === 'singleton_method');
299
- if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
296
+ // Mirror the Ruby method header carve-out used by embedding_text.
297
+ const isRubyMethodChunk = language === 'ruby'
298
+ && (chunkType === 'method' || chunkType === 'singleton_method');
299
+ if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
300
300
  lines.push(`# ${chunkType}: ${symbol}`);
301
301
  }
302
302
  // Mirror embedding_text: surface sibling symbol names so the LI MaxSim
@@ -391,27 +391,27 @@ export class ASTChunker {
391
391
  const tsChunks = await provider.parseFileToChunks(content, langInfo.id);
392
392
  if (!tsChunks || tsChunks.length === 0) return null;
393
393
 
394
- return tsChunks.map(chunk => {
395
- const isTopLevelRubyMethod = langInfo.id === 'ruby'
396
- && (chunk.type === 'method' || chunk.type === 'singleton_method')
397
- && !chunk.parentSymbol;
398
-
399
- return this.buildChunk(
400
- chunk.text, filePath, langInfo.id,
401
- isTopLevelRubyMethod ? 'code' : chunk.type,
402
- isTopLevelRubyMethod ? null : chunk.name,
403
- chunk.startLine, chunk.endLine,
404
- {
405
- chunkId: chunk.chunkId,
406
- parentChunkId: chunk.parentChunkId,
407
- parentSymbol: chunk.parentSymbol,
408
- parentType: chunk.parentType,
409
- signature: chunk.signature || null,
410
- additionalSymbols: chunk.additionalSymbols || null,
411
- }
412
- );
413
- });
414
- }
394
+ return tsChunks.map(chunk => {
395
+ const isTopLevelRubyMethod = langInfo.id === 'ruby'
396
+ && (chunk.type === 'method' || chunk.type === 'singleton_method')
397
+ && !chunk.parentSymbol;
398
+
399
+ return this.buildChunk(
400
+ chunk.text, filePath, langInfo.id,
401
+ isTopLevelRubyMethod ? 'code' : chunk.type,
402
+ isTopLevelRubyMethod ? null : chunk.name,
403
+ chunk.startLine, chunk.endLine,
404
+ {
405
+ chunkId: chunk.chunkId,
406
+ parentChunkId: chunk.parentChunkId,
407
+ parentSymbol: chunk.parentSymbol,
408
+ parentType: chunk.parentType,
409
+ signature: chunk.signature || null,
410
+ additionalSymbols: chunk.additionalSymbols || null,
411
+ }
412
+ );
413
+ });
414
+ }
415
415
 
416
416
  parseBraceBasedFile(filePath, content, language, patterns, comment, multiLine) {
417
417
  const chunks = [];
@@ -942,6 +942,7 @@ export class ASTChunker {
942
942
  // Production embedding text. With the default variant (current) this
943
943
  // is byte-identical to shipped v6.2; under a research-only variant
944
944
  // (SWEET_SEARCH_EMBED_TEXT_VARIANT) it produces the experimental form.
945
+ const activeVariant = getEmbedTextVariant();
945
946
  const embedding_text = buildEmbeddingText({
946
947
  content,
947
948
  relativePath,
@@ -954,17 +955,19 @@ export class ASTChunker {
954
955
  // Research isolation: always build the shipped (current-variant)
955
956
  // form alongside, so the LI stage can read a canonical input even
956
957
  // when an R1 experiment is active on embedding_text. In production
957
- // (variant=current) this is the same content as embedding_text and
958
- // pickLiInput's preference for li_greedy_text changes nothing.
959
- const li_greedy_text = buildEmbeddingText({
960
- variant: 'current',
961
- content,
962
- relativePath,
963
- language,
964
- chunkType,
965
- symbol,
966
- hierarchyInfo,
967
- });
958
+ // (variant=current) it IS the embedding_text reuse the string
959
+ // instead of rebuilding it.
960
+ const li_greedy_text = activeVariant === 'current'
961
+ ? embedding_text
962
+ : buildEmbeddingText({
963
+ variant: 'current',
964
+ content,
965
+ relativePath,
966
+ language,
967
+ chunkType,
968
+ symbol,
969
+ hierarchyInfo,
970
+ });
968
971
 
969
972
  // Build LI text via the v6.2 builder (language-conditioned path
970
973
  // policy). See buildLiText() docstring for rationale.
@@ -1034,26 +1037,26 @@ export class ASTChunker {
1034
1037
  const parts = [];
1035
1038
  parts.push(`# ${chunk.metadata.path}`);
1036
1039
 
1037
- const isRubyMethodChunk = chunk.metadata.language === 'ruby'
1038
- && (chunk.metadata.chunk_type === 'method'
1039
- || chunk.metadata.chunk_type === 'singleton_method');
1040
- const hasOnlySelfScope = scopeChain
1041
- && scopeChain.length === 1
1042
- && scopeChain[0] === chunk.metadata.symbol
1043
- && !chunk.metadata.parent_symbol;
1044
-
1045
- if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
1046
- parts.push(`# Scope: ${scopeChain.join(' > ')}`);
1047
- } else if (chunk.metadata.parent_symbol) {
1048
- // Preserve cAST parent context when no scope chain from code graph
1049
- parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
1050
- }
1051
- // Keep Ruby method metadata, but avoid injecting the method name into
1052
- // the production embedding/LI text. Top-level Ruby method chunks also
1053
- // skip self-only scope above.
1054
- if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
1055
- parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
1056
- }
1040
+ const isRubyMethodChunk = chunk.metadata.language === 'ruby'
1041
+ && (chunk.metadata.chunk_type === 'method'
1042
+ || chunk.metadata.chunk_type === 'singleton_method');
1043
+ const hasOnlySelfScope = scopeChain
1044
+ && scopeChain.length === 1
1045
+ && scopeChain[0] === chunk.metadata.symbol
1046
+ && !chunk.metadata.parent_symbol;
1047
+
1048
+ if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
1049
+ parts.push(`# Scope: ${scopeChain.join(' > ')}`);
1050
+ } else if (chunk.metadata.parent_symbol) {
1051
+ // Preserve cAST parent context when no scope chain from code graph
1052
+ parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
1053
+ }
1054
+ // Keep Ruby method metadata, but avoid injecting the method name into
1055
+ // the production embedding/LI text. Top-level Ruby method chunks also
1056
+ // skip self-only scope above.
1057
+ if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
1058
+ parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
1059
+ }
1057
1060
 
1058
1061
  if (chunk.metadata.additional_symbols && chunk.metadata.additional_symbols.length > 0) {
1059
1062
  parts.push(`# Additional: ${chunk.metadata.additional_symbols.join(', ')}`);
@@ -1267,12 +1267,14 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
1267
1267
  // Best-effort — never block startup on the embedding module.
1268
1268
  if (process.env.SWEET_SEARCH_ORT_BACKGROUND !== '0') {
1269
1269
  try {
1270
- const [{ configureLocalModelRuntime }, { backgroundIntraOpThreads }] = await Promise.all([
1271
- import('../embedding/embedding-local-model.js'),
1272
- import('../infrastructure/onnx-session-utils.js'),
1273
- ]);
1274
- configureLocalModelRuntime({ intraOpThreads: backgroundIntraOpThreads(), background: true });
1275
- log('INFO', 'ORT background profile armed for maintainer daemon (force_spinning_stop + arena-off + bg threads)');
1270
+ // Arms BOTH resident CPU sessions (dense + late-interaction) — the LI
1271
+ // session previously stayed on the foreground profile (arena ON +
1272
+ // allow_spinning) inside the daemon; measured consequence: 354 × 128MB
1273
+ // arena extensions (~34GB RSS) after one edit-heavy day plus ~40% idle
1274
+ // CPU from spinning workers.
1275
+ const { armBackgroundOrtProfiles } = await import('./model-pool.js');
1276
+ armBackgroundOrtProfiles();
1277
+ log('INFO', 'ORT background profile armed for maintainer daemon (dense + LI: force_spinning_stop + arena-off + bg threads)');
1276
1278
  } catch (err) {
1277
1279
  log('WARN', `ORT background profile arming failed (continuing on foreground profile): ${err?.message ?? err}`);
1278
1280
  }
@@ -1314,15 +1316,30 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
1314
1316
  // system-RAM tier auto-enables a cap (small-RAM hosts). Best-effort, guarded
1315
1317
  // so a missing module is a no-op.
1316
1318
  let rssRegistration = null;
1319
+ // D.5 per-process recycle ceiling: fleet-budget eviction only sheds the
1320
+ // longest-IDLE daemon, so an active maintainer needs its own line. Resolved
1321
+ // once; checked at tick boundaries below. 0 ⇒ disabled.
1322
+ let rssCeiling = { ceilingBytes: 0, minUptimeMs: 0, shouldRecycleForRss: null };
1317
1323
  try {
1318
1324
  const mod = await import('./rss-budget.mjs');
1319
1325
  if (typeof mod.isEnabled === 'function' && mod.isEnabled()
1320
1326
  && typeof mod.registerDaemon === 'function') {
1321
1327
  rssRegistration = await mod.registerDaemon({ pid: process.pid, stateDir: ctx.stateDir, kind: 'maintainer' });
1322
1328
  }
1329
+ if (typeof mod.maintainerRssCeilingBytes === 'function') {
1330
+ rssCeiling = {
1331
+ ceilingBytes: mod.maintainerRssCeilingBytes(),
1332
+ minUptimeMs: mod.maintainerRssMinUptimeMs(),
1333
+ shouldRecycleForRss: mod.shouldRecycleForRss,
1334
+ };
1335
+ if (rssCeiling.ceilingBytes > 0) {
1336
+ log('INFO', `Maintainer RSS recycle ceiling armed: ${(rssCeiling.ceilingBytes / 1048576).toFixed(0)}MB (min uptime ${rssCeiling.minUptimeMs}ms; override SWEET_SEARCH_MAINTAINER_RSS_MAX_MB)`);
1337
+ }
1338
+ }
1323
1339
  } catch (err) {
1324
1340
  log('WARN', `RSS-budget registry unavailable (no soft cap on this daemon): ${err?.message ?? err}`);
1325
1341
  }
1342
+ let completedTicks = 0;
1326
1343
 
1327
1344
  // D.1 idle-TTL: an unattended maintainer self-shuts-down after the configured
1328
1345
  // wall-clock idle budget so N resident model-loaded daemons collapse to 1–2
@@ -1432,6 +1449,23 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
1432
1449
  maintenanceBacklog: backlog,
1433
1450
  skipped: tickCounters?.skipped === true,
1434
1451
  });
1452
+ // D.5: per-process RSS recycle check — tick boundary only (the tick
1453
+ // above has published; a recycle here can never tear an artifact).
1454
+ completedTicks += 1;
1455
+ if (rssCeiling.ceilingBytes > 0 && typeof rssCeiling.shouldRecycleForRss === 'function') {
1456
+ const rssNow = process.memoryUsage.rss();
1457
+ const verdict = rssCeiling.shouldRecycleForRss({
1458
+ rssBytes: rssNow,
1459
+ ceilingBytes: rssCeiling.ceilingBytes,
1460
+ uptimeMs: process.uptime() * 1000,
1461
+ minUptimeMs: rssCeiling.minUptimeMs,
1462
+ ticksCompleted: completedTicks,
1463
+ });
1464
+ if (verdict.recycle) {
1465
+ log('WARN', `Maintainer RSS ${(rssNow / 1048576).toFixed(0)}MB exceeds recycle ceiling ${(rssCeiling.ceilingBytes / 1048576).toFixed(0)}MB; requesting clean shutdown for on-demand respawn. If this repeats right after startup, raise SWEET_SEARCH_MAINTAINER_RSS_MAX_MB.`);
1466
+ shutdownRequested = true;
1467
+ }
1468
+ }
1435
1469
  } catch (err) {
1436
1470
  if (err instanceof MaintainerLifecycleAbort) {
1437
1471
  log('WARN', `Reconcile v2 lifecycle abort: ${err.message}. Cleaning up cancellation-orphaned temps and exiting cleanly.`);
@@ -1457,6 +1491,10 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
1457
1491
  log('ERROR', `Reconcile v2 tick failed: ${err?.message ?? err}`);
1458
1492
  }
1459
1493
  }
1494
+ // Skip the sleep when a shutdown was requested during this iteration
1495
+ // (idle-TTL, signal, or D.5 RSS recycle) — exit promptly instead of
1496
+ // waiting out one more interval.
1497
+ if (shutdownRequested) break;
1460
1498
  await sleepWithProgress(intervalMs, lock.lockFile, {
1461
1499
  // G6 early-wake: break the sleep the instant the watcher reports new
1462
1500
  // events so a fresh edit is reconciled without waiting out the interval.
@@ -2482,6 +2520,11 @@ async function main() {
2482
2520
  const dryRun = process.argv.includes('--dry-run');
2483
2521
  const merkleOnce = process.argv.includes('--merkle-once');
2484
2522
 
2523
+ // Name the resident daemon so ps/Activity Monitor shows what it is instead
2524
+ // of an anonymous `node`. Cosmetic-only: all liveness/takeover checks are
2525
+ // lock/pid-based, never command-line matching.
2526
+ try { process.title = 'sweet-search-maintainer'; } catch { /* best-effort */ }
2527
+
2485
2528
  // A.1 (Tier-1, UNGATED): demote the maintainer daemon to low OS priority so
2486
2529
  // the foreground (editor / git / shell) never feels the background indexer's
2487
2530
  // CPU. Identical index output — only *when* CPU is granted changes. Covers