sweet-search 2.6.8 → 2.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/core/cli.js +7 -0
- package/core/graph/graph-extractor.js +24 -14
- package/core/graph/hcgs-generator.js +4 -3
- package/core/incremental-indexing/application/production-li-delta.mjs +28 -3
- package/core/incremental-indexing/application/production-reconciler.mjs +40 -11
- package/core/incremental-indexing/application/reconciler.mjs +1 -1
- package/core/incremental-indexing/domain/encoder-input.mjs +9 -4
- package/core/indexing/artifact-builder.js +5 -3
- package/core/indexing/ast-chunker.js +65 -62
- package/core/indexing/index-maintainer.mjs +49 -6
- package/core/indexing/indexer-build.js +18 -4
- package/core/indexing/indexer-phases.js +45 -24
- package/core/indexing/indexer-utils.js +96 -6
- package/core/indexing/indexing-file-policy.js +30 -7
- package/core/indexing/model-pool.js +26 -0
- package/core/indexing/rss-budget.mjs +65 -0
- package/core/infrastructure/code-graph-repository.js +18 -18
- package/core/infrastructure/db-utils.js +29 -0
- package/core/infrastructure/simd-distance.js +35 -21
- package/core/infrastructure/tombstone-bitmap-reader.js +3 -1
- package/core/ranking/file-kind-ranking.js +14 -2
- package/core/ranking/late-interaction-index.js +86 -15
- package/core/ranking/late-interaction-model.js +25 -4
- package/core/ranking/mmr.js +15 -10
- package/core/search/search-anchor.js +21 -3
- package/core/search/search-boost.js +25 -13
- package/core/search/search-fusion.js +13 -5
- package/core/search/search-read-semantic.js +13 -12
- package/core/search/search-server.js +2 -1
- package/core/vector-store/binary-hnsw-index.js +338 -33
- package/core/vector-store/float-vector-store.js +9 -5
- package/eval/agent-read-workflows/bin/_ss-helpers.mjs +11 -0
- package/package.json +8 -8
package/README.md
CHANGED
|
@@ -711,6 +711,21 @@ without you ever running a command.
|
|
|
711
711
|
- **Resource-polite:** ticks are budgeted (≤50 files / ≤2 s CPU per tick), run CPU-only (the GPU is reserved for cold full indexing), and the interval auto-tunes from load average, churn, and backlog.
|
|
712
712
|
- `sweet-search reconcile status` / `reconcile inspect <path>` explain exactly what the daemon thinks and why. Opt out any time with `SWEET_SEARCH_RECONCILE_V2=0`.
|
|
713
713
|
|
|
714
|
+
**Memory controls.** The resident daemons show up in `ps` / Activity Monitor as
|
|
715
|
+
`sweet-search-maintainer` and `sweet-search-daemon`. A maintainer's steady state is
|
|
716
|
+
roughly 2–3 GB (embedding + late-interaction models stay loaded so ticks are fast),
|
|
717
|
+
and four independent mechanisms keep that bounded:
|
|
718
|
+
|
|
719
|
+
| Mechanism | Default | Override |
|
|
720
|
+
|-----------|---------|----------|
|
|
721
|
+
| Background ORT profile (arena-off + parked threads) in the maintainer | on | `SWEET_SEARCH_ORT_BACKGROUND=0` |
|
|
722
|
+
| Per-process recycle ceiling — the maintainer finishes its tick, exits cleanly, and respawns fresh on the next edit when its RSS crosses the line | clamp(25 % of RAM, 4 GiB, 8 GiB) | `SWEET_SEARCH_MAINTAINER_RSS_MAX_MB` (0 disables) |
|
|
723
|
+
| Idle TTL — unattended daemons shut down and respawn on demand | tier-aware | `SWEET_SEARCH_MAINTAINER_IDLE_TTL_MS` / `SWEET_SEARCH_DAEMON_IDLE_TTL_MS` |
|
|
724
|
+
| Fleet RSS budget — across all repos' daemons, the longest-idle one is evicted when the sum crosses a RAM-scaled budget | tier-aware | `SWEET_SEARCH_RSS_BUDGET_FRACTION` |
|
|
725
|
+
|
|
726
|
+
A recycle or eviction never touches index state: every tick publishes atomically
|
|
727
|
+
before the process exits, and the next edit (or query) respawns a fresh daemon.
|
|
728
|
+
|
|
714
729
|
</details>
|
|
715
730
|
|
|
716
731
|
## 🦀 The Native Engine Room
|
package/core/cli.js
CHANGED
|
@@ -94,6 +94,13 @@ if (args[0] === 'init') {
|
|
|
94
94
|
await runIndexer();
|
|
95
95
|
}
|
|
96
96
|
} else if (args[0] === '--serve' || args[0] === '--stop') {
|
|
97
|
+
// Name the resident daemon so ps/Activity Monitor shows what it is instead
|
|
98
|
+
// of an anonymous `node`. Entrypoint-scoped: library consumers of the
|
|
99
|
+
// search modules are never retitled. Liveness checks are lock/pid-based,
|
|
100
|
+
// never command-line matching, so this is cosmetic-only.
|
|
101
|
+
if (args[0] === '--serve') {
|
|
102
|
+
try { process.title = 'sweet-search-daemon'; } catch { /* best-effort */ }
|
|
103
|
+
}
|
|
97
104
|
// Warm search server lifecycle is implemented in JS.
|
|
98
105
|
const { runCli } = await import('./search/index.js');
|
|
99
106
|
await runCli(args);
|
|
@@ -2351,7 +2351,28 @@ function resolveThrows(exceptionName, entityByName) {
|
|
|
2351
2351
|
* Insert entities and relationships into database
|
|
2352
2352
|
* Uses better-sqlite3 (sync API, no .free() needed)
|
|
2353
2353
|
*/
|
|
2354
|
-
|
|
2354
|
+
/**
|
|
2355
|
+
* Rebuild + optimize the external-content FTS5 mirrors from the entities
|
|
2356
|
+
* table. 'rebuild' reconstructs the whole FTS index from current content, so
|
|
2357
|
+
* only the LAST rebuild before any FTS read matters — batch loops should
|
|
2358
|
+
* pass { syncFts: false } to insertGraph and call this once at the end.
|
|
2359
|
+
*/
|
|
2360
|
+
export function rebuildGraphFts(db) {
|
|
2361
|
+
try {
|
|
2362
|
+
db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('rebuild')`);
|
|
2363
|
+
db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('rebuild')`);
|
|
2364
|
+
console.log(' FTS5 indexes rebuilt (porter + trigram)');
|
|
2365
|
+
|
|
2366
|
+
// Best-effort post-build compaction for faster reads.
|
|
2367
|
+
db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('optimize')`);
|
|
2368
|
+
db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('optimize')`);
|
|
2369
|
+
console.log(' FTS5 indexes optimized (segments merged)');
|
|
2370
|
+
} catch (err) {
|
|
2371
|
+
// FTS5 rebuild/optimize failed, ignore
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
|
|
2375
|
+
export function insertGraph(db, entities, relationships, hasFts5 = false, { syncFts = true } = {}) {
|
|
2355
2376
|
// Insert entities with HCGS hierarchy support
|
|
2356
2377
|
// Includes signature_hash for collision-proof backup/restore
|
|
2357
2378
|
const entityStmt = db.prepare(`
|
|
@@ -2481,19 +2502,8 @@ export function insertGraph(db, entities, relationships, hasFts5 = false) {
|
|
|
2481
2502
|
// }
|
|
2482
2503
|
|
|
2483
2504
|
// Rebuild FTS indexes if available
|
|
2484
|
-
if (hasFts5) {
|
|
2485
|
-
|
|
2486
|
-
db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('rebuild')`);
|
|
2487
|
-
db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('rebuild')`);
|
|
2488
|
-
console.log(' FTS5 indexes rebuilt (porter + trigram)');
|
|
2489
|
-
|
|
2490
|
-
// Best-effort post-build compaction for faster reads.
|
|
2491
|
-
db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('optimize')`);
|
|
2492
|
-
db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('optimize')`);
|
|
2493
|
-
console.log(' FTS5 indexes optimized (segments merged)');
|
|
2494
|
-
} catch (err) {
|
|
2495
|
-
// FTS5 rebuild/optimize failed, ignore
|
|
2496
|
-
}
|
|
2505
|
+
if (hasFts5 && syncFts) {
|
|
2506
|
+
rebuildGraphFts(db);
|
|
2497
2507
|
}
|
|
2498
2508
|
}
|
|
2499
2509
|
|
|
@@ -428,9 +428,10 @@ async function generateSummariesForEntities(entityIds, options = {}) {
|
|
|
428
428
|
const dbWrite = new Database(dbPath);
|
|
429
429
|
const stmt = dbWrite.prepare(`UPDATE entities SET summary = NULL WHERE id = ?`);
|
|
430
430
|
|
|
431
|
-
|
|
432
|
-
stmt.run(entity.id);
|
|
433
|
-
}
|
|
431
|
+
const clearSummaries = dbWrite.transaction((rows) => {
|
|
432
|
+
for (const entity of rows) stmt.run(entity.id);
|
|
433
|
+
});
|
|
434
|
+
clearSummaries(entities);
|
|
434
435
|
|
|
435
436
|
dbWrite.close();
|
|
436
437
|
|
|
@@ -117,6 +117,7 @@ export async function applyLateInteractionDelta({
|
|
|
117
117
|
liEncoder,
|
|
118
118
|
pickLiInput,
|
|
119
119
|
onProgress = null,
|
|
120
|
+
readerCache = null,
|
|
120
121
|
}) {
|
|
121
122
|
const progress = typeof onProgress === 'function'
|
|
122
123
|
? (phase) => { onProgress(phase); }
|
|
@@ -128,11 +129,35 @@ export async function applyLateInteractionDelta({
|
|
|
128
129
|
const encode = liEncoder || ((texts) => encodeDocumentsCpu(texts));
|
|
129
130
|
const existing = fs.existsSync(indexPath);
|
|
130
131
|
const segmented = existing ? segmentedState(indexPath) : null;
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
|
|
133
|
+
// E.1-LI reader cache: reuse the loaded read view across the tick's files.
|
|
134
|
+
// Only the SEGMENTED path is cacheable — everything the reader serves there
|
|
135
|
+
// (config fields, positions/counts of pre-tick docs) is immutable within a
|
|
136
|
+
// tick: per-file ops only reference the file's own pre-tick docs, appends
|
|
137
|
+
// go through appendGrowingSegment (which re-reads the manifest from disk),
|
|
138
|
+
// and tombstone sidecar state is opened fresh per call. The legacy path
|
|
139
|
+
// MUTATES the loaded index (rewriteLegacyIndex), so it always loads fresh
|
|
140
|
+
// and drops any cached reader.
|
|
141
|
+
const cacheable = !!(readerCache && segmented);
|
|
142
|
+
let index;
|
|
143
|
+
if (cacheable && readerCache.index && readerCache.key === indexPath) {
|
|
144
|
+
index = readerCache.index;
|
|
145
|
+
progress('li:init-cached');
|
|
146
|
+
} else {
|
|
147
|
+
index = new LateInteractionIndex({ indexPath, loadExisting: true });
|
|
148
|
+
await index.init();
|
|
149
|
+
progress('li:init');
|
|
150
|
+
if (cacheable) {
|
|
151
|
+
readerCache.key = indexPath;
|
|
152
|
+
readerCache.index = index;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
134
155
|
|
|
135
156
|
if (existing && !segmented) {
|
|
157
|
+
if (readerCache) {
|
|
158
|
+
readerCache.key = null;
|
|
159
|
+
readerCache.index = null;
|
|
160
|
+
}
|
|
136
161
|
return rewriteLegacyIndex(index, ops, encode, pickLiInput, progress);
|
|
137
162
|
}
|
|
138
163
|
|
|
@@ -34,6 +34,7 @@ import { floatToBinary, normalizedFloatToInt8, truncateForHNSW } from '../../inf
|
|
|
34
34
|
import { extractSparseGramDeltaRecord } from '../../infrastructure/native-sparse-gram.js';
|
|
35
35
|
import { migrateEntitiesSchema, migrateRelationshipsSchema } from '../infrastructure/schema-migrations.mjs';
|
|
36
36
|
import { readMaintenanceState as readMaintenanceStateFromArtifacts } from '../infrastructure/maintenance-state-reader.mjs';
|
|
37
|
+
import { prepareCached } from '../../infrastructure/db-utils.js';
|
|
37
38
|
|
|
38
39
|
const DIRTY_QUEUE = 'index-maintainer-queue.jsonl';
|
|
39
40
|
const PROCESSING_QUEUE = 'index-maintainer-queue.processing.jsonl';
|
|
@@ -270,9 +271,9 @@ async function enrichChunksFromGraph(chunks, stateDir, tickCtx = null) {
|
|
|
270
271
|
ownConn = true;
|
|
271
272
|
}
|
|
272
273
|
try {
|
|
273
|
-
const entityStmt = db
|
|
274
|
-
const fileEntityStmt = db
|
|
275
|
-
const importStmt = db
|
|
274
|
+
const entityStmt = prepareCached(db, 'SELECT type, name, start_line, end_line FROM entities WHERE file_path = ? AND epoch_retired IS NULL ORDER BY start_line ASC');
|
|
275
|
+
const fileEntityStmt = prepareCached(db, 'SELECT id FROM entities WHERE file_path = ? AND logical_entity_id = ? AND epoch_retired IS NULL ORDER BY epoch_written DESC LIMIT 1');
|
|
276
|
+
const importStmt = prepareCached(db, "SELECT DISTINCT target_name FROM relationships WHERE source_id = ? AND type IN ('imports', 'plainImport') AND epoch_retired IS NULL ORDER BY target_name");
|
|
276
277
|
for (const chunk of chunks) {
|
|
277
278
|
const file = chunk.file || chunk.metadata?.relative_path;
|
|
278
279
|
const symbol = chunk.metadata?.symbol;
|
|
@@ -376,6 +377,23 @@ class ProductionReconcileAdapter {
|
|
|
376
377
|
// when the flag is on; null when disabled.
|
|
377
378
|
this._cutoffCache = null;
|
|
378
379
|
this._cutoffDirty = false;
|
|
380
|
+
// merkle-state.json is written exactly once per tick (persistManifest);
|
|
381
|
+
// every other reader gets one shared parse. `undefined` = not read yet,
|
|
382
|
+
// `null` = missing/corrupt file.
|
|
383
|
+
this._merkleCache = undefined;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Shared read of merkle-state.json — parsed once per adapter lifetime and
|
|
388
|
+
* invalidated by persistManifest (the only writer). All consumers treat the
|
|
389
|
+
* object as read-only.
|
|
390
|
+
* @returns {object|null}
|
|
391
|
+
*/
|
|
392
|
+
_merkleRead() {
|
|
393
|
+
if (this._merkleCache === undefined) {
|
|
394
|
+
this._merkleCache = readJson(path.join(this.stateDir, MERKLE_STATE), null);
|
|
395
|
+
}
|
|
396
|
+
return this._merkleCache;
|
|
379
397
|
}
|
|
380
398
|
|
|
381
399
|
progress(phase) {
|
|
@@ -401,7 +419,7 @@ class ProductionReconcileAdapter {
|
|
|
401
419
|
applyGraphDelta: (file, hashes, epoch, ctx) => this.applyGraphDelta(file, hashes, epoch, ctx),
|
|
402
420
|
applyVectorDelta: (file, chunks, hashes, epoch, ctx) => this.applyVectorDelta(file, chunks, hashes, epoch, ctx),
|
|
403
421
|
applyBinaryHNSWDelta: (file, ops, epoch, ctx) => this.applyBinaryHNSWDelta(file, ops, epoch, ctx),
|
|
404
|
-
applyLIDelta: (file, ops, epoch) => this.applyLIDelta(file, ops,
|
|
422
|
+
applyLIDelta: (file, ops, epoch, ctx) => this.applyLIDelta(file, ops, ctx),
|
|
405
423
|
applySparseGramDelta: (file, ops, epoch) => this.applySparseGramDelta(file, ops, epoch),
|
|
406
424
|
readMaintenanceState: () => this.readMaintenanceState(),
|
|
407
425
|
scheduleMaintenance: (job) => enqueueMaintenanceJob(this.stateDir, job),
|
|
@@ -697,7 +715,7 @@ class ProductionReconcileAdapter {
|
|
|
697
715
|
// dropped if it was never indexed, and retired if it was (so the index
|
|
698
716
|
// converges to a fresh full rebuild). Existence + shape + size are sync;
|
|
699
717
|
// gitignore is ONE batched check over the admissible candidates.
|
|
700
|
-
const merkle =
|
|
718
|
+
const merkle = (this._merkleRead() ?? { files: {} }).files || {};
|
|
701
719
|
const info = rels.map((rel) => {
|
|
702
720
|
const abs = path.join(this.projectRoot, rel);
|
|
703
721
|
const exists = fs.existsSync(abs);
|
|
@@ -736,7 +754,7 @@ class ProductionReconcileAdapter {
|
|
|
736
754
|
async hashFile(file) {
|
|
737
755
|
const rel = typeof file === 'string' ? file : file.path;
|
|
738
756
|
const abs = path.join(this.projectRoot, rel);
|
|
739
|
-
const merkle =
|
|
757
|
+
const merkle = this._merkleRead() ?? { files: {} };
|
|
740
758
|
// Deleted on disk, or flagged inadmissible by readDirtySet (a previously
|
|
741
759
|
// indexed file that became excluded/oversized/gitignored): retire it so all
|
|
742
760
|
// tiers tombstone and the merkle entry is dropped.
|
|
@@ -784,7 +802,7 @@ class ProductionReconcileAdapter {
|
|
|
784
802
|
ownConn = true;
|
|
785
803
|
}
|
|
786
804
|
try {
|
|
787
|
-
const oldRows = db
|
|
805
|
+
const oldRows = prepareCached(db, 'SELECT rowid, id, logical_entity_id, signature_hash FROM entities WHERE file_path = ? AND epoch_retired IS NULL').all(rel);
|
|
788
806
|
const oldByLogical = new Map(oldRows.map((r) => [r.logical_entity_id || r.id, r]));
|
|
789
807
|
const oldIds = oldRows.map((r) => r.id);
|
|
790
808
|
const extractor = new GraphExtractor();
|
|
@@ -811,7 +829,7 @@ class ProductionReconcileAdapter {
|
|
|
811
829
|
let tombstone = 0;
|
|
812
830
|
const liveIdFor = new Map();
|
|
813
831
|
const tx = db.transaction(() => {
|
|
814
|
-
const retireEntity = db
|
|
832
|
+
const retireEntity = prepareCached(db, 'UPDATE entities SET epoch_retired = ?, stale_since = COALESCE(stale_since, ?) WHERE id = ? AND epoch_retired IS NULL');
|
|
815
833
|
const retireRel = oldIds.length > 0
|
|
816
834
|
? db.prepare(`UPDATE relationships SET epoch_retired = ? WHERE source_id IN (${oldIds.map(() => '?').join(',')}) AND epoch_retired IS NULL`)
|
|
817
835
|
: null;
|
|
@@ -913,7 +931,7 @@ class ProductionReconcileAdapter {
|
|
|
913
931
|
// tier write. The merkle still records the new content hash so the
|
|
914
932
|
// file is not re-queued forever.
|
|
915
933
|
const prevTouched = this.touched.get(rel) || {};
|
|
916
|
-
const prevChunkIds =
|
|
934
|
+
const prevChunkIds = (this._merkleRead() ?? { files: {} }).files?.[rel]?.chunkIds || prevTouched.chunkIds || [];
|
|
917
935
|
this.touched.set(rel, { ...prevTouched, hash: hashes, chunkIds: prevChunkIds, content: hashes.content });
|
|
918
936
|
if (ctx) ctx.persistedFiles.add(rel);
|
|
919
937
|
return {
|
|
@@ -1139,7 +1157,7 @@ class ProductionReconcileAdapter {
|
|
|
1139
1157
|
return { ops: { binary_hnsw_append: append, binary_hnsw_tombstone: tombstone }, manifest: { path: 'codebase-binary-hnsw.idx' } };
|
|
1140
1158
|
}
|
|
1141
1159
|
|
|
1142
|
-
async applyLIDelta(file, ops) {
|
|
1160
|
+
async applyLIDelta(file, ops, ctx = null) {
|
|
1143
1161
|
if (!Array.isArray(ops) || ops.length === 0) return { ops: { li_segment_append: 0, li_tombstone: 0 } };
|
|
1144
1162
|
// LI generated-content parity: full indexing's buildLateInteractionIndex runs
|
|
1145
1163
|
// applyIndexingChunkPolicy so @generated / config-excluded files never reach
|
|
@@ -1152,12 +1170,21 @@ class ProductionReconcileAdapter {
|
|
|
1152
1170
|
? ops.filter((op) => !(op.addId && op.chunk))
|
|
1153
1171
|
: ops;
|
|
1154
1172
|
const { applyLateInteractionDelta } = await import('./production-li-delta.mjs');
|
|
1173
|
+
// E.1-LI: tick-scoped READER cache. The per-file delta previously paid a
|
|
1174
|
+
// full loadExisting init (segment manifest + doc positions over a multi-
|
|
1175
|
+
// hundred-MB index) once per file; within one tick the reader view it
|
|
1176
|
+
// needs is stable (per-file ops only reference the file's own pre-tick
|
|
1177
|
+
// docs, appendGrowingSegment re-reads the manifest from disk, and segment
|
|
1178
|
+
// compaction only runs in maintenance after the tick). Null ctx (per-file
|
|
1179
|
+
// path) ⇒ fresh load per file, exactly as before.
|
|
1180
|
+
const readerCache = ctx ? (ctx._liReaderCache ??= { key: null, index: null }) : null;
|
|
1155
1181
|
const { appended, tombstone } = await applyLateInteractionDelta({
|
|
1156
1182
|
indexPath: path.join(this.stateDir, 'codebase-late-interaction.db'),
|
|
1157
1183
|
ops: filteredOps,
|
|
1158
1184
|
liEncoder: this.liEncoder,
|
|
1159
1185
|
pickLiInput,
|
|
1160
1186
|
onProgress: () => this.progress('production:li-delta'),
|
|
1187
|
+
readerCache,
|
|
1161
1188
|
});
|
|
1162
1189
|
return { ops: { li_segment_append: appended, li_tombstone: tombstone }, manifest: { path: 'codebase-late-interaction.db', segments: 'codebase-late-interaction.db.segments/manifest.json' } };
|
|
1163
1190
|
}
|
|
@@ -1170,7 +1197,7 @@ class ProductionReconcileAdapter {
|
|
|
1170
1197
|
* @returns {number}
|
|
1171
1198
|
*/
|
|
1172
1199
|
_publishedEpoch() {
|
|
1173
|
-
const merkle =
|
|
1200
|
+
const merkle = this._merkleRead() ?? {};
|
|
1174
1201
|
return Number.isInteger(merkle?.epoch) ? merkle.epoch : -1;
|
|
1175
1202
|
}
|
|
1176
1203
|
|
|
@@ -1225,6 +1252,8 @@ class ProductionReconcileAdapter {
|
|
|
1225
1252
|
merkle.epoch = manifest.epoch;
|
|
1226
1253
|
merkle.stats = { ...(merkle.stats || {}), totalFiles: Object.keys(merkle.files).length };
|
|
1227
1254
|
safeWriteJson(merklePath, merkle);
|
|
1255
|
+
// Invalidate the shared read cache — this is the one merkle writer.
|
|
1256
|
+
this._merkleCache = undefined;
|
|
1228
1257
|
// E.6: persist the updated chunk-cutoff cache once per tick (after the
|
|
1229
1258
|
// merkle advances). Best-effort; a failure only costs a redundant re-embed.
|
|
1230
1259
|
if (this._cutoffCache && this._cutoffDirty) {
|
|
@@ -461,7 +461,7 @@ export class Reconciler {
|
|
|
461
461
|
if (bin?.ops?.binary_hnsw_append != null) ops.binary_hnsw_append = bin.ops.binary_hnsw_append;
|
|
462
462
|
if (bin?.ops?.binary_hnsw_tombstone != null) ops.binary_hnsw_tombstone = bin.ops.binary_hnsw_tombstone;
|
|
463
463
|
this.progress('reconciler:li:start');
|
|
464
|
-
const li = await this.adapters.applyLIDelta?.(file, vec?.tokenOps ?? [], epoch);
|
|
464
|
+
const li = await this.adapters.applyLIDelta?.(file, vec?.tokenOps ?? [], epoch, tickCtx);
|
|
465
465
|
this.progress('reconciler:li:done');
|
|
466
466
|
collectManifestTier(manifestTiers, 'lateInteraction', li);
|
|
467
467
|
if (li?.ops?.li_segment_append != null) ops.li_segment_append = li.ops.li_segment_append;
|
|
@@ -192,13 +192,17 @@ function firstSafeRelativePath(...candidates) {
|
|
|
192
192
|
* carried inside the encoder-input dependency registry; see
|
|
193
193
|
* `core/incremental-indexing/domain/encoder-deps.mjs`).
|
|
194
194
|
*
|
|
195
|
+
* Pass `{ includeDedup: false }` when the caller never persists the dedup
|
|
196
|
+
* fingerprint (the full-indexing insert path) to skip that hash entirely.
|
|
197
|
+
*
|
|
195
198
|
* @param {object} chunk
|
|
196
|
-
* @
|
|
199
|
+
* @param {{includeDedup?: boolean}} [opts]
|
|
200
|
+
* @returns {{chunk_text_hash:string, embedding_input_hash:string, li_input_hash:string, metadata_fingerprint:string, dedup_fingerprint?:string}}
|
|
197
201
|
*/
|
|
198
|
-
export function chunkInputHashes(chunk) {
|
|
202
|
+
export function chunkInputHashes(chunk, { includeDedup = true } = {}) {
|
|
199
203
|
const text = chunk?.content || chunk?.text || '';
|
|
200
204
|
const meta = chunk?.metadata || {};
|
|
201
|
-
|
|
205
|
+
const hashes = {
|
|
202
206
|
chunk_text_hash: contentHashSync(text),
|
|
203
207
|
embedding_input_hash: denseInputHash(chunk),
|
|
204
208
|
li_input_hash: liInputHash(chunk),
|
|
@@ -214,8 +218,9 @@ export function chunkInputHashes(chunk) {
|
|
|
214
218
|
policy_embed: EMBED_TEXT_POLICY_VERSION,
|
|
215
219
|
policy_li: LI_INPUT_POLICY_VERSION,
|
|
216
220
|
})),
|
|
217
|
-
dedup_fingerprint: dedupFingerprint(chunk),
|
|
218
221
|
};
|
|
222
|
+
if (includeDedup) hashes.dedup_fingerprint = dedupFingerprint(chunk);
|
|
223
|
+
return hashes;
|
|
219
224
|
}
|
|
220
225
|
|
|
221
226
|
/**
|
|
@@ -52,7 +52,7 @@ export const ARTIFACT_THRESHOLDS = {
|
|
|
52
52
|
stateFile: '.sweet-search/artifact-rebuild-state.json',
|
|
53
53
|
};
|
|
54
54
|
|
|
55
|
-
import { BinaryHNSWIndex } from '../vector-store/binary-hnsw-index.js';
|
|
55
|
+
import { BinaryHNSWIndex, int8SidecarCount } from '../vector-store/binary-hnsw-index.js';
|
|
56
56
|
import { truncateForHNSW, fisherYatesShuffle, normalizedFloatToInt8, floatToBinary } from '../infrastructure/quantization.js';
|
|
57
57
|
import { FloatVectorStore, getFloatStorePath } from '../vector-store/float-vector-store.js';
|
|
58
58
|
|
|
@@ -828,8 +828,10 @@ export async function getArtifactStats() {
|
|
|
828
828
|
if (existsSync(int8SidecarPath)) {
|
|
829
829
|
try {
|
|
830
830
|
const fileStats = await fs.stat(int8SidecarPath);
|
|
831
|
-
|
|
832
|
-
|
|
831
|
+
// Header-only count (NDJSON v2 sidecar; v1 back-compat) — never
|
|
832
|
+
// JSON.parse the whole file, which exceeds V8's string ceiling on
|
|
833
|
+
// large indexes.
|
|
834
|
+
const count = await int8SidecarCount(int8SidecarPath);
|
|
833
835
|
|
|
834
836
|
stats.int8Sidecar = {
|
|
835
837
|
exists: true,
|
|
@@ -151,12 +151,12 @@ function buildEmbeddingText({ variant: variantOverride, content, relativePath, l
|
|
|
151
151
|
const pathLine = relativePath ? `# ${relativePath}` : null;
|
|
152
152
|
const parentLine = hierarchyInfo?.parentSymbol
|
|
153
153
|
? `# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}` : null;
|
|
154
|
-
// Ruby method chunks keep metadata anchors, but omit method-name header
|
|
155
|
-
// text so top-level Ruby snippets stay aligned with the pre-method-boundary
|
|
156
|
-
// embedding surface.
|
|
157
|
-
const isRubyMethodChunk = language === 'ruby'
|
|
158
|
-
&& (chunkType === 'method' || chunkType === 'singleton_method');
|
|
159
|
-
const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
|
|
154
|
+
// Ruby method chunks keep metadata anchors, but omit method-name header
|
|
155
|
+
// text so top-level Ruby snippets stay aligned with the pre-method-boundary
|
|
156
|
+
// embedding surface.
|
|
157
|
+
const isRubyMethodChunk = language === 'ruby'
|
|
158
|
+
&& (chunkType === 'method' || chunkType === 'singleton_method');
|
|
159
|
+
const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
|
|
160
160
|
? `# ${chunkType}: ${symbol}` : null;
|
|
161
161
|
const langLine = (language && language !== 'text')
|
|
162
162
|
? `# Language: ${language}` : null;
|
|
@@ -293,10 +293,10 @@ function buildLiText({ content, relativePath, language, chunkType, symbol, hiera
|
|
|
293
293
|
if (hierarchyInfo?.parentSymbol) {
|
|
294
294
|
lines.push(`# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}`);
|
|
295
295
|
}
|
|
296
|
-
// Mirror the Ruby method header carve-out used by embedding_text.
|
|
297
|
-
const isRubyMethodChunk = language === 'ruby'
|
|
298
|
-
&& (chunkType === 'method' || chunkType === 'singleton_method');
|
|
299
|
-
if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
|
|
296
|
+
// Mirror the Ruby method header carve-out used by embedding_text.
|
|
297
|
+
const isRubyMethodChunk = language === 'ruby'
|
|
298
|
+
&& (chunkType === 'method' || chunkType === 'singleton_method');
|
|
299
|
+
if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
|
|
300
300
|
lines.push(`# ${chunkType}: ${symbol}`);
|
|
301
301
|
}
|
|
302
302
|
// Mirror embedding_text: surface sibling symbol names so the LI MaxSim
|
|
@@ -391,27 +391,27 @@ export class ASTChunker {
|
|
|
391
391
|
const tsChunks = await provider.parseFileToChunks(content, langInfo.id);
|
|
392
392
|
if (!tsChunks || tsChunks.length === 0) return null;
|
|
393
393
|
|
|
394
|
-
return tsChunks.map(chunk => {
|
|
395
|
-
const isTopLevelRubyMethod = langInfo.id === 'ruby'
|
|
396
|
-
&& (chunk.type === 'method' || chunk.type === 'singleton_method')
|
|
397
|
-
&& !chunk.parentSymbol;
|
|
398
|
-
|
|
399
|
-
return this.buildChunk(
|
|
400
|
-
chunk.text, filePath, langInfo.id,
|
|
401
|
-
isTopLevelRubyMethod ? 'code' : chunk.type,
|
|
402
|
-
isTopLevelRubyMethod ? null : chunk.name,
|
|
403
|
-
chunk.startLine, chunk.endLine,
|
|
404
|
-
{
|
|
405
|
-
chunkId: chunk.chunkId,
|
|
406
|
-
parentChunkId: chunk.parentChunkId,
|
|
407
|
-
parentSymbol: chunk.parentSymbol,
|
|
408
|
-
parentType: chunk.parentType,
|
|
409
|
-
signature: chunk.signature || null,
|
|
410
|
-
additionalSymbols: chunk.additionalSymbols || null,
|
|
411
|
-
}
|
|
412
|
-
);
|
|
413
|
-
});
|
|
414
|
-
}
|
|
394
|
+
return tsChunks.map(chunk => {
|
|
395
|
+
const isTopLevelRubyMethod = langInfo.id === 'ruby'
|
|
396
|
+
&& (chunk.type === 'method' || chunk.type === 'singleton_method')
|
|
397
|
+
&& !chunk.parentSymbol;
|
|
398
|
+
|
|
399
|
+
return this.buildChunk(
|
|
400
|
+
chunk.text, filePath, langInfo.id,
|
|
401
|
+
isTopLevelRubyMethod ? 'code' : chunk.type,
|
|
402
|
+
isTopLevelRubyMethod ? null : chunk.name,
|
|
403
|
+
chunk.startLine, chunk.endLine,
|
|
404
|
+
{
|
|
405
|
+
chunkId: chunk.chunkId,
|
|
406
|
+
parentChunkId: chunk.parentChunkId,
|
|
407
|
+
parentSymbol: chunk.parentSymbol,
|
|
408
|
+
parentType: chunk.parentType,
|
|
409
|
+
signature: chunk.signature || null,
|
|
410
|
+
additionalSymbols: chunk.additionalSymbols || null,
|
|
411
|
+
}
|
|
412
|
+
);
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
415
|
|
|
416
416
|
parseBraceBasedFile(filePath, content, language, patterns, comment, multiLine) {
|
|
417
417
|
const chunks = [];
|
|
@@ -942,6 +942,7 @@ export class ASTChunker {
|
|
|
942
942
|
// Production embedding text. With the default variant (current) this
|
|
943
943
|
// is byte-identical to shipped v6.2; under a research-only variant
|
|
944
944
|
// (SWEET_SEARCH_EMBED_TEXT_VARIANT) it produces the experimental form.
|
|
945
|
+
const activeVariant = getEmbedTextVariant();
|
|
945
946
|
const embedding_text = buildEmbeddingText({
|
|
946
947
|
content,
|
|
947
948
|
relativePath,
|
|
@@ -954,17 +955,19 @@ export class ASTChunker {
|
|
|
954
955
|
// Research isolation: always build the shipped (current-variant)
|
|
955
956
|
// form alongside, so the LI stage can read a canonical input even
|
|
956
957
|
// when an R1 experiment is active on embedding_text. In production
|
|
957
|
-
// (variant=current)
|
|
958
|
-
//
|
|
959
|
-
const li_greedy_text =
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
958
|
+
// (variant=current) it IS the embedding_text — reuse the string
|
|
959
|
+
// instead of rebuilding it.
|
|
960
|
+
const li_greedy_text = activeVariant === 'current'
|
|
961
|
+
? embedding_text
|
|
962
|
+
: buildEmbeddingText({
|
|
963
|
+
variant: 'current',
|
|
964
|
+
content,
|
|
965
|
+
relativePath,
|
|
966
|
+
language,
|
|
967
|
+
chunkType,
|
|
968
|
+
symbol,
|
|
969
|
+
hierarchyInfo,
|
|
970
|
+
});
|
|
968
971
|
|
|
969
972
|
// Build LI text via the v6.2 builder (language-conditioned path
|
|
970
973
|
// policy). See buildLiText() docstring for rationale.
|
|
@@ -1034,26 +1037,26 @@ export class ASTChunker {
|
|
|
1034
1037
|
const parts = [];
|
|
1035
1038
|
parts.push(`# ${chunk.metadata.path}`);
|
|
1036
1039
|
|
|
1037
|
-
const isRubyMethodChunk = chunk.metadata.language === 'ruby'
|
|
1038
|
-
&& (chunk.metadata.chunk_type === 'method'
|
|
1039
|
-
|| chunk.metadata.chunk_type === 'singleton_method');
|
|
1040
|
-
const hasOnlySelfScope = scopeChain
|
|
1041
|
-
&& scopeChain.length === 1
|
|
1042
|
-
&& scopeChain[0] === chunk.metadata.symbol
|
|
1043
|
-
&& !chunk.metadata.parent_symbol;
|
|
1044
|
-
|
|
1045
|
-
if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
|
|
1046
|
-
parts.push(`# Scope: ${scopeChain.join(' > ')}`);
|
|
1047
|
-
} else if (chunk.metadata.parent_symbol) {
|
|
1048
|
-
// Preserve cAST parent context when no scope chain from code graph
|
|
1049
|
-
parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
|
|
1050
|
-
}
|
|
1051
|
-
// Keep Ruby method metadata, but avoid injecting the method name into
|
|
1052
|
-
// the production embedding/LI text. Top-level Ruby method chunks also
|
|
1053
|
-
// skip self-only scope above.
|
|
1054
|
-
if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
|
|
1055
|
-
parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
|
|
1056
|
-
}
|
|
1040
|
+
const isRubyMethodChunk = chunk.metadata.language === 'ruby'
|
|
1041
|
+
&& (chunk.metadata.chunk_type === 'method'
|
|
1042
|
+
|| chunk.metadata.chunk_type === 'singleton_method');
|
|
1043
|
+
const hasOnlySelfScope = scopeChain
|
|
1044
|
+
&& scopeChain.length === 1
|
|
1045
|
+
&& scopeChain[0] === chunk.metadata.symbol
|
|
1046
|
+
&& !chunk.metadata.parent_symbol;
|
|
1047
|
+
|
|
1048
|
+
if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
|
|
1049
|
+
parts.push(`# Scope: ${scopeChain.join(' > ')}`);
|
|
1050
|
+
} else if (chunk.metadata.parent_symbol) {
|
|
1051
|
+
// Preserve cAST parent context when no scope chain from code graph
|
|
1052
|
+
parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
|
|
1053
|
+
}
|
|
1054
|
+
// Keep Ruby method metadata, but avoid injecting the method name into
|
|
1055
|
+
// the production embedding/LI text. Top-level Ruby method chunks also
|
|
1056
|
+
// skip self-only scope above.
|
|
1057
|
+
if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
|
|
1058
|
+
parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
|
|
1059
|
+
}
|
|
1057
1060
|
|
|
1058
1061
|
if (chunk.metadata.additional_symbols && chunk.metadata.additional_symbols.length > 0) {
|
|
1059
1062
|
parts.push(`# Additional: ${chunk.metadata.additional_symbols.join(', ')}`);
|
|
@@ -1267,12 +1267,14 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
|
|
|
1267
1267
|
// Best-effort — never block startup on the embedding module.
|
|
1268
1268
|
if (process.env.SWEET_SEARCH_ORT_BACKGROUND !== '0') {
|
|
1269
1269
|
try {
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1270
|
+
// Arms BOTH resident CPU sessions (dense + late-interaction) — the LI
|
|
1271
|
+
// session previously stayed on the foreground profile (arena ON +
|
|
1272
|
+
// allow_spinning) inside the daemon; measured consequence: 354 × 128MB
|
|
1273
|
+
// arena extensions (~34GB RSS) after one edit-heavy day plus ~40% idle
|
|
1274
|
+
// CPU from spinning workers.
|
|
1275
|
+
const { armBackgroundOrtProfiles } = await import('./model-pool.js');
|
|
1276
|
+
armBackgroundOrtProfiles();
|
|
1277
|
+
log('INFO', 'ORT background profile armed for maintainer daemon (dense + LI: force_spinning_stop + arena-off + bg threads)');
|
|
1276
1278
|
} catch (err) {
|
|
1277
1279
|
log('WARN', `ORT background profile arming failed (continuing on foreground profile): ${err?.message ?? err}`);
|
|
1278
1280
|
}
|
|
@@ -1314,15 +1316,30 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
|
|
|
1314
1316
|
// system-RAM tier auto-enables a cap (small-RAM hosts). Best-effort, guarded
|
|
1315
1317
|
// so a missing module is a no-op.
|
|
1316
1318
|
let rssRegistration = null;
|
|
1319
|
+
// D.5 per-process recycle ceiling: fleet-budget eviction only sheds the
|
|
1320
|
+
// longest-IDLE daemon, so an active maintainer needs its own line. Resolved
|
|
1321
|
+
// once; checked at tick boundaries below. 0 ⇒ disabled.
|
|
1322
|
+
let rssCeiling = { ceilingBytes: 0, minUptimeMs: 0, shouldRecycleForRss: null };
|
|
1317
1323
|
try {
|
|
1318
1324
|
const mod = await import('./rss-budget.mjs');
|
|
1319
1325
|
if (typeof mod.isEnabled === 'function' && mod.isEnabled()
|
|
1320
1326
|
&& typeof mod.registerDaemon === 'function') {
|
|
1321
1327
|
rssRegistration = await mod.registerDaemon({ pid: process.pid, stateDir: ctx.stateDir, kind: 'maintainer' });
|
|
1322
1328
|
}
|
|
1329
|
+
if (typeof mod.maintainerRssCeilingBytes === 'function') {
|
|
1330
|
+
rssCeiling = {
|
|
1331
|
+
ceilingBytes: mod.maintainerRssCeilingBytes(),
|
|
1332
|
+
minUptimeMs: mod.maintainerRssMinUptimeMs(),
|
|
1333
|
+
shouldRecycleForRss: mod.shouldRecycleForRss,
|
|
1334
|
+
};
|
|
1335
|
+
if (rssCeiling.ceilingBytes > 0) {
|
|
1336
|
+
log('INFO', `Maintainer RSS recycle ceiling armed: ${(rssCeiling.ceilingBytes / 1048576).toFixed(0)}MB (min uptime ${rssCeiling.minUptimeMs}ms; override SWEET_SEARCH_MAINTAINER_RSS_MAX_MB)`);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1323
1339
|
} catch (err) {
|
|
1324
1340
|
log('WARN', `RSS-budget registry unavailable (no soft cap on this daemon): ${err?.message ?? err}`);
|
|
1325
1341
|
}
|
|
1342
|
+
let completedTicks = 0;
|
|
1326
1343
|
|
|
1327
1344
|
// D.1 idle-TTL: an unattended maintainer self-shuts-down after the configured
|
|
1328
1345
|
// wall-clock idle budget so N resident model-loaded daemons collapse to 1–2
|
|
@@ -1432,6 +1449,23 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
|
|
|
1432
1449
|
maintenanceBacklog: backlog,
|
|
1433
1450
|
skipped: tickCounters?.skipped === true,
|
|
1434
1451
|
});
|
|
1452
|
+
// D.5: per-process RSS recycle check — tick boundary only (the tick
|
|
1453
|
+
// above has published; a recycle here can never tear an artifact).
|
|
1454
|
+
completedTicks += 1;
|
|
1455
|
+
if (rssCeiling.ceilingBytes > 0 && typeof rssCeiling.shouldRecycleForRss === 'function') {
|
|
1456
|
+
const rssNow = process.memoryUsage.rss();
|
|
1457
|
+
const verdict = rssCeiling.shouldRecycleForRss({
|
|
1458
|
+
rssBytes: rssNow,
|
|
1459
|
+
ceilingBytes: rssCeiling.ceilingBytes,
|
|
1460
|
+
uptimeMs: process.uptime() * 1000,
|
|
1461
|
+
minUptimeMs: rssCeiling.minUptimeMs,
|
|
1462
|
+
ticksCompleted: completedTicks,
|
|
1463
|
+
});
|
|
1464
|
+
if (verdict.recycle) {
|
|
1465
|
+
log('WARN', `Maintainer RSS ${(rssNow / 1048576).toFixed(0)}MB exceeds recycle ceiling ${(rssCeiling.ceilingBytes / 1048576).toFixed(0)}MB; requesting clean shutdown for on-demand respawn. If this repeats right after startup, raise SWEET_SEARCH_MAINTAINER_RSS_MAX_MB.`);
|
|
1466
|
+
shutdownRequested = true;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1435
1469
|
} catch (err) {
|
|
1436
1470
|
if (err instanceof MaintainerLifecycleAbort) {
|
|
1437
1471
|
log('WARN', `Reconcile v2 lifecycle abort: ${err.message}. Cleaning up cancellation-orphaned temps and exiting cleanly.`);
|
|
@@ -1457,6 +1491,10 @@ async function runReconcileV2Main({ runOnce, merkleOnce }) {
|
|
|
1457
1491
|
log('ERROR', `Reconcile v2 tick failed: ${err?.message ?? err}`);
|
|
1458
1492
|
}
|
|
1459
1493
|
}
|
|
1494
|
+
// Skip the sleep when a shutdown was requested during this iteration
|
|
1495
|
+
// (idle-TTL, signal, or D.5 RSS recycle) — exit promptly instead of
|
|
1496
|
+
// waiting out one more interval.
|
|
1497
|
+
if (shutdownRequested) break;
|
|
1460
1498
|
await sleepWithProgress(intervalMs, lock.lockFile, {
|
|
1461
1499
|
// G6 early-wake: break the sleep the instant the watcher reports new
|
|
1462
1500
|
// events so a fresh edit is reconciled without waiting out the interval.
|
|
@@ -2482,6 +2520,11 @@ async function main() {
|
|
|
2482
2520
|
const dryRun = process.argv.includes('--dry-run');
|
|
2483
2521
|
const merkleOnce = process.argv.includes('--merkle-once');
|
|
2484
2522
|
|
|
2523
|
+
// Name the resident daemon so ps/Activity Monitor shows what it is instead
|
|
2524
|
+
// of an anonymous `node`. Cosmetic-only: all liveness/takeover checks are
|
|
2525
|
+
// lock/pid-based, never command-line matching.
|
|
2526
|
+
try { process.title = 'sweet-search-maintainer'; } catch { /* best-effort */ }
|
|
2527
|
+
|
|
2485
2528
|
// A.1 (Tier-1, UNGATED): demote the maintainer daemon to low OS priority so
|
|
2486
2529
|
// the foreground (editor / git / shell) never feels the background indexer's
|
|
2487
2530
|
// CPU. Identical index output — only *when* CPU is granted changes. Covers
|