sweet-search 2.5.13 → 2.6.0

package/core/search/session-daemon-prewarm.mjs

@@ -22,7 +22,15 @@
  */
 
 import { spawn } from 'node:child_process';
-import { existsSync, readFileSync, openSync, writeSync, closeSync, unlinkSync } from 'node:fs';
+import {
+  existsSync,
+  readFileSync,
+  readdirSync,
+  openSync,
+  writeSync,
+  closeSync,
+  unlinkSync,
+} from 'node:fs';
 import { connect } from 'node:net';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -44,6 +52,92 @@ const log = (msg) => {
   if (verbose) process.stderr.write(`[sweet-search prewarm] ${msg}\n`);
 };
 
+function pageCacheSweepEnabled() {
+  const v = String(process.env.SWEET_SEARCH_PREWARM_PAGE_CACHE || '').trim().toLowerCase();
+  return v !== '0' && v !== 'false' && v !== 'off' && v !== 'no';
+}
+
+function readAndDiscard(filePath, deadlineMs) {
+  if (!filePath || Date.now() >= deadlineMs || !existsSync(filePath)) return false;
+  try {
+    readFileSync(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function pageCacheSweep() {
+  const rawBudgetMs = Number(process.env.SWEET_SEARCH_PREWARM_PAGE_CACHE_MS || 3500);
+  const budgetMs = Number.isFinite(rawBudgetMs) ? Math.max(250, rawBudgetMs) : 3500;
+  const deadlineMs = Date.now() + budgetMs;
+  let warmed = 0;
+  try {
+    const { DB_PATHS, LATE_INTERACTION_CONFIG } = await import('../infrastructure/config/index.js');
+    const files = [
+      DB_PATHS.codeGraph,
+      DB_PATHS.lateInteraction,
+      DB_PATHS.sparseGramIndex,
+      // codebase.db is opened by the read tool (CodebaseRepository) on every
+      // invocation but was the only query-path DB the sweep omitted; warming
+      // its page cache means the first read call pays no cold SQLite I/O.
+      // readAndDiscard no-ops on an absent file, so this is harmless when the
+      // index predates codebase.db or it was never built.
+      DB_PATHS.codebase,
+    ];
+
+    const segmentDir = DB_PATHS.lateInteraction ? `${DB_PATHS.lateInteraction}.segments` : null;
+    if (segmentDir && existsSync(segmentDir)) {
+      try {
+        for (const name of readdirSync(segmentDir)) {
+          if (name.endsWith('.bin')) files.push(join(segmentDir, name));
+        }
+      } catch {
+        /* best-effort page-cache prewarm */
+      }
+    }
+
+    const modelConfig = LATE_INTERACTION_CONFIG.activeModel;
+    if (modelConfig?.hfId) {
+      try {
+        const { getModelCacheDir } = await import('../infrastructure/model-fetcher.js');
+        const modelDir = getModelCacheDir(modelConfig.hfId);
+        files.push(join(modelDir, modelConfig.onnxFile));
+        files.push(join(modelDir, 'tokenizer.json'));
+      } catch {
+        /* model cache unavailable; do not fetch or load sessions here */
+      }
+    }
+
+    for (const filePath of files) {
+      if (readAndDiscard(filePath, deadlineMs)) warmed++;
+    }
+    log(`page-cache sweep warmed ${warmed} file(s)`);
+  } catch (err) {
+    log(`page-cache sweep non-fatal: ${err?.message || err}`);
+  }
+}
+
+function spawnPageCacheSweep() {
+  if (!pageCacheSweepEnabled()) return;
+  if (process.env.SWEET_SEARCH_PAGE_CACHE_SWEEP === '1') return;
+  try {
+    const child = spawn(process.execPath, [fileURLToPath(import.meta.url)], {
+      detached: true,
+      stdio: 'ignore',
+      cwd: process.cwd(),
+      env: {
+        ...process.env,
+        SWEET_SEARCH_PAGE_CACHE_SWEEP: '1',
+      },
+    });
+    child.unref();
+    log(`page-cache sweep spawned (pid ${child.pid}, detached)`);
+  } catch (err) {
+    log(`page-cache sweep spawn non-fatal: ${err?.message || err}`);
+  }
+}
+
 /** Does a process with this PID exist right now? Handles EPERM (alien user) as "alive". */
 function pidAlive(pid) {
   if (!pid || !Number.isFinite(pid)) return false;
@@ -165,9 +259,24 @@ async function prewarmServer() {
   }
 }
 
+if (process.env.SWEET_SEARCH_PAGE_CACHE_SWEEP === '1') {
+  try {
+    await pageCacheSweep();
+  } catch (err) {
+    log(`page-cache sweep non-fatal: ${err?.message || err}`);
+  }
+  process.exit(0);
+}
+
 // The search server and the index maintainer are independent: a stuck/already
 // running server must not stop the maintainer from starting, and vice versa.
 // Each is isolated in its own try so one failing never blocks the other.
+try {
+  spawnPageCacheSweep();
+} catch (err) {
+  log(`page-cache sweep spawn non-fatal: ${err?.message || err}`);
+}
+
 try {
   await prewarmServer();
 } catch (err) {

package/core/search/sweet-search.js

@@ -17,7 +17,6 @@ import { getGlobalLocalReranker } from '../ranking/local-reranker.js';
 import { QueryRouter, routeQuery } from '../query/query-router.js';
 import { GraphSearch } from '../graph/graph-search.js';
 import { SYMBOL_KIND_WEIGHTS, DEFINITION_TYPES } from '../infrastructure/constants.js';
-import { HNSWIndex } from '../vector-store/hnsw-index.js';
 import { BinaryHNSWIndex } from '../vector-store/binary-hnsw-index.js';
 import { Reranker } from '../ranking/flashrank.js';
 import { LateInteractionIndex } from '../ranking/late-interaction-index.js';
@@ -133,9 +132,7 @@ export class SweetSearch {
     this._manifestGraphDbPath = this.graphDbPath;
     this.graphSearch = new GraphSearch(this.graphDbPath);
     this.codeGraphRepo = new CodeGraphRepository(this.graphDbPath);
-    this.hnswPath = options.hnswPath || DB_PATHS.hnswIndex;
     this.binaryHnswPath = options.binaryHnswPath || DB_PATHS.binaryHnswIndex;
-    this.hnswIndex = new HNSWIndex({ indexPath: this.hnswPath });
     this.binaryHnswIndex = new BinaryHNSWIndex({ indexPath: this.binaryHnswPath });
     this.reranker = new Reranker(options);
     this.lateInteractionIndex = new LateInteractionIndex(options.lateInteractionOptions || {});
@@ -228,7 +225,6 @@ export class SweetSearch {
     this._syncManifestPaths(this._readReconcileManifest());
 
     this.hasGraphIndex = existsSync(this.graphDbPath);
-    this.hasHnswIndex = existsSync(this.hnswPath.replace('.idx', '.meta.json'));
     this.hasBinaryHnswIndex = existsSync(this.binaryHnswPath.replace('.idx', '.meta.json'));
     this.hasCodebaseIndex = existsSync(this.codebaseDbPath);
     this.hasLateInteractionIndex = existsSync(this.lateInteractionIndex.indexPath);
@@ -271,16 +267,6 @@ export class SweetSearch {
       }
     }
 
-    if (this.hasHnswIndex) {
-      try {
-        await this.hnswIndex.load(undefined, { mmap: true });
-        this.log(`HNSW: Loaded ${this.hnswIndex.getStats().totalVectors} vectors (mmap)`);
-      } catch (err) {
-        this.log(`HNSW: Failed to load: ${err.message}`);
-        this.hasHnswIndex = false;
-      }
-    }
-
     if (this.hasLateInteractionIndex && this.useLateInteraction) {
       try {
         await this.lateInteractionIndex.init();
@@ -433,8 +419,6 @@ export class SweetSearch {
     return {
       codebaseDbPath: this._resolveStatePath(manifest.vectors?.path),
       graphDbPath: this._resolveStatePath(manifest.codeGraph?.path),
-      hnswPath: this._resolveStatePath(manifest.hnsw?.path),
-      hnswStalePath: this._resolveStatePath(manifest.hnsw?.stale),
       binaryHnswPath: this._resolveStatePath(manifest.binaryHnsw?.path),
       binaryHnswStalePath: this._resolveStatePath(manifest.binaryHnsw?.stale),
       lateInteractionIndexPath: liIndexPath,
@@ -461,10 +445,6 @@ export class SweetSearch {
       this.graphSearch = new GraphSearch(this._manifestGraphDbPath);
       this.codeGraphRepo = new CodeGraphRepository(this._manifestGraphDbPath);
     }
-    if (paths.hnswPath && (paths.hnswPath !== this.hnswPath || paths.hnswStalePath !== this.hnswIndex?.stalePath)) {
-      this.hnswPath = paths.hnswPath;
-      this.hnswIndex = new HNSWIndex({ indexPath: this.hnswPath, stalePath: paths.hnswStalePath || `${this.hnswPath}.stale.bin` });
-    }
     if (paths.binaryHnswPath && (paths.binaryHnswPath !== this.binaryHnswPath || paths.binaryHnswStalePath !== this.binaryHnswIndex?.stalePath)) {
       this.binaryHnswPath = paths.binaryHnswPath;
       this.binaryHnswIndex = new BinaryHNSWIndex({ indexPath: this.binaryHnswPath, stalePath: paths.binaryHnswStalePath || `${this.binaryHnswPath}.stale.bin` });
@@ -513,23 +493,6 @@ export class SweetSearch {
       }
     }
 
-    if (!grepOnly) {
-      this.hasHnswIndex = existsSync(this.hnswPath.replace('.idx', '.meta.json'));
-    }
-    if (!grepOnly && this.hasHnswIndex) {
-      try {
-        const nextHnsw = new HNSWIndex({
-          indexPath: this.hnswPath,
-          stalePath: this.hnswIndex?.stalePath || `${this.hnswPath}.stale.bin`,
-        });
-        await nextHnsw.load(undefined, { mmap: true });
-        this.hnswIndex = nextHnsw;
-      } catch (err) {
-        this.log(`HNSW: Failed to reload after manifest publish: ${err.message}`);
-        this.hasHnswIndex = false;
-      }
-    }
-
     if (!grepOnly) {
       this.hasLateInteractionIndex = existsSync(this.lateInteractionIndex.indexPath);
     }
@@ -1098,7 +1061,6 @@ Object.assign(SweetSearch.prototype, {
   semanticSearch3Stage: semantic.semanticSearch3Stage,
   semanticSearchStandard: semantic.semanticSearchStandard,
   shouldSkipRerank: semantic.shouldSkipRerank,
-  getAdaptiveCandidateCount: semantic.getAdaptiveCandidateCount,
   hybridSearchV2: hybrid.hybridSearchV2,
   hybridSearch: hybrid.hybridSearch,
   patternSearch: pattern.patternSearch,