sweet-search 2.5.2 → 2.5.3

package/core/incremental-indexing/infrastructure/vector-gc.mjs

@@ -0,0 +1,133 @@
+/**
+ * Retired-vector physical GC for `codebase.db`.
+ *
+ * `applyDiff` (vector-delta-writer) only *tombstones* superseded rows by
+ * setting `epoch_retired`; nothing ever deletes them, so `codebase.db` grows
+ * without bound under a long-lived daemon. This module physically removes
+ * retired rows once no reader can still observe them.
+ *
+ * Safety (strict visibility, plan § 8.1.1):
+ *   A row is visible to a reader pinned at manifest epoch E iff
+ *     epoch_written <= E AND (epoch_retired IS NULL OR epoch_retired > E).
+ *   So a row with `epoch_retired = R` is invisible to every reader whose
+ *   pinned epoch E satisfies E >= R. The smallest epoch any live reader
+ *   pins is `minLiveEpoch` (from reader heartbeats); the repository always
+ *   re-syncs to the *latest* manifest, so a reader's query epoch is never
+ *   below its heartbeat epoch. Therefore deleting rows with
+ *   `epoch_retired <= frontier`, where `frontier = minLiveEpoch ??
+ *   currentManifestEpoch`, can never remove a row any reader still sees:
+ *     - readers present  → frontier = minLiveEpoch <= every reader's epoch.
+ *     - no readers       → frontier = currentManifestEpoch; any future
+ *       reader reads a manifest at epoch >= currentManifestEpoch (monotonic),
+ *       so deleted rows (retired <= currentManifestEpoch <= future E) are
+ *       already invisible to it.
+ *
+ * The float/binary HNSW indices keep their own vector copies and are rebuilt
+ * from live rows (`epoch_retired IS NULL`); they never read retired rows, so
+ * deleting retired DB rows cannot desync them.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import Database from 'better-sqlite3';
+
+export const DEFAULT_GC_BATCH = 2000;
+export const DEFAULT_GC_MAX_ROWS = 100_000;
+
+/**
+ * Delete retired rows at or below `frontier` in bounded batches.
+ *
+ * Uses a `rowid IN (SELECT … LIMIT ?)` subquery so it works regardless of
+ * whether SQLite was compiled with `SQLITE_ENABLE_UPDATE_DELETE_LIMIT`.
+ *
+ * @param {import('better-sqlite3').Database} db   open read-write connection
+ * @param {number} frontier                        inclusive prune boundary
+ * @param {{batchSize?:number, maxRows?:number}} [opts]
+ * @returns {{deleted:number, batches:number, hitCap:boolean}}
+ */
+export function pruneRetiredVectors(db, frontier, opts = {}) {
+  if (!Number.isInteger(frontier)) {
+    throw new Error(`pruneRetiredVectors: frontier must be an integer, got ${frontier}`);
+  }
+  const batchSize = Number.isInteger(opts.batchSize) && opts.batchSize > 0 ? opts.batchSize : DEFAULT_GC_BATCH;
+  const maxRows = Number.isInteger(opts.maxRows) && opts.maxRows > 0 ? opts.maxRows : DEFAULT_GC_MAX_ROWS;
+
+  const stmt = db.prepare(`
+    DELETE FROM vectors
+     WHERE rowid IN (
+       SELECT rowid FROM vectors
+        WHERE epoch_retired IS NOT NULL AND epoch_retired <= ?
+        LIMIT ?
+     )
+  `);
+
+  let deleted = 0;
+  let batches = 0;
+  let hitCap = false;
+  for (;;) {
+    const remainingCap = maxRows - deleted;
+    if (remainingCap <= 0) { hitCap = true; break; }
+    const take = Math.min(batchSize, remainingCap);
+    const res = stmt.run(frontier, take);
+    const changes = res.changes ?? 0;
+    deleted += changes;
+    batches += 1;
+    if (changes < take) break; // drained
+  }
+  return { deleted, batches, hitCap };
+}
+
+/**
+ * Run retired-vector GC against `<stateDir>/codebase.db`.
+ *
+ * Computes the safe prune frontier from reader heartbeats (falling back to
+ * the current manifest epoch when no readers are live), prunes in bounded
+ * batches, then issues a PASSIVE WAL checkpoint to keep the WAL from
+ * growing without ever blocking concurrent readers. Never throws on a
+ * missing DB / column / heartbeat dir — returns `{ skipped }` instead.
+ *
+ * @param {string} stateDir
+ * @param {{
+ *   dbPath?:string, batchSize?:number, maxRows?:number,
+ *   minLiveEpoch?:(dir:string)=>(number|null),
+ *   readManifest?:(dir:string)=>(object|null),
+ * }} [deps]
+ * @returns {{deleted:number, frontier:number, hadReaders:boolean, batches:number, hitCap:boolean}|{skipped:string}}
+ */
+export function runVectorGc(stateDir, deps = {}) {
+  const dbPath = deps.dbPath || path.join(stateDir, 'codebase.db');
+  if (!fs.existsSync(dbPath)) return { skipped: 'no-vector-db' };
+
+  const minLiveEpochFn = deps.minLiveEpoch;
+  const readManifestFn = deps.readManifest;
+  if (typeof minLiveEpochFn !== 'function' || typeof readManifestFn !== 'function') {
+    throw new Error('runVectorGc: minLiveEpoch and readManifest deps are required');
+  }
+
+  let frontier = null;
+  let hadReaders = false;
+  const live = minLiveEpochFn(stateDir);
+  if (Number.isInteger(live)) {
+    frontier = live;
+    hadReaders = true;
+  } else {
+    const manifest = readManifestFn(stateDir);
+    if (Number.isInteger(manifest?.epoch)) frontier = manifest.epoch;
+  }
+  if (!Number.isInteger(frontier)) return { skipped: 'no-frontier' };
+
+  const db = new Database(dbPath);
+  try {
+    const cols = db.prepare('PRAGMA table_info(vectors)').all().map((c) => c.name);
+    if (!cols.includes('epoch_retired')) return { skipped: 'no-epoch-column' };
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = NORMAL');
+    const result = pruneRetiredVectors(db, frontier, deps);
+    if (result.deleted > 0) {
+      try { db.pragma('wal_checkpoint(PASSIVE)'); } catch { /* best-effort */ }
+    }
+    return { ...result, frontier, hadReaders };
+  } finally {
+    db.close();
+  }
+}

package/core/incremental-indexing/infrastructure/worktree-stamp.mjs

@@ -0,0 +1,155 @@
+/**
+ * Cross-worktree DB stamping.
+ *
+ * Plan § 8.5 / § 14.2.4. Sweet-search's `.sweet-search/` directory holds
+ * the index for one project; when a user has multiple worktrees pointing
+ * at the same repo, they either get their own state dir each (the
+ * default) or share one. Sharing is allowed only when:
+ *
+ *   - `project_root` matches the recorded stamp, AND
+ *   - `git common-dir` matches the recorded stamp.
+ *
+ * A mismatched stamp aborts daemon startup with a clear remediation
+ * message instead of silently mixing index histories.
+ *
+ * Stamp file: `.sweet-search/worktree-stamp.json`:
+ *
+ *   {
+ *     "projectRoot": "/Users/x/repo",
+ *     "gitCommonDir": "/Users/x/repo/.git",
+ *     "stampedAt": "2026-05-16T00:00:00.000Z"
+ *   }
+ *
+ * The stamp is informational and additive — the lockfile (`lockfile.mjs`)
+ * is what enforces single-writer semantics; the stamp catches the
+ * "two worktrees writing to the same state dir" footgun before it
+ * corrupts the index.
+ */
+
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const STAMP_FILENAME = 'worktree-stamp.json';
+
+function stampPath(stateDir) {
+  return path.join(stateDir, STAMP_FILENAME);
+}
+
+/**
+ * Resolve `git rev-parse --git-common-dir` for the given project root.
+ * Returns `null` when not a git repo or git is unavailable.
+ *
+ * @param {string} projectRoot
+ * @returns {string|null}
+ */
+export function gitCommonDir(projectRoot) {
+  try {
+    const r = spawnSync('git', ['rev-parse', '--git-common-dir'], {
+      cwd: projectRoot,
+      encoding: 'utf-8',
+    });
+    if (r.status !== 0) return null;
+    const out = (r.stdout || '').trim();
+    if (!out) return null;
+    return path.isAbsolute(out) ? out : path.resolve(projectRoot, out);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read the stamp from disk, returning `null` when missing.
+ *
+ * @param {string} stateDir
+ * @returns {{projectRoot:string, gitCommonDir:string|null, stampedAt:string}|null}
+ */
+export function readStamp(stateDir) {
+  const p = stampPath(stateDir);
+  if (!fs.existsSync(p)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(p, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Write a stamp for the current project. Caller invokes on first daemon
+ * start after a freshly initialised `.sweet-search/` dir.
+ *
+ * @param {string} stateDir
+ * @param {string} projectRoot
+ */
+export function writeStamp(stateDir, projectRoot) {
+  fs.mkdirSync(stateDir, { recursive: true });
+  const stamp = {
+    projectRoot: path.resolve(projectRoot),
+    gitCommonDir: gitCommonDir(projectRoot),
+    stampedAt: new Date().toISOString(),
+  };
+  fs.writeFileSync(stampPath(stateDir), JSON.stringify(stamp, null, 2));
+  return stamp;
+}
+
+/**
+ * Verify the stamp matches the current project. Returns
+ *   { ok: true }                  when the stamp matches (or is absent).
+ *   { ok: false, reason, expected, actual }   on mismatch.
+ *
+ * Caller should:
+ *   - on absent stamp → call `writeStamp` to mint one;
+ *   - on mismatch → log ERROR with the remediation message and exit.
+ *
+ * @param {string} stateDir
+ * @param {string} projectRoot
+ * @returns {{ok:boolean, reason?:string, expected?:object, actual?:object}}
+ */
+export function verifyStamp(stateDir, projectRoot) {
+  const stamp = readStamp(stateDir);
+  if (!stamp) return { ok: true, reason: 'absent' };
+  const actualProject = path.resolve(projectRoot);
+  const actualGit = gitCommonDir(projectRoot);
+  if (stamp.projectRoot !== actualProject) {
+    return {
+      ok: false,
+      reason: 'projectRoot-mismatch',
+      expected: { projectRoot: stamp.projectRoot, gitCommonDir: stamp.gitCommonDir },
+      actual: { projectRoot: actualProject, gitCommonDir: actualGit },
+    };
+  }
+  if (stamp.gitCommonDir && actualGit && stamp.gitCommonDir !== actualGit) {
+    return {
+      ok: false,
+      reason: 'gitCommonDir-mismatch',
+      expected: { projectRoot: stamp.projectRoot, gitCommonDir: stamp.gitCommonDir },
+      actual: { projectRoot: actualProject, gitCommonDir: actualGit },
+    };
+  }
+  return { ok: true, reason: 'match' };
+}
+
+/**
+ * Compose the operator-facing error message for a stamp mismatch.
+ *
+ * @param {{reason:string, expected:object, actual:object}} mismatch
+ * @returns {string}
+ */
+export function formatStampMismatch(mismatch) {
+  const linesExpected = [
+    `  expected projectRoot:    ${mismatch.expected.projectRoot}`,
+    `  expected gitCommonDir:   ${mismatch.expected.gitCommonDir ?? '(none)'}`,
+  ];
+  const linesActual = [
+    `  actual   projectRoot:    ${mismatch.actual.projectRoot}`,
+    `  actual   gitCommonDir:   ${mismatch.actual.gitCommonDir ?? '(none)'}`,
+  ];
+  return [
+    `[sweet-search] worktree-stamp mismatch (${mismatch.reason}):`,
+    ...linesExpected,
+    ...linesActual,
+    '',
+    'Remediation: either use a per-worktree .sweet-search/ directory (default)',
+    'or reset the shared index with `npm run index` from this worktree.',
+  ].join('\n');
+}

package/core/incremental-indexing/infrastructure/wsl2-detect.mjs

@@ -0,0 +1,115 @@
+/**
+ * WSL2 detection + watcher default policy.
+ *
+ * Plan § 34.6 / § 37.3.3. The earlier blanket "WSL2 → polling-only" rule
+ * over-penalised users on native Linux paths inside WSL2; an earlier
+ * draft proposed parsing `df -T` or `/proc/mounts` to detect the actual
+ * filesystem type, but the parser was brittle across Linux distros, WSL
+ * versions, and mount-point naming.
+ *
+ * The pragmatic compromise:
+ *   1. Detect WSL2 via `/proc/version` containing `microsoft` or `WSL`.
+ *   2. If detected, recommend `SWEET_SEARCH_WATCH=0` **as default** (not
+ *      forced).
+ *   3. If the user explicitly set `SWEET_SEARCH_WATCH=1`, respect it.
+ *   4. If the watcher fails at startup (inotify error on a 9p mount),
+ *      fall back to polling with a clear remediation message rather
+ *      than crashing.
+ *
+ * This module is read-only — it inspects the host environment and
+ * returns a recommendation. The caller (Phase 4 file watcher setup)
+ * acts on the recommendation.
+ */
+
+import fs from 'node:fs';
+
+let cached = null;
+
+function detectInner() {
+  // Containers also typically have inotify problems with bind mounts.
+  let kubernetes = false;
+  let docker = false;
+  try {
+    if (process.env.KUBERNETES_SERVICE_HOST) kubernetes = true;
+    if (fs.existsSync('/.dockerenv')) docker = true;
+    if (!docker && fs.existsSync('/proc/1/cgroup')) {
+      const c = fs.readFileSync('/proc/1/cgroup', 'utf-8');
+      if (/docker|kubepods|containerd/i.test(c)) docker = true;
+    }
+  } catch {
+    // ignore
+  }
+
+  let wsl2 = false;
+  try {
+    if (fs.existsSync('/proc/version')) {
+      const v = fs.readFileSync('/proc/version', 'utf-8').toLowerCase();
+      if (v.includes('microsoft') || v.includes('wsl')) wsl2 = true;
+    }
+  } catch {
+    // ignore
+  }
+
+  return {
+    wsl2,
+    docker,
+    kubernetes,
+    container: docker || kubernetes,
+    platform: process.platform,
+  };
+}
+
+export function detectEnvironment() {
+  if (cached === null) cached = detectInner();
+  return cached;
+}
+
+/**
+ * Decide the watcher's default-enabled state for this host.
+ *
+ *   - Explicit env var wins (`SWEET_SEARCH_WATCH=1` or `0`).
+ *   - WSL2 → polling default with override available.
+ *   - Container → polling default.
+ *   - Windows native → polling default (Phase 4 ships node:fs.watch; the
+ *     Rust notify binding via `ReadDirectoryChangesW` is Phase 6).
+ *   - Otherwise → watcher enabled.
+ *
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {{watcherEnabled:boolean, reason:string}}
+ */
+export function watcherDefault(env = process.env) {
+  const explicit = env.SWEET_SEARCH_WATCH;
+  if (explicit === '1' || explicit === 'true' || explicit === 'on') {
+    return { watcherEnabled: true, reason: 'env-override-on' };
+  }
+  if (explicit === '0' || explicit === 'false' || explicit === 'off') {
+    return { watcherEnabled: false, reason: 'env-override-off' };
+  }
+  const detect = detectEnvironment();
+  if (detect.wsl2) {
+    return {
+      watcherEnabled: false,
+      reason: 'wsl2-polling-default (set SWEET_SEARCH_WATCH=1 if your project is on native ext4 e.g. /home/user/project)',
+    };
+  }
+  if (detect.container) {
+    return { watcherEnabled: false, reason: 'container-polling-default' };
+  }
+  if (detect.platform === 'win32') {
+    return { watcherEnabled: false, reason: 'win32-polling-default (Phase 6 Rust notify binding pending)' };
+  }
+  return { watcherEnabled: true, reason: 'native-os-watcher' };
+}
+
+/**
+ * Format the watcher startup message that goes to stderr. Plan § 34.6
+ * step 2 specifies the user-facing copy.
+ *
+ * @param {{watcherEnabled:boolean, reason:string}} decision
+ * @returns {string}
+ */
+export function formatWatcherNotice(decision) {
+  return `[sweet-search] file watcher: ${decision.watcherEnabled ? 'on' : 'off'} (${decision.reason})`;
+}
+
+export const __testing = { detectInner, resetCache: () => { cached = null; } };

package/core/indexing/admission-policy.js

@@ -0,0 +1,139 @@
+/**
+ * Shared file-admission policy for full + incremental indexing.
+ *
+ * Full indexing (`discoverFiles` in indexer-utils.js) and incremental indexing
+ * (`dirty-scan` producer + `production-reconciler` consumer) MUST admit exactly
+ * the same files: a file full indexing would skip must never be newly indexed by
+ * incremental, and a file full indexing would admit must be eligible for
+ * incremental. This module is the single definition of that decision so the two
+ * paths cannot drift.
+ *
+ * A file is admitted iff ALL of:
+ *   1. include allowlist — matches a project `include` glob (minimatch)
+ *   2. NOT excluded      — `buildPathFilter` deny-list (default deny dirs/exts +
+ *                          project `exclude` globs + `.sweet-search-ignore`)
+ *   3. NOT oversized      — size ≤ project `maxFileSize`
+ *   4. NOT gitignored     — `git check-ignore` alignment (agentic paths exempt),
+ *                          only when the worktree is a git repo
+ *
+ * The exclude/deny component is delegated to `buildPathFilter` (incremental
+ * infra) so its rules — and its tests — stay the single source for "deny", and
+ * gitignore is delegated to `gitignore-filter` so it matches full indexing.
+ * This module only adds the include allowlist + size + the wiring.
+ *
+ * Shape checks (include + deny) are synchronous and I/O-free so producers can
+ * prune cheaply during a tree walk. Size is a single `stat`. Gitignore is async
+ * and batched (one `git check-ignore` per call) — never per-file.
+ */
+
+import path from 'node:path';
+import { statSync, existsSync } from 'node:fs';
+import { Minimatch } from 'minimatch';
+
+import { loadProjectConfig } from '../infrastructure/config/index.js';
+import { buildPathFilter } from '../incremental-indexing/infrastructure/path-filter.mjs';
+import { getGitIgnoredPathSet, isGitignoreAllowlistedAgenticPath, toPosixPath } from './gitignore-filter.js';
+
+const MM_OPTS = { dot: true, nocase: false };
+const DEFAULT_MAX_FILE_SIZE = 1 * 1024 * 1024;
+
+function normalizeRel(rel) {
+  return String(rel || '').replace(/\\/g, '/').replace(/^\.\//, '');
+}
+
+/**
+ * Build an admission policy bound to a project root.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.projectRoot]
+ * @param {object} [opts.config]              Pre-loaded loadProjectConfig() result.
+ * @param {boolean} [opts.allowSweetSearchDir] Lift the `.sweet-search` deny (daemon self-paths).
+ */
+export function createAdmissionPolicy({ projectRoot = process.cwd(), config, allowSweetSearchDir = false } = {}) {
+  const cfg = config || loadProjectConfig(projectRoot);
+  const includeGlobs = Array.isArray(cfg.include) ? cfg.include : [];
+  const excludeGlobs = Array.isArray(cfg.exclude) ? cfg.exclude : [];
+  const includeMatchers = includeGlobs.map((g) => new Minimatch(g, MM_OPTS));
+  const isDenied = buildPathFilter({ projectRoot, allowSweetSearchDir });
+  const maxFileSize = typeof cfg.maxFileSize === 'number' ? cfg.maxFileSize : DEFAULT_MAX_FILE_SIZE;
+  const respectGitignore = cfg.respectGitignore !== false;
+  const hasGit = existsSync(path.join(projectRoot, '.git'));
+
+  /** Include allowlist only (matches a project include glob). */
+  function matchesInclude(rel) {
+    const r = normalizeRel(rel);
+    if (!r) return false;
+    return includeMatchers.some((m) => m.match(r));
+  }
+
+  /** Deny-list only (true ⇒ excluded). Mirrors buildPathFilter; used for directory pruning. */
+  function isExcluded(rel) {
+    return isDenied(normalizeRel(rel));
+  }
+
+  /** Synchronous shape gate: include allowlist AND not excluded. No I/O. */
+  function admitsShape(rel) {
+    const r = normalizeRel(rel);
+    if (!r) return false;
+    return matchesInclude(r) && !isDenied(r);
+  }
+
+  /** True if the file at `absPath` exceeds maxFileSize. A stat error ⇒ true (treat as inadmissible, matching full indexing which drops un-statable files). */
+  function isOversizedAbs(absPath) {
+    try {
+      return statSync(absPath).size > maxFileSize;
+    } catch {
+      return true;
+    }
+  }
+
+  /**
+   * Batched gitignore: returns the subset of `rels` that git would ignore
+   * (posix-normalised). Empty when gitignore is disabled, the worktree is not a
+   * git repo, or git is unavailable — matching full indexing's fallback to
+   * "admit everything" rather than dropping files on a git failure.
+   */
+  async function gitignoredSet(rels, { silent = true } = {}) {
+    if (!respectGitignore || !hasGit) return new Set();
+    const candidates = [];
+    for (const rel of rels) {
+      const r = normalizeRel(rel);
+      if (!r || isGitignoreAllowlistedAgenticPath(r)) continue;
+      candidates.push(r);
+    }
+    if (candidates.length === 0) return new Set();
+    const ignored = await getGitIgnoredPathSet(candidates, { projectRoot, silent });
+    if (!ignored) return new Set();
+    const out = new Set();
+    for (const p of ignored) out.add(toPosixPath(p));
+    return out;
+  }
+
+  /**
+   * Convenience for batch discovery: drop gitignored paths from `rels`.
+   * Equivalent to full indexing's applyGitignoreAlignment over an already
+   * shape+size filtered list.
+   */
+  async function applyGitignore(rels, { silent = true } = {}) {
+    const list = [...rels];
+    const ignored = await gitignoredSet(list, { silent });
+    if (ignored.size === 0) return { files: list, gitignored: 0 };
+    const files = list.filter((rel) => !ignored.has(toPosixPath(normalizeRel(rel))));
+    return { files, gitignored: list.length - files.length };
+  }
+
+  return {
+    projectRoot,
+    includeGlobs,
+    excludeGlobs,
+    maxFileSize,
+    respectGitignore,
+    hasGit,
+    matchesInclude,
+    isExcluded,
+    admitsShape,
+    isOversizedAbs,
+    gitignoredSet,
+    applyGitignore,
+  };
+}

package/core/indexing/artifact-builder.js

@@ -56,6 +56,20 @@ import { BinaryHNSWIndex } from '../vector-store/binary-hnsw-index.js';
 import { truncateForHNSW, fisherYatesShuffle, normalizedFloatToInt8, floatToBinary } from '../infrastructure/quantization.js';
 import { FloatVectorStore, getFloatStorePath } from '../vector-store/float-vector-store.js';
 
+function hasVectorColumn(db, column) {
+  try {
+    return db.prepare('PRAGMA table_info(vectors)').all().some((col) => col.name === column);
+  } catch (_err) {
+    return false;
+  }
+}
+
+function liveVectorSql(db, alias = '') {
+  if (!hasVectorColumn(db, 'epoch_retired')) return '1=1';
+  const prefix = alias ? `${alias}.` : '';
+  return `${prefix}epoch_retired IS NULL`;
+}
+
 // =============================================================================
 // THRESHOLD CHECKING FUNCTIONS
 // =============================================================================
@@ -387,7 +401,8 @@ async function buildHnswIndexFromDb(db, options = {}) {
   } = options;
 
   const dimension = options.dimension || Math.ceil(floatDimension / 8);
-  const totalVectors = db.prepare('SELECT COUNT(*) as c FROM vectors').get().c;
+  const vectorWhere = liveVectorSql(db);
+  const totalVectors = db.prepare(`SELECT COUNT(*) as c FROM vectors WHERE ${vectorWhere}`).get().c;
 
   const index = new BinaryHNSWIndex({
     dimension,
@@ -408,19 +423,20 @@ async function buildHnswIndexFromDb(db, options = {}) {
     db.exec('CREATE TEMP TABLE IF NOT EXISTS artifact_order (pos INTEGER PRIMARY KEY, vector_rowid INTEGER)');
     db.exec('DELETE FROM artifact_order');
 
-    let indices = Array.from({ length: totalVectors }, (_, i) => i + 1); // 1-based rowids
+    const rowidRows = db.prepare(`SELECT rowid FROM vectors WHERE ${vectorWhere} ORDER BY rowid`).all();
+    let indices = rowidRows.map((row) => row.rowid);
 
     if (insertionOrder === 'shuffle') {
       fisherYatesShuffle(indices);
     } else if (insertionOrder === 'diversity') {
-      const filePaths = db.prepare('SELECT metadata FROM vectors ORDER BY rowid').all().map(r => {
-        try { return JSON.parse(r.metadata)?.file || '_unknown'; } catch (_e) { return '_unknown'; }
-      });
+      const rows = db.prepare(`SELECT rowid, metadata FROM vectors WHERE ${vectorWhere} ORDER BY rowid`).all();
       const buckets = new Map();
-      for (let i = 0; i < filePaths.length; i++) {
-        const dir = filePaths[i].replace(/\/[^/]+$/, '') || '_unknown';
+      for (const row of rows) {
+        let filePath = '_unknown';
+        try { filePath = JSON.parse(row.metadata)?.file || '_unknown'; } catch (_e) {}
+        const dir = filePath.replace(/\/[^/]+$/, '') || '_unknown';
         if (!buckets.has(dir)) buckets.set(dir, []);
-        buckets.get(dir).push(i + 1);
+        buckets.get(dir).push(row.rowid);
       }
       const dirs = [...buckets.keys()];
       fisherYatesShuffle(dirs);
@@ -448,7 +464,7 @@ async function buildHnswIndexFromDb(db, options = {}) {
       ORDER BY o.pos
     `);
   } else {
-    stmt = db.prepare('SELECT id, embedding, metadata FROM vectors ORDER BY rowid');
+    stmt = db.prepare(`SELECT id, embedding, metadata FROM vectors WHERE ${vectorWhere} ORDER BY rowid`);
   }
 
   const startTime = performance.now();
@@ -578,11 +594,12 @@ export async function saveArtifacts(hnswIndex) {
  */
 /** Build and save a FloatVectorStore by streaming from SQLite cursor. */
 async function buildAndSaveFloatStoreFromDb(db, floatDimension, floatStorePath) {
-  const totalVectors = db.prepare('SELECT COUNT(*) as c FROM vectors').get().c;
+  const vectorWhere = liveVectorSql(db);
+  const totalVectors = db.prepare(`SELECT COUNT(*) as c FROM vectors WHERE ${vectorWhere}`).get().c;
   console.log(`Building float vector store (${totalVectors} vectors, ${floatDimension}d)...`);
   const floatStore = new FloatVectorStore();
   const floatEntries = [];
-  const stmt = db.prepare('SELECT id, embedding FROM vectors ORDER BY rowid');
+  const stmt = db.prepare(`SELECT id, embedding FROM vectors WHERE ${vectorWhere} ORDER BY rowid`);
   for (const row of stmt.iterate()) {
     const embedding = new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.length / 4);
     floatEntries.push({
@@ -619,7 +636,7 @@ export async function buildFromCodebaseDb(codebaseDbPath = DB_PATHS.codebase, op
   const db = new Database(codebaseDbPath, insertionOrder === 'sequential' ? { readonly: true } : {});
   applyReadPragmas(db);
 
-  const totalVectors = db.prepare('SELECT COUNT(*) as c FROM vectors').get().c;
+  const totalVectors = db.prepare(`SELECT COUNT(*) as c FROM vectors WHERE ${liveVectorSql(db)}`).get().c;
 
   console.log(`Found ${totalVectors} vectors`);