sweet-search 2.5.2 → 2.5.4

package/core/incremental-indexing/infrastructure/artifact-temp-sweep.mjs

@@ -0,0 +1,163 @@
+/**
+ * Crash-orphan temp sweep for the reconcile state directory.
+ *
+ * Every per-tier writer in the incremental-indexing context stages its
+ * output to a sibling temp path and then `rename`s it into the canonical
+ * name:
+ *   - HNSW / Binary HNSW sidecars: `<name>.tmp.<pid>`
+ *   - manifest / merkle / metrics (production-reconciler, index-maintainer,
+ *     operator-cli, production-li-delta): `<name>.tmp.<pid>`
+ *   - reconcile manifest + LI segment manifest: `<name>.json.tmp`
+ *   - sparse-gram + LI segment compaction: `<name>.compacting.tmp`
+ *   - tombstone bitmaps: `<name>.bin.tmp`
+ *   - LI stub self-heal: `<name>.selfheal.tmp`
+ *
+ * The rename is atomic and the writers unlink their own temp on an
+ * in-process error, so under normal operation no temp survives a tick. A
+ * `SIGKILL` between stage and rename, however, leaves the temp orphaned —
+ * and because `*.tmp.<pid>` and sparse `*.compacting.tmp` carry
+ * per-process / per-epoch names, repeated crashes leak monotonically.
+ *
+ * Readers never consult these paths (they read canonical names plus the
+ * manifest-referenced delta/segment lists; `listDeltaSegments` and the LI
+ * segment manifest both ignore non-canonical suffixes), so an orphan is a
+ * disk-usage / operator-confusion problem, not a correctness one. This
+ * sweep runs once at daemon startup, AFTER the state lock is held (so the
+ * reconcile daemon is the single writer), and removes orphans older than a
+ * grace window. The grace window protects a temp another writer might still
+ * be mid-rename on — defensive belt-and-braces, since the lock already
+ * excludes a second reconcile daemon.
+ *
+ * Safety contract:
+ *   - Only files whose basename matches one of the reconcile staging-temp
+ *     suffixes above are ever removed. Canonical artifacts (`*.usearch`,
+ *     `*.meta.json`, `*.vectors.json`, `*.db`, `*.ssgrmdelta`, `*.sslx`,
+ *     `reconcile-manifest.json`, `*.idx`, `*.stale.bin`, `merkle-state.json`,
+ *     `*.jsonl`, …) never match.
+ *   - The cold-build full-artifact stages owned by the *indexer* context
+ *     (`*.db.tmp`, `*.idx.tmp`) are deliberately NOT matched — those are
+ *     fixed-name (overwritten on the next cold build, so they don't leak)
+ *     and may be a large in-flight build the reconcile daemon must not
+ *     touch.
+ *   - SQLite `-wal` / `-shm`, the dirty / processing / dead-letter queues,
+ *     the lockfile, and reader heartbeats are never matched and never
+ *     removed.
+ *   - Directories are never removed; the LI self-heal owns `*.tmp.segments`
+ *     directories, which this sweep refuses to recurse into.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const DEFAULT_TMP_SWEEP_MAX_AGE_MS = 60_000;
+
+// pid-suffixed staging temps: `foo.tmp.12345`
+const PID_TMP_RE = /\.tmp\.\d+$/;
+
+// Reconcile / maintenance staging-temp suffixes (explicit allowlist — see the
+// module header for why this is an allowlist and not a `*.tmp` catch-all).
+const ORPHAN_TEMP_SUFFIXES = [
+  '.compacting.tmp', // sparse-gram + LI segment compaction
+  '.selfheal.tmp',   // LI stub self-heal
+  '.json.tmp',       // reconcile manifest + LI segment manifest
+  '.bin.tmp',        // tombstone bitmaps
+];
+
+/**
+ * True when a basename is a reconcile/maintenance crash-orphan staging temp
+ * (and therefore safe to remove), false for canonical artifacts and
+ * cold-build full-artifact stages.
+ *
+ * @param {string} name
+ * @returns {boolean}
+ */
+export function isOrphanTempName(name) {
+  if (typeof name !== 'string' || name.length === 0) return false;
+  if (PID_TMP_RE.test(name)) return true;
+  for (const suffix of ORPHAN_TEMP_SUFFIXES) {
+    if (name.endsWith(suffix)) return true;
+  }
+  return false;
+}
+
+/**
+ * True for the canonical artifact subdirs that hold compaction temps
+ * (sparse-gram `*.deltas`, LI `*.segments`). Orphan / self-heal directories
+ * (those containing `.tmp.`) are skipped so the sweep never recurses into a
+ * `*.tmp.segments` directory the LI self-heal is responsible for migrating.
+ *
+ * @param {string} name
+ * @returns {boolean}
+ */
+export function isScannableArtifactSubdir(name) {
+  if (typeof name !== 'string') return false;
+  if (name.includes('.tmp.')) return false;
+  return name.endsWith('.deltas') || name.endsWith('.segments');
+}
+
+function sweepDir(dir, ctx) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    if (!entry.isFile()) continue;
+    if (!isOrphanTempName(entry.name)) continue;
+    const full = path.join(dir, entry.name);
+    ctx.summary.scanned += 1;
+    let stat;
+    try { stat = fs.statSync(full); } catch { continue; }
+    // Clamp to >= 0: `mtimeMs` is a sub-millisecond float while `now` is an
+    // integer, so a file written microseconds ago can read as "in the
+    // future" and produce a spuriously negative age.
+    const ageMs = Math.max(0, ctx.now - stat.mtimeMs);
+    if (ageMs < ctx.maxAgeMs) {
+      ctx.summary.skippedRecent += 1;
+      continue;
+    }
+    try {
+      fs.unlinkSync(full);
+      ctx.summary.removed += 1;
+      ctx.summary.bytesReclaimed += stat.size;
+      ctx.summary.removedPaths.push(full);
+    } catch {
+      // Tolerate races / permission issues — best-effort cleanup.
+    }
+  }
+}
+
+/**
+ * Remove crash-orphaned reconcile staging temps from the state directory.
+ * Scans the top level plus immediate `*.deltas` / `*.segments` artifact
+ * subdirs. Pure of environment access — callers thread `maxAgeMs` / `now`.
+ *
+ * @param {string} stateDir
+ * @param {{maxAgeMs?:number, now?:number}} [opts]
+ * @returns {{scanned:number, removed:number, skippedRecent:number, bytesReclaimed:number, removedPaths:string[]}}
+ */
+export function sweepStaleArtifactTemps(stateDir, opts = {}) {
+  const summary = { scanned: 0, removed: 0, skippedRecent: 0, bytesReclaimed: 0, removedPaths: [] };
+  if (!stateDir) return summary;
+  const maxAgeMs = Number.isFinite(opts.maxAgeMs) && opts.maxAgeMs >= 0
+    ? opts.maxAgeMs
+    : DEFAULT_TMP_SWEEP_MAX_AGE_MS;
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const ctx = { maxAgeMs, now, summary };
+
+  let topEntries;
+  try {
+    topEntries = fs.readdirSync(stateDir, { withFileTypes: true });
+  } catch {
+    return summary;
+  }
+
+  sweepDir(stateDir, ctx);
+  for (const entry of topEntries) {
+    if (!entry.isDirectory()) continue;
+    if (!isScannableArtifactSubdir(entry.name)) continue;
+    sweepDir(path.join(stateDir, entry.name), ctx);
+  }
+  return summary;
+}

package/core/incremental-indexing/infrastructure/baseline-readiness.mjs

@@ -0,0 +1,121 @@
+/**
+ * Baseline-readiness gate for the default-on incremental maintainer.
+ *
+ * Product contract: the incremental reconciler must NEVER be the first index
+ * builder for a non-empty repo. The first index must come from the normal full
+ * indexing path (`sweet-search index`). Before a complete baseline exists, the
+ * maintainer stays dormant and reports `waiting_for_initial_index`; once a
+ * complete baseline exists (including a valid empty one) reconcile runs normally.
+ *
+ * Why this is needed: the daemon's tick is a producer (`dirty-scan.mjs` diffs the
+ * tree against `merkle-state.json`) plus a consumer (`production-reconciler.mjs`,
+ * whose adapters call `createVectorSchema`/`createGraphSchema`). With no baseline,
+ * `merkle-state.json` is absent so the producer enqueues the WHOLE tree, and the
+ * consumer then builds `codebase.db` / `code-graph.db` / HNSW / LI / sparse from
+ * scratch one budget-bounded tick at a time — leaving a PARTIAL index that search
+ * mistakes for a complete one.
+ *
+ * "Complete baseline" is proven by what the FULL indexer writes in its final phase
+ * (`indexing/indexer-phases.js::updateIncrementalStatePhase`), which the
+ * incremental reconciler does NOT produce on its own:
+ *
+ *   1. `reconcile-manifest.json` published at `epoch >= 1`. The full indexer
+ *      publishes this as its LAST step, so its presence means vectors + graph +
+ *      HNSW + LI + sparse all finished building. A crash before that step leaves
+ *      no manifest; a corrupt manifest reads back as null. (epoch alone is NOT a
+ *      discriminator: a reconciler-only first tick also yields epoch 1.)
+ *   2. `merkle-state.json` carrying a `config_fingerprint`. ONLY the full
+ *      indexer's tracker (`indexing/incremental-tracker.js::updateState`) writes
+ *      this field; the reconciler's `persistManifest` never adds it (it only
+ *      preserves one already present). So `config_fingerprint` present ⟺ a full
+ *      index ran at least once — the exact signal that distinguishes a real
+ *      baseline from the reconciler-only partial state the old bug produced.
+ *   3. The vectors DB named by the manifest exists on disk (the artifact search
+ *      reads). Guards a manually-deleted / half-written baseline.
+ *
+ * A valid EMPTY baseline (a full index that produced an empty-but-valid index)
+ * satisfies 1-3 with zero tracked files, so it counts as ready. A
+ * partially-written, corrupt, or reconciler-only baseline fails 1 or 2 and does
+ * not. The check is read-only: it never mutates the state dir.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { readManifest } from './manifest.mjs';
+
+/** Status label surfaced in logs and `reconcile status` when no baseline exists. */
+export const WAITING_FOR_INITIAL_INDEX = 'waiting_for_initial_index';
+
+const MERKLE_STATE = 'merkle-state.json';
+const DEFAULT_VECTORS_DB = 'codebase.db';
+
+function readJsonSafe(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Does the merkle state carry a `config_fingerprint`? The full indexer writes a
+ * populated object; the reconciler never adds one. Accept either a non-empty
+ * object or a non-empty string for forward/backward tolerance.
+ */
+function hasConfigFingerprint(merkle) {
+  const fp = merkle ? merkle.config_fingerprint : null;
+  if (!fp) return false;
+  if (typeof fp === 'string') return fp.length > 0;
+  if (typeof fp === 'object') return Object.keys(fp).length > 0;
+  return false;
+}
+
+/**
+ * Whether a complete baseline index exists for `stateDir`.
+ *
+ * @param {string} stateDir   The `.sweet-search` directory.
+ * @returns {{ready: boolean, reason: string}}
+ *   `reason` is one of: `ready`, `no-state-dir`, `no-manifest`,
+ *   `manifest-epoch-zero`, `no-merkle-state`, `no-config-fingerprint`,
+ *   `missing-vectors-db`.
+ */
+export function hasCompleteBaseIndex(stateDir) {
+  if (!stateDir || !fs.existsSync(stateDir)) {
+    return { ready: false, reason: 'no-state-dir' };
+  }
+
+  const manifest = readManifest(stateDir);
+  if (!manifest) {
+    return { ready: false, reason: 'no-manifest' };
+  }
+  if (!Number.isInteger(manifest.epoch) || manifest.epoch < 1) {
+    return { ready: false, reason: 'manifest-epoch-zero' };
+  }
+
+  const merkle = readJsonSafe(path.join(stateDir, MERKLE_STATE));
+  if (!merkle) {
+    return { ready: false, reason: 'no-merkle-state' };
+  }
+  if (!hasConfigFingerprint(merkle)) {
+    return { ready: false, reason: 'no-config-fingerprint' };
+  }
+
+  const vectorsRel = (manifest.vectors && manifest.vectors.path) || DEFAULT_VECTORS_DB;
+  const vectorsPath = path.isAbsolute(vectorsRel) ? vectorsRel : path.join(stateDir, vectorsRel);
+  if (!fs.existsSync(vectorsPath)) {
+    return { ready: false, reason: 'missing-vectors-db' };
+  }
+
+  return { ready: true, reason: 'ready' };
+}
+
+/**
+ * Readiness plus a human/machine-facing `state` label for status surfaces.
+ *
+ * @param {string} stateDir
+ * @returns {{ready: boolean, reason: string, state: 'indexed'|'waiting_for_initial_index'}}
+ */
+export function baselineStatus(stateDir) {
+  const result = hasCompleteBaseIndex(stateDir);
+  return { ...result, state: result.ready ? 'indexed' : WAITING_FOR_INITIAL_INDEX };
+}

package/core/incremental-indexing/infrastructure/dirty-set.mjs

@@ -0,0 +1,233 @@
+/**
+ * In-memory dirty path set.
+ *
+ * Plan § 6.1, § 9.1-§ 9.5. The watcher and polling backstop both push
+ * paths into this set; the reconcile tick drains it at every tick start.
+ *
+ * Guarantees:
+ *   - Paths are normalised to forward-slash form (cross-platform).
+ *   - Duplicate inserts are coalesced (Set semantics).
+ *   - Insertion order is preserved on drain for deterministic tests.
+ *   - A bounded-size policy guards against burst overflow (50 k events
+ *     from `git checkout` in <1 s — plan § 11). Past the cap, the set
+ *     keeps the most recent entries and emits a `dropped` count via the
+ *     callback so the next polling backstop sweep can re-discover them.
+ *
+ * The set has no global state. Path canonicalisation uses filesystem
+ * realpaths when an ancestor exists so watcher events cannot enter the
+ * dirty set through symlink escapes.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const DEFAULT_MAX = 100_000;
+
+function normalise(p) {
+  if (typeof p !== 'string') return null;
+  return p.replace(/\\/g, '/');
+}
+
+export class DirtySet {
+  /**
+   * @param {object} [options]
+   * @param {number} [options.maxSize]            Hard cap on entries.
+   * @param {(payload:{dropped:number})=>void} [options.onOverflow]
+   */
+  constructor({ maxSize = DEFAULT_MAX, onOverflow } = {}) {
+    this._set = new Map(); // path → { addedAt, source, meta }
+    this._maxSize = maxSize;
+    this._onOverflow = onOverflow;
+    this._totalEnqueued = 0;
+    this._totalDropped = 0;
+  }
+
+  get size() {
+    return this._set.size;
+  }
+
+  get maxSize() {
+    return this._maxSize;
+  }
+
+  /**
+   * Enqueue a path. `source` is one of:
+   *   - 'watcher'   - notify / FSEvents / inotify
+   *   - 'polling'   - mtime backstop sweep
+   *   - 'cli'       - explicit `sweet-search index --add <path>` hint
+   *   - 'queue'     - drained from index-maintainer-queue.jsonl
+   *
+   * @param {string} filePath
+   * @param {string} [source='watcher']
+   * @param {object} [meta]
+   * @returns {boolean}   true if newly inserted (or refreshed), false on drop.
+   */
+  add(filePath, source = 'watcher', meta) {
+    const p = normalise(filePath);
+    if (!p) return false;
+    this._totalEnqueued += 1;
+    if (this._set.has(p)) {
+      const entry = this._set.get(p);
+      entry.lastSource = source;
+      entry.lastSeenAt = Date.now();
+      if (meta) entry.meta = { ...entry.meta, ...meta };
+      return true;
+    }
+    if (this._set.size >= this._maxSize) {
+      // Drop the oldest entry to keep the most recent — those reflect the
+      // current edit pattern best. The dropped path will be re-discovered
+      // by the next polling backstop sweep.
+      const firstKey = this._set.keys().next().value;
+      if (firstKey !== undefined) {
+        this._set.delete(firstKey);
+        this._totalDropped += 1;
+        if (this._onOverflow) this._onOverflow({ dropped: 1, droppedTotal: this._totalDropped });
+      }
+    }
+    this._set.set(p, {
+      addedAt: Date.now(),
+      lastSeenAt: Date.now(),
+      firstSource: source,
+      lastSource: source,
+      meta: meta || null,
+    });
+    return true;
+  }
+
+  /**
+   * Bulk-add an iterable of paths. Useful when polling identifies a
+   * batch of dirty paths in one syscall pass.
+   *
+   * @param {Iterable<string>} paths
+   * @param {string} [source]
+   */
+  addMany(paths, source = 'polling') {
+    let added = 0;
+    for (const p of paths) {
+      if (this.add(p, source)) added += 1;
+    }
+    return added;
+  }
+
+  has(filePath) {
+    return this._set.has(normalise(filePath));
+  }
+
+  /**
+   * Drain the set into a sorted array. Returns the snapshot the caller
+   * should process this tick; subsequent inserts go into the next tick.
+   *
+   * @returns {Array<{path:string, firstSource:string, lastSource:string, addedAt:number, lastSeenAt:number, meta:object|null}>}
+   */
+  drain() {
+    const out = [];
+    for (const [p, entry] of this._set.entries()) {
+      out.push({ path: p, ...entry });
+    }
+    this._set.clear();
+    return out;
+  }
+
+  /**
+   * Peek without draining — primarily for tests / debug.
+   *
+   * @returns {string[]}
+   */
+  peek() {
+    return Array.from(this._set.keys());
+  }
+
+  /**
+   * Remove a single path without draining the rest.
+   *
+   * @param {string} filePath
+   */
+  remove(filePath) {
+    return this._set.delete(normalise(filePath));
+  }
+
+  /**
+   * Diagnostic counters for the operator dashboard (plan § 20.2).
+   *
+   * @returns {{size:number, maxSize:number, totalEnqueued:number, totalDropped:number}}
+   */
+  stats() {
+    return {
+      size: this._set.size,
+      maxSize: this._maxSize,
+      totalEnqueued: this._totalEnqueued,
+      totalDropped: this._totalDropped,
+    };
+  }
+}
+
+/**
+ * Resolve a path to its canonical absolute form within a project root.
+ * Plan § 22.1 / § 22.4: canonicalise to drop case-insensitive collisions
+ * and to anchor paths inside the indexed tree. The lexical check catches
+ * `../` traversal; the realpath check catches symlink parents that point
+ * outside the worktree while still allowing delete events for missing
+ * files under a real in-tree parent.
+ *
+ * @param {string} projectRoot
+ * @param {string} relativeOrAbsolute
+ * @returns {string|null}     null if the path escapes projectRoot.
+ */
+export function canonicaliseInsideRoot(projectRoot, relativeOrAbsolute) {
+  if (typeof relativeOrAbsolute !== 'string') return null;
+  const absoluteInput = path.isAbsolute(relativeOrAbsolute);
+  const abs = absoluteInput
+    ? relativeOrAbsolute
+    : path.resolve(projectRoot, relativeOrAbsolute);
+  const resolvedAbs = path.resolve(abs);
+  const resolvedRoot = path.resolve(projectRoot);
+  if (!absoluteInput && !isInsidePath(resolvedRoot, resolvedAbs)) return null;
+
+  const rootReal = realpathOrNull(resolvedRoot);
+  if (!rootReal) return null;
+
+  const existing = nearestExistingAncestor(
+    resolvedAbs,
+    absoluteInput ? path.parse(resolvedAbs).root : resolvedRoot,
+  );
+  if (!existing) return null;
+  const existingReal = realpathOrNull(existing.path);
+  if (!existingReal) return null;
+
+  const materializedReal = existing.rest
+    ? path.join(existingReal, existing.rest)
+    : existingReal;
+  if (!isInsidePath(rootReal, materializedReal)) return null;
+
+  return materializedReal.replace(/\\/g, '/');
+}
+
+function isInsidePath(root, candidate) {
+  const rel = path.relative(root, candidate);
+  return rel === '' || (rel && !rel.startsWith('..') && !path.isAbsolute(rel));
+}
+
+function realpathOrNull(filePath) {
+  try {
+    return fs.realpathSync.native(filePath);
+  } catch {
+    return null;
+  }
+}
+
+function nearestExistingAncestor(absPath, root) {
+  let current = absPath;
+  const rest = [];
+  while (isInsidePath(root, current)) {
+    try {
+      fs.lstatSync(current);
+      return { path: current, rest: rest.join(path.sep) };
+    } catch (err) {
+      if (err?.code !== 'ENOENT' && err?.code !== 'ENOTDIR') return null;
+      if (current === root) return null;
+      rest.unshift(path.basename(current));
+      current = path.dirname(current);
+    }
+  }
+  return null;
+}