sweet-search 2.5.2 → 2.5.3

package/core/indexing/maintainer-launcher.mjs

@@ -0,0 +1,137 @@
+/**
+ * Shared, idempotent launcher for the reconcile-v2 incremental-index maintainer.
+ *
+ * This is the single place that knows how to start the maintainer daemon. It is
+ * called from every entry point that should keep the index fresh:
+ *   - the warm search-server startup (`core/search/search-server.js`) — the
+ *     durable, non-MCP guarantee that any normal `sweet-search` use starts the
+ *     maintainer;
+ *   - the Claude/Codex SessionStart prewarm hook (best-effort convenience);
+ *   - the MCP server startup, *only* when MCP is actually enabled/used.
+ *
+ * Design contract (all required by the durable-startup spec):
+ *   - respects the default-on opt-out (`SWEET_SEARCH_RECONCILE_V2=0|false|off`)
+ *     via the canonical `reconcileEnablement` policy;
+ *   - skips when the project has no `.sweet-search` state dir (nothing to
+ *     maintain yet);
+ *   - skips when a *live* maintainer already holds `index-maintainer.lock`;
+ *   - relies on the daemon's own `O_EXCL` state lock as the HARD no-duplicate
+ *     guarantee — the liveness probe here is only an optimization so we don't
+ *     spawn a process that would immediately exit;
+ *   - spawns fully detached with the right cwd and a pinned
+ *     `SWEET_SEARCH_PROJECT_ROOT` (the maintainer's package copy resolves its
+ *     own PROJECT_ROOT from `__dirname`, so the env pin is load-bearing);
+ *   - returns quickly (a few `fs` stats + a detached spawn);
+ *   - is stdout-clean — it NEVER writes to stdout, only to stderr when verbose,
+ *     so machine-readable commands that call it stay parseable;
+ *   - is safe to call often (repeated calls are cheap no-ops once a maintainer
+ *     is up).
+ */
+
+import { spawn } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { reconcileEnablement } from '../incremental-indexing/domain/interval-autotune.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export const MAINTAINER_LOCK_FILENAME = 'index-maintainer.lock';
+
+/** Default maintainer entry: the sibling daemon in this same context. */
+export function defaultMaintainerEntry() {
+  return join(__dirname, 'index-maintainer.mjs');
+}
+
+/** Does a process with this pid exist right now? EPERM (alien owner) = alive. */
+function pidAlive(pid) {
+  if (!Number.isFinite(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return err.code === 'EPERM';
+  }
+}
+
+/** Resolve the reconcile state dir the same way the daemon's v2 context does. */
+export function resolveStateDir(env = process.env, cwd = process.cwd()) {
+  if (env.SWEET_SEARCH_STATE_DIR) return env.SWEET_SEARCH_STATE_DIR;
+  const root = env.SWEET_SEARCH_PROJECT_ROOT || cwd;
+  return join(root, '.sweet-search');
+}
+
+/** True when a live maintainer already holds the lock for this state dir. */
+export function maintainerAlive(stateDir) {
+  const lockFile = join(stateDir, MAINTAINER_LOCK_FILENAME);
+  if (!existsSync(lockFile)) return false;
+  try {
+    const { pid } = JSON.parse(readFileSync(lockFile, 'utf-8'));
+    return pidAlive(Number(pid));
+  } catch {
+    // Unreadable/corrupt lock → treat as not-alive; the daemon's O_EXCL
+    // acquire + stale-lock reclaim handle the real arbitration on spawn.
+    return false;
+  }
+}
+
+/**
+ * Start the maintainer if it should run and isn't already running.
+ *
+ * @param {{
+ *   env?: NodeJS.ProcessEnv,
+ *   cwd?: string,
+ *   verbose?: boolean,
+ *   maintainerEntry?: string,
+ *   log?: (msg: string) => void,
+ * }} [options]
+ * @returns {{spawned: boolean, reason: 'opted-out'|'entry-missing'|'no-state-dir'|'already-running'|'spawned'|'error', pid?: number, stateDir?: string, error?: string}}
+ */
+export function launchMaintainer(options = {}) {
+  const env = options.env || process.env;
+  const cwd = options.cwd || process.cwd();
+  const verbose = options.verbose ?? !!env.SWEET_SEARCH_PREWARM_VERBOSE;
+  // stdout-clean: log only to stderr, only when verbose.
+  const log = options.log
+    || ((msg) => { if (verbose) process.stderr.write(`[sweet-search maintainer-launch] ${msg}\n`); });
+
+  const maintainerEntry = options.maintainerEntry
+    || env.SWEET_SEARCH_MAINTAINER_ENTRY
+    || defaultMaintainerEntry();
+
+  if (!reconcileEnablement(env).enabled) {
+    log('disabled via SWEET_SEARCH_RECONCILE_V2 opt-out');
+    return { spawned: false, reason: 'opted-out' };
+  }
+  if (!existsSync(maintainerEntry)) {
+    log(`maintainer entry missing: ${maintainerEntry}`);
+    return { spawned: false, reason: 'entry-missing' };
+  }
+  const stateDir = resolveStateDir(env, cwd);
+  if (!existsSync(stateDir)) {
+    log(`no index state dir (${stateDir}); skipping (run sweet-search index first)`);
+    return { spawned: false, reason: 'no-state-dir', stateDir };
+  }
+  if (maintainerAlive(stateDir)) {
+    log('maintainer already running for this state dir');
+    return { spawned: false, reason: 'already-running', stateDir };
+  }
+
+  try {
+    const child = spawn(process.execPath, [maintainerEntry], {
+      detached: true,
+      stdio: 'ignore',
+      cwd,
+      env: {
+        ...env,
+        SWEET_SEARCH_PROJECT_ROOT: env.SWEET_SEARCH_PROJECT_ROOT || cwd,
+      },
+    });
+    child.unref();
+    log(`maintainer spawned (pid ${child.pid}, detached)`);
+    return { spawned: true, reason: 'spawned', pid: child.pid, stateDir };
+  } catch (err) {
+    log(`maintainer spawn failed (non-fatal): ${err?.message || err}`);
+    return { spawned: false, reason: 'error', stateDir, error: err?.message || String(err) };
+  }
+}

package/core/indexing/merkle-tracker.js

@@ -1,244 +1,251 @@
-#!/usr/bin/env node
-
-/**
- * Merkle Tree-Based File Tracking for Incremental Indexing
- *
- * Tracks file content hashes and chunk IDs to enable efficient change detection
- * and selective re-indexing. Supports atomic updates and persistent state.
- */
-
-import { readFile, writeFile, mkdir } from 'fs/promises';
-import { createHash } from 'crypto';
-import { dirname, resolve } from 'path';
-import { existsSync } from 'fs';
-
-export class MerkleTracker {
-  constructor(statePath = '.sweet-search/merkle/codebase-state.json') {
-    this.statePath = resolve(statePath);
-    this.state = {
-      version: 1,
-      lastFullIndex: null,
-      lastIncrementalIndex: null,
-      stats: {
-        totalFiles: 0,
-        totalChunks: 0,
-        lastDuration: 0
-      },
-      files: {}
-    };
-  }
-
-  async load() {
-    try {
-      if (existsSync(this.statePath)) {
-        const content = await readFile(this.statePath, 'utf8');
-        const loaded = JSON.parse(content);
-
-        if (loaded.version !== 1) {
-          console.warn(`Unknown state version ${loaded.version}, starting fresh`);
-          return;
-        }
-
-        this.state = loaded;
-        console.log(`Loaded tracking state: ${this.state.stats.totalFiles} files, ${this.state.stats.totalChunks} chunks`);
-      } else {
-        console.log('No existing state found, starting fresh');
-      }
-    } catch (error) {
-      console.error('Failed to load tracking state:', error.message);
-      console.log('Starting with fresh state');
-    }
-  }
-
-  async save() {
-    try {
-      const dir = dirname(this.statePath);
-      await mkdir(dir, { recursive: true });
-
-      const tmpPath = `${this.statePath}.tmp`;
-      const content = JSON.stringify(this.state, null, 2);
-      await writeFile(tmpPath, content, 'utf8');
-      await writeFile(this.statePath, content, 'utf8');
-
-      try {
-        const { unlink } = await import('fs/promises');
-        if (existsSync(tmpPath)) {
-          await unlink(tmpPath);
-        }
-      } catch (e) {}
-
-      console.log(`Saved tracking state: ${this.state.stats.totalFiles} files, ${this.state.stats.totalChunks} chunks`);
-    } catch (error) {
-      console.error('Failed to save tracking state:', error.message);
-      throw error;
-    }
-  }
-
-  needsReindex(filePath, contentHash) {
-    const tracked = this.state.files[filePath];
-    if (!tracked) return true;
-    if (tracked.contentHash !== contentHash) return true;
-    return false;
-  }
-
-  async findChangedFiles(allFiles) {
-    const changed = [];
-    for (const filePath of allFiles) {
-      try {
-        const content = await readFile(filePath, 'utf8');
-        const hash = MerkleTracker.computeHash(content);
-        if (this.needsReindex(filePath, hash)) {
-          changed.push(filePath);
-        }
-      } catch (err) {
-        changed.push(filePath);
-      }
-    }
-    return changed;
-  }
-
-  getChunkIds(filePath) {
-    const tracked = this.state.files[filePath];
-    return tracked ? tracked.chunkIds : [];
-  }
-
-  async updateFile(filePath, contentHash = null, chunkIds = [], size = 0) {
-    const now = new Date().toISOString();
-
-    if (!contentHash) {
-      try {
-        const content = await readFile(filePath, 'utf8');
-        contentHash = MerkleTracker.computeHash(content);
-        size = content.length;
-      } catch (err) {
-        contentHash = 'unknown';
-      }
-    }
-
-    this.state.files[filePath] = {
-      contentHash,
-      lastIndexed: now,
-      chunkIds,
-      chunkCount: chunkIds.length,
-      size
-    };
-
-    this.state.lastIncrementalIndex = now;
-    this._recalculateStats();
-  }
-
-  removeFile(filePath) {
-    const chunkIds = this.getChunkIds(filePath);
-    delete this.state.files[filePath];
-    this._recalculateStats();
-    return chunkIds;
-  }
-
-  getTrackedFiles() {
-    return Object.keys(this.state.files);
-  }
-
-  getStats() {
-    return { ...this.state.stats };
-  }
-
-  getSummary() {
-    const fileCount = Object.keys(this.state.files).length;
-    const totalChunks = Object.values(this.state.files).reduce(
-      (sum, file) => sum + file.chunkCount,
-      0
-    );
-    const totalSize = Object.values(this.state.files).reduce(
-      (sum, file) => sum + (file.size || 0),
-      0
-    );
-
-    return {
-      fileCount,
-      totalChunks,
-      totalSize,
-      lastFullIndex: this.state.lastFullIndex,
-      lastIncrementalIndex: this.state.lastIncrementalIndex,
-      lastDuration: this.state.stats.lastDuration
-    };
-  }
-
-  clear() {
-    this.state = {
-      version: 1,
-      lastFullIndex: null,
-      lastIncrementalIndex: null,
-      stats: {
-        totalFiles: 0,
-        totalChunks: 0,
-        lastDuration: 0
-      },
-      files: {}
-    };
-  }
-
-  _recalculateStats() {
-    const files = Object.values(this.state.files);
-    this.state.stats.totalFiles = files.length;
-    this.state.stats.totalChunks = files.reduce(
-      (sum, file) => sum + file.chunkCount,
-      0
-    );
-  }
-
-  static computeHash(content) {
-    return createHash('sha256')
-      .update(content)
-      .digest('hex')
-      .slice(0, 16);
-  }
-
-  static async computeFileHash(filePath) {
-    const content = await readFile(filePath, 'utf8');
-    return MerkleTracker.computeHash(content);
-  }
-}
-
-// CLI Interface
-if (import.meta.url === `file://${process.argv[1]}`) {
-  const tracker = new MerkleTracker();
-  await tracker.load();
-
-  const command = process.argv[2];
-
-  switch (command) {
-    case 'stats':
-      console.log(JSON.stringify(tracker.getSummary(), null, 2));
-      break;
-
-    case 'list':
-      console.log(tracker.getTrackedFiles().join('\n'));
-      break;
-
-    case 'clear':
-      tracker.clear();
-      await tracker.save();
-      console.log('Tracking state cleared');
-      break;
-
-    case 'inspect':
-      if (!process.argv[3]) {
-        console.error('Usage: merkle-tracker.js inspect <file-path>');
-        process.exit(1);
-      }
-      const metadata = tracker.state.files[process.argv[3]];
-      console.log(JSON.stringify(metadata, null, 2));
-      break;
-
-    default:
-      console.log(`
-Merkle File Tracker CLI
-
-Usage:
-  merkle-tracker.js stats              Show tracking statistics
-  merkle-tracker.js list               List all tracked files
-  merkle-tracker.js inspect <file>     Show metadata for a file
-  merkle-tracker.js clear              Clear all tracking state
-`);
-  }
-}
+#!/usr/bin/env node
+
+/**
+ * Merkle Tree-Based File Tracking for Incremental Indexing
+ *
+ * Tracks file content hashes and chunk IDs to enable efficient change detection
+ * and selective re-indexing. Supports atomic updates and persistent state.
+ */
+
+import { readFile, mkdir, open, rename, unlink } from 'fs/promises';
+import { dirname, resolve } from 'path';
+import { existsSync } from 'fs';
+import { contentHashSync, HASH_ALGORITHM } from '../incremental-indexing/infrastructure/hashing.mjs';
+
+export class MerkleTracker {
+  constructor(statePath = '.sweet-search/merkle/codebase-state.json') {
+    this.statePath = resolve(statePath);
+    this.state = {
+      version: 2,
+      hashAlgorithm: HASH_ALGORITHM,
+      lastFullIndex: null,
+      lastIncrementalIndex: null,
+      stats: {
+        totalFiles: 0,
+        totalChunks: 0,
+        lastDuration: 0
+      },
+      files: {}
+    };
+  }
+
+  async load() {
+    try {
+      if (existsSync(this.statePath)) {
+        const content = await readFile(this.statePath, 'utf8');
+        const loaded = JSON.parse(content);
+
+        if (loaded.version !== 2 || loaded.hashAlgorithm !== HASH_ALGORITHM) {
+          console.warn(`Unknown state version ${loaded.version}, starting fresh`);
+          return;
+        }
+
+        this.state = loaded;
+        console.log(`Loaded tracking state: ${this.state.stats.totalFiles} files, ${this.state.stats.totalChunks} chunks`);
+      } else {
+        console.log('No existing state found, starting fresh');
+      }
+    } catch (error) {
+      console.error('Failed to load tracking state:', error.message);
+      console.log('Starting with fresh state');
+    }
+  }
+
+  async save() {
+    try {
+      const dir = dirname(this.statePath);
+      await mkdir(dir, { recursive: true });
+
+      const tmpPath = `${this.statePath}.tmp`;
+      const content = JSON.stringify(this.state, null, 2);
+      let handle = await open(tmpPath, 'w');
+      try {
+        await handle.writeFile(content, 'utf8');
+        await handle.sync();
+      } finally {
+        await handle.close();
+      }
+      await rename(tmpPath, this.statePath);
+      try {
+        handle = await open(dir, 'r');
+        try { await handle.sync(); } finally { await handle.close(); }
+      } catch {
+        // Some container/tmpfs filesystems reject directory fsync.
+      }
+
+      console.log(`Saved tracking state: ${this.state.stats.totalFiles} files, ${this.state.stats.totalChunks} chunks`);
+    } catch (error) {
+      try {
+        await unlink(`${this.statePath}.tmp`);
+      } catch {}
+      console.error('Failed to save tracking state:', error.message);
+      throw error;
+    }
+  }
+
+  needsReindex(filePath, contentHash) {
+    const tracked = this.state.files[filePath];
+    if (!tracked) return true;
+    if (tracked.contentHash !== contentHash) return true;
+    return false;
+  }
+
+  async findChangedFiles(allFiles) {
+    const changed = [];
+    for (const filePath of allFiles) {
+      try {
+        const content = await readFile(filePath);
+        const hash = MerkleTracker.computeHash(content);
+        if (this.needsReindex(filePath, hash)) {
+          changed.push(filePath);
+        }
+      } catch (err) {
+        changed.push(filePath);
+      }
+    }
+    return changed;
+  }
+
+  getChunkIds(filePath) {
+    const tracked = this.state.files[filePath];
+    return tracked ? tracked.chunkIds : [];
+  }
+
+  async updateFile(filePath, contentHash = null, chunkIds = [], size = 0) {
+    const now = new Date().toISOString();
+
+    if (!contentHash) {
+      try {
+        const content = await readFile(filePath);
+        contentHash = MerkleTracker.computeHash(content);
+        size = content.length;
+      } catch (err) {
+        contentHash = 'unknown';
+      }
+    }
+
+    this.state.files[filePath] = {
+      contentHash,
+      lastIndexed: now,
+      chunkIds,
+      chunkCount: chunkIds.length,
+      size
+    };
+
+    this.state.lastIncrementalIndex = now;
+    this._recalculateStats();
+  }
+
+  removeFile(filePath) {
+    const chunkIds = this.getChunkIds(filePath);
+    delete this.state.files[filePath];
+    this._recalculateStats();
+    return chunkIds;
+  }
+
+  getTrackedFiles() {
+    return Object.keys(this.state.files);
+  }
+
+  getStats() {
+    return { ...this.state.stats };
+  }
+
+  getSummary() {
+    const fileCount = Object.keys(this.state.files).length;
+    const totalChunks = Object.values(this.state.files).reduce(
+      (sum, file) => sum + file.chunkCount,
+      0
+    );
+    const totalSize = Object.values(this.state.files).reduce(
+      (sum, file) => sum + (file.size || 0),
+      0
+    );
+
+    return {
+      fileCount,
+      totalChunks,
+      totalSize,
+      lastFullIndex: this.state.lastFullIndex,
+      lastIncrementalIndex: this.state.lastIncrementalIndex,
+      lastDuration: this.state.stats.lastDuration
+    };
+  }
+
+  clear() {
+    this.state = {
+      version: 2,
+      hashAlgorithm: HASH_ALGORITHM,
+      lastFullIndex: null,
+      lastIncrementalIndex: null,
+      stats: {
+        totalFiles: 0,
+        totalChunks: 0,
+        lastDuration: 0
+      },
+      files: {}
+    };
+  }
+
+  _recalculateStats() {
+    const files = Object.values(this.state.files);
+    this.state.stats.totalFiles = files.length;
+    this.state.stats.totalChunks = files.reduce(
+      (sum, file) => sum + file.chunkCount,
+      0
+    );
+  }
+
+  static computeHash(content) {
+    return contentHashSync(content);
+  }
+
+  static async computeFileHash(filePath) {
+    const content = await readFile(filePath);
+    return MerkleTracker.computeHash(content);
+  }
+}
+
+// CLI Interface
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const tracker = new MerkleTracker();
+  await tracker.load();
+
+  const command = process.argv[2];
+
+  switch (command) {
+    case 'stats':
+      console.log(JSON.stringify(tracker.getSummary(), null, 2));
+      break;
+
+    case 'list':
+      console.log(tracker.getTrackedFiles().join('\n'));
+      break;
+
+    case 'clear':
+      tracker.clear();
+      await tracker.save();
+      console.log('Tracking state cleared');
+      break;
+
+    case 'inspect':
+      if (!process.argv[3]) {
+        console.error('Usage: merkle-tracker.js inspect <file-path>');
+        process.exit(1);
+      }
+      const metadata = tracker.state.files[process.argv[3]];
+      console.log(JSON.stringify(metadata, null, 2));
+      break;
+
+    default:
+      console.log(`
+Merkle File Tracker CLI
+
+Usage:
+  merkle-tracker.js stats              Show tracking statistics
+  merkle-tracker.js list               List all tracked files
+  merkle-tracker.js inspect <file>     Show metadata for a file
+  merkle-tracker.js clear              Clear all tracking state
+`);
+  }
+}