seer-mcp 0.1.0

package/src/indexer/symbolhistory.ts

@@ -0,0 +1,172 @@
+import { Store } from '../db/store.js';
+import {
+  commitsForFile, fileDiffInfo, isGitRepo, gitHeadSha, gitRemoteUrl,
+  extractPrNumber, githubPrUrl,
+} from './git.js';
+import { buildContinuity } from './continuity.js';
+
+/**
+ * Symbol-history pass — for every indexed function/method/constructor/class
+ * symbol, walk the commits that touched its file and record the ones whose
+ * diff hunks overlap the symbol's current line span.
+ *
+ * Limitations (documented honestly):
+ *   - Renames within a file: we do NOT reparse historical file versions, so
+ *     a function rename will appear as "history starts here" at the rename
+ *     commit. The symbol_key column captures `kind:qualified_name` so a
+ *     future enhancement (key-match on historical AST parse) could recover
+ *     pre-rename history without touching this schema.
+ *   - File renames: `git log --follow` is used per-file so file renames
+ *     ARE preserved; but cross-file moves are not.
+ *   - Line numbers shift across history. We compare a commit's hunks against
+ *     the current symbol's line range — older overlaps are approximate.
+ *     match_strategy = 'overlap' / confidence < 1.0 reflect that.
+ *
+ * Performance: bounded by `maxCommitsPerFile` (default 200) and per-call
+ * timeout. For a 10k-symbol codebase this runs in ~1-2 min once; afterwards
+ * the `git_index_state` lets us short-circuit if HEAD is unchanged.
+ */
+
+export interface SymbolHistoryOptions {
+  /** Cap commits processed per file. Default 200. */
+  maxCommitsPerFile?: number;
+  /** Cap history-since lookback in seconds. Default no limit. */
+  since?: number;
+  /** Only re-run if HEAD differs from `git_index_state.last_head_sha`. Default true. */
+  skipIfHeadUnchanged?: boolean;
+  /** Logger; defaults to stderr. */
+  log?: (msg: string) => void;
+}
+
+export interface SymbolHistoryResult {
+  filesProcessed: number;
+  symbolsProcessed: number;
+  historyRowsInserted: number;
+  skipped: boolean;
+  elapsedMs: number;
+}
+
+export async function buildSymbolHistory(
+  repoRoot: string,
+  store: Store,
+  options: SymbolHistoryOptions = {},
+): Promise<SymbolHistoryResult> {
+  const start = Date.now();
+  const log = options.log ?? ((m: string) => process.stderr.write(`[symbol-history] ${m}\n`));
+  if (!isGitRepo(repoRoot)) {
+    log('not a git repo, skipping');
+    return { filesProcessed: 0, symbolsProcessed: 0, historyRowsInserted: 0, skipped: true, elapsedMs: Date.now() - start };
+  }
+  const head = gitHeadSha(repoRoot);
+  const state = store.getGitIndexState();
+  // Skip only if symbol-history specifically has already been built against
+  // this HEAD. Reading `lastHistoryHeadSha` instead of the generic
+  // `lastHeadSha` means a previous file-level churn pass (which also stamps
+  // git_index_state) can't trick us into skipping when history was never
+  // actually built.
+  if (options.skipIfHeadUnchanged !== false && state && state.lastHistoryHeadSha === head && head !== null) {
+    log(`HEAD ${head?.slice(0, 8)} unchanged; skipping`);
+    return { filesProcessed: 0, symbolsProcessed: 0, historyRowsInserted: 0, skipped: true, elapsedMs: Date.now() - start };
+  }
+  const remote = gitRemoteUrl(repoRoot);
+  const maxCommits = options.maxCommitsPerFile ?? 200;
+  const symbols = store.listSymbolsForHistoryIndex();
+  if (symbols.length === 0) {
+    return { filesProcessed: 0, symbolsProcessed: 0, historyRowsInserted: 0, skipped: false, elapsedMs: Date.now() - start };
+  }
+
+  // Group symbols by file path so we walk `git log` once per file.
+  const byFile = new Map<string, typeof symbols>();
+  for (const s of symbols) {
+    let arr = byFile.get(s.filePath);
+    if (!arr) { arr = []; byFile.set(s.filePath, arr); }
+    arr.push(s);
+  }
+
+  let totalInserts = 0;
+  let processedFiles = 0;
+  for (const [filePath, fileSymbols] of byFile) {
+    processedFiles++;
+    const commits = await commitsForFile(repoRoot, filePath, { limit: maxCommits, since: options.since });
+    if (commits.length === 0) continue;
+    // For each commit, fetch its diff hunks (in the new-file coordinate
+    // space). A symbol whose [line_start..line_end] range overlaps any hunk
+    // is considered touched.
+    for (let i = 0; i < commits.length; i++) {
+      const c = commits[i];
+      // Use the per-commit path resolved from `git log --follow --name-status`.
+      // If that wasn't available (older git or no rename detected), fall back
+      // to the current path — that's correct for commits where the file
+      // hadn't been renamed.
+      const lookupPath = c.pathAtCommit ?? filePath;
+      const info = await fileDiffInfo(repoRoot, c.sha, lookupPath);
+      if (info.hunks.length === 0 && !info.isFileAddition) continue;
+      const totalAdded = info.hunks.reduce((acc, h) => acc + h.newLines, 0);
+      const totalRemoved = info.hunks.reduce((acc, h) => acc + h.oldLines, 0);
+      const prNum = extractPrNumber(c.message);
+      const prUrl = prNum != null ? githubPrUrl(remote, prNum) : null;
+      for (const sym of fileSymbols) {
+        // The current symbol's line span is in the HEAD file. As commits get
+        // older, line numbers drift — so we accept any overlap in any commit
+        // as "this commit touched the line range that now holds this symbol".
+        // confidence drops with commit age to flag the drift.
+        //
+        // Special case: if this commit added the file (isFileAddition), the
+        // pre-commit state had no symbols at all — every current symbol
+        // got its initial existence from this commit, so attribute all of
+        // them regardless of where the hunk lines up. Match strategy
+        // 'file-addition' so the caller can tell apart "this commit
+        // created the file" from "this commit overlapped the symbol".
+        const symStart = sym.lineStart + 1; // git is 1-indexed
+        const symEnd = sym.lineEnd + 1;
+        let strategy: 'overlap' | 'file-addition';
+        if (info.isFileAddition) {
+          strategy = 'file-addition';
+        } else {
+          const overlaps = info.hunks.some(h => {
+            const hStart = h.newStart;
+            const hEnd = h.newStart + Math.max(0, h.newLines - 1);
+            return hStart <= symEnd && hEnd >= symStart;
+          });
+          if (!overlaps) continue;
+          strategy = 'overlap';
+        }
+        const ageDays = Math.max(0, (Date.now() / 1000 - c.committedAt) / 86400);
+        // Confidence: 1.0 for HEAD..HEAD~1, decays with age but never below 0.3.
+        const confidence = Math.max(0.3, 1.0 - Math.min(0.7, ageDays / 365 * 0.1));
+        store.insertSymbolHistory(
+          sym.id, sym.symbolKey, c.sha,
+          c.authorName, c.authorEmail, c.committedAt, c.message,
+          totalAdded, totalRemoved,
+          prNum, prUrl,
+          strategy, confidence,
+        );
+        totalInserts++;
+      }
+    }
+  }
+
+  // v10 — run the continuity heuristics for symbols whose recorded history is
+  // shallow. Strictly additive (lives in symbol_history_continuity); never
+  // touches symbol_history rows.
+  try {
+    if (store.hasV10()) {
+      const cont = buildContinuity(store, { historyThreshold: 1 });
+      log(`continuity: considered=${cont.candidatesConsidered}, inserted=${cont.inserted}, skipped=${cont.skipped}`);
+    }
+  } catch (err) {
+    log(`continuity pass failed: ${(err as Error).message}`);
+  }
+
+  // Stamp the history-specific HEAD marker (not the generic one) so a future
+  // run can skip this work without colliding with file-level churn's stamp.
+  store.setHistoryHeadSha(repoRoot, head, remote);
+  log(`done: ${processedFiles} files, ${totalInserts} history rows`);
+  return {
+    filesProcessed: processedFiles,
+    symbolsProcessed: symbols.length,
+    historyRowsInserted: totalInserts,
+    skipped: false,
+    elapsedMs: Date.now() - start,
+  };
+}

package/src/indexer/watcher.ts

@@ -0,0 +1,206 @@
+import path from 'path';
+import { Store } from '../db/store.js';
+import { Indexer } from './index.js';
+
+/**
+ * Background file watcher that keeps the index warm.
+ *
+ * Design split:
+ *   - Watcher (this module) reacts to file events and marks files dirty in a
+ *     shared in-memory set. Debounces bursts so a `git checkout` doesn't
+ *     hammer the indexer.
+ *   - JIT sync ([./freshness.ts]) does the actual correctness work — when an
+ *     MCP query comes in, it consults the dirty set + on-disk hashes and
+ *     reindexes just what needs it.
+ *
+ * Why both? The watcher alone is racy (filesystem events on Windows arrive
+ * out of order, can be missed under load) and the JIT alone is slow on
+ * burst-y workloads (every query re-walks the workspace). Together the
+ * watcher absorbs steady-state edits so most JIT runs are no-ops, while JIT
+ * provides the per-query correctness guarantee.
+ *
+ * We use chokidar because it abstracts the OS-level oddities (FSEvents on
+ * macOS, ReadDirectoryChangesW on Windows, inotify on Linux) and provides
+ * a uniform API. It's the same library Aider uses for its watch mode.
+ */
+
+import chokidar, { FSWatcher } from 'chokidar';
+
+export interface WatcherOptions {
+  /** Debounce window in ms before triggering a reindex pass. Default 250ms. */
+  debounceMs?: number;
+  /** Logger; default writes to stderr (so stdio MCP isn't polluted). */
+  log?: (msg: string) => void;
+  /**
+   * Skip these paths. Defaults match the indexer's ignore list — we don't
+   * want chokidar to fire on `node_modules/` updates that the index would
+   * skip anyway. The list is intentionally narrower than the full glob; the
+   * indexer's `discoverFiles()` re-applies all rules at reindex time.
+   */
+  ignored?: (string | RegExp)[];
+}
+
+export class SeerWatcher {
+  private watcher: FSWatcher | null = null;
+  private dirty = new Set<string>();
+  private debounceTimer: NodeJS.Timeout | null = null;
+  private indexing = false;
+  private indexingQueue = false;
+  private lastSyncMs: number = 0;
+  private lastSyncResult: { dirtyReindexed: number; added: number; removed: number; elapsedMs: number } | null = null;
+  private logFn: (msg: string) => void;
+  private debounceMs: number;
+
+  constructor(
+    private repoRoot: string,
+    private store: Store,
+    private indexer: Indexer,
+    options: WatcherOptions = {},
+  ) {
+    this.debounceMs = options.debounceMs ?? 250;
+    this.logFn = options.log ?? ((m) => process.stderr.write(`[watcher] ${m}\n`));
+  }
+
+  /**
+   * Start watching. Idempotent: calling twice is a no-op.
+   */
+  start(): void {
+    if (this.watcher) return;
+    const abs = path.resolve(this.repoRoot);
+    this.watcher = chokidar.watch(abs, {
+      ignored: [
+        // chokidar accepts globs and functions. We use simple substring tests
+        // because the chokidar glob engine is finicky on Windows paths.
+        (p: string) => {
+          const norm = p.replace(/\\/g, '/');
+          return (
+            norm.includes('/node_modules/') ||
+            norm.includes('/.git/') ||
+            norm.includes('/dist/') ||
+            norm.includes('/build/') ||
+            norm.includes('/.seer/') ||
+            norm.includes('/target/') ||
+            norm.includes('/obj/')
+          );
+        },
+      ],
+      persistent: true,
+      ignoreInitial: true,  // Don't fire for files that already exist
+      awaitWriteFinish: {
+        // Some editors write through a tmp + rename dance. Wait for the file
+        // to stop growing before considering it changed.
+        stabilityThreshold: 100,
+        pollInterval: 50,
+      },
+    });
+
+    const onChange = (filePath: string): void => {
+      this.dirty.add(filePath);
+      this.scheduleSync();
+    };
+    this.watcher.on('add', onChange);
+    this.watcher.on('change', onChange);
+    this.watcher.on('unlink', onChange);
+    this.watcher.on('error', (err: unknown) => this.logFn(`error: ${err}`));
+    this.logFn(`watching ${abs}`);
+  }
+
+  async stop(): Promise<void> {
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+      this.debounceTimer = null;
+    }
+    if (this.watcher) {
+      await this.watcher.close();
+      this.watcher = null;
+    }
+  }
+
+  /** Whether anything is currently waiting to be reindexed. */
+  isDirty(): boolean {
+    return this.dirty.size > 0 || this.indexing || this.indexingQueue;
+  }
+
+  dirtyCount(): number { return this.dirty.size; }
+
+  /**
+   * Snapshot for `seer_health`: when did we last reconcile, how many files,
+   * how long did it take. Lets agents tell whether the index they're querying
+   * is "this happened just now" vs "we haven't run in 10 minutes."
+   */
+  syncStatus(): {
+    watching: boolean;
+    dirtyCount: number;
+    indexing: boolean;
+    lastSyncMs: number;
+    lastSync: { dirtyReindexed: number; added: number; removed: number; elapsedMs: number } | null;
+  } {
+    return {
+      watching: this.watcher !== null,
+      dirtyCount: this.dirty.size,
+      indexing: this.indexing,
+      lastSyncMs: this.lastSyncMs,
+      lastSync: this.lastSyncResult,
+    };
+  }
+
+  /**
+   * Force an immediate sync, bypassing the debounce. Used by `seer_reindex`
+   * and JIT — both need the dirty set drained NOW, not when a timer fires.
+   */
+  async syncNow(): Promise<void> {
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+      this.debounceTimer = null;
+    }
+    await this.runSync();
+  }
+
+  private scheduleSync(): void {
+    if (this.debounceTimer) clearTimeout(this.debounceTimer);
+    this.debounceTimer = setTimeout(() => {
+      this.debounceTimer = null;
+      // Fire-and-forget — uncaught rejection would just kill the process.
+      // Errors are reported through the log function.
+      this.runSync().catch(err => this.logFn(`sync failed: ${err.message ?? err}`));
+    }, this.debounceMs);
+  }
+
+  /**
+   * Reindex the workspace. Serialized — if a sync is already running, we
+   * queue at most one follow-up and coalesce further requests into it. This
+   * matches the indexer's contract that only one writer can be active.
+   */
+  private async runSync(): Promise<void> {
+    if (this.indexing) {
+      this.indexingQueue = true;
+      return;
+    }
+    this.indexing = true;
+    try {
+      const dirtyCount = this.dirty.size;
+      // Clear the dirty set BEFORE running the index so any events arriving
+      // during the index are caught for the next pass instead of being lost.
+      this.dirty.clear();
+      const start = Date.now();
+      const result = await this.indexer.indexDirectory(this.repoRoot, { quiet: true });
+      this.lastSyncMs = Date.now();
+      this.lastSyncResult = {
+        dirtyReindexed: dirtyCount,
+        added: result.filesIndexed,
+        removed: 0, // indexer doesn't expose pruned count; OK for now
+        elapsedMs: result.elapsedMs,
+      };
+      this.logFn(
+        `sync: ${dirtyCount} dirty event(s), ${result.filesIndexed} reindexed, ` +
+        `${result.filesReusedFromCache} reused, ${result.elapsedMs}ms`,
+      );
+    } finally {
+      this.indexing = false;
+      if (this.indexingQueue) {
+        this.indexingQueue = false;
+        this.runSync().catch(err => this.logFn(`requeued sync failed: ${err.message ?? err}`));
+      }
+    }
+  }
+}