syncstaff-mcp 0.2.3

package/dist/lib/index/incremental.js

@@ -0,0 +1,332 @@
+/**
+ * Updating an index without rebuilding it, and the one rule that makes that
+ * safe to do.
+ *
+ * The rule: an incrementally updated index must be byte-identical to a full
+ * rebuild of the same working tree. Not "close enough", not "eventually
+ * consistent" — identical, including the fingerprint.
+ *
+ * That is not a tidiness preference. The fingerprint is what two agents on two
+ * machines compare to decide whether a blast radius computed over there means
+ * anything over here. If an incremental update can produce a different
+ * fingerprint from a full build over identical bytes, then the comparison
+ * starts failing for reasons that have nothing to do with the code, agents
+ * stop trusting each other's analysis, and the cheapest way to make the system
+ * work again is to turn incremental updates off. Worse, if it produces the
+ * same fingerprint over *different* content, two agents believe they agree
+ * when they do not — which is the failure this whole subsystem exists to
+ * prevent, arrived at by a new route.
+ *
+ * So the invariant has a test that builds both ways and compares the whole
+ * serialised index, and every design decision below is downstream of keeping
+ * that test true.
+ *
+ * The second rule follows from the first: when the change set cannot be
+ * trusted, rebuild everything. A partial update from an incomplete change list
+ * is the one outcome worse than a slow one, because it is wrong and looks
+ * fine.
+ */
+import { buildFileIndex, isProbablyBinary } from "./file-index.js";
+import { aliasSignature, discoverAliases } from "./aliases.js";
+import { discoverFiles } from "./discovery.js";
+import { hashContent, indexFingerprint } from "./fingerprint.js";
+import { readFileSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { join } from "node:path";
+import { byCodeUnit } from "./order.js";
+import { invalidateVectorCache } from "./vector-cache.js";
+/**
+ * Load a local snapshot when one exists, then keep it current through the
+ * digest-verified incremental path. A cold checkout is written immediately so
+ * the next hook invocation can skip unchanged parsing work.
+ */
+export function loadOrUpdateFileIndex(root, registry, options) {
+    const now = options.now?.() ?? new Date();
+    const currentCommit = options.currentCommit === undefined ? readCurrentCommit(root) : options.currentCommit;
+    const previousMetadata = options.archive.getMetadata();
+    const persisted = options.archive.getIndex();
+    const stale = Boolean(persisted &&
+        previousMetadata.indexedCommit &&
+        currentCommit &&
+        previousMetadata.indexedCommit !== currentCommit);
+    const stalenessWarning = stale
+        ? `Local index was built against ${previousMetadata.indexedCommit}, but live HEAD is ${currentCommit}; rebuilt.`
+        : null;
+    const metadata = {
+        indexedCommit: currentCommit ?? previousMetadata.indexedCommit,
+        indexedAt: now.toISOString(),
+    };
+    if (!persisted || persisted.root !== root || stale) {
+        if (persisted)
+            invalidateVectorCache(persisted.fingerprint);
+        const index = buildFileIndex(root, registry, options);
+        options.archive.putIndex(index, metadata);
+        return {
+            index,
+            reason: "full-rebuild",
+            rebuildCause: stale
+                ? "indexed commit diverged from live HEAD"
+                : persisted
+                    ? "persisted snapshot belongs to another checkout"
+                    : "no persisted snapshot",
+            changed: { added: [], modified: [], removed: [] },
+            reparsed: index.files.length,
+            source: "cold",
+            freshness: freshness(metadata, now, stalenessWarning),
+        };
+    }
+    const update = updateFileIndex(persisted, root, registry, options);
+    const indexChanged = update.reparsed > 0 || update.index.fingerprint !== persisted.fingerprint;
+    const indexedAt = !indexChanged && previousMetadata.indexedAt
+        ? previousMetadata.indexedAt
+        : now.toISOString();
+    options.archive.putIndex(update.index, { indexedCommit: metadata.indexedCommit, indexedAt });
+    return {
+        ...update,
+        source: update.reparsed === 0 ? "restored" : "incremental",
+        freshness: freshness({ indexedCommit: metadata.indexedCommit, indexedAt }, now, stalenessWarning),
+    };
+}
+function freshness(metadata, now, stalenessWarning) {
+    let age = null;
+    if (metadata.indexedAt) {
+        const timestamp = Date.parse(metadata.indexedAt);
+        if (Number.isFinite(timestamp))
+            age = Math.max(0, now.getTime() - timestamp);
+    }
+    return {
+        indexed_commit: metadata.indexedCommit,
+        index_age: age,
+        staleness_warning: stalenessWarning,
+    };
+}
+function readCurrentCommit(root) {
+    try {
+        return execFileSync("git", ["-C", root, "rev-parse", "--verify", "HEAD"], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim() || null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Produce the current index, re-reading as little as possible.
+ *
+ * Discovery still runs in full every time. It is one `git ls-files`, it is the
+ * only way to notice additions and deletions, and skipping it to save
+ * milliseconds is how an index quietly stops matching the repository.
+ */
+export function updateFileIndex(previous, root, registry, options = {}) {
+    const readFile = options.readFile ?? ((path) => readFileSync(path, "utf8"));
+    // The parsers that produced `previous` are not the parsers running now if a
+    // backend was registered, removed, or upgraded. Every cached parse result is
+    // then suspect, and reusing them would produce an index that claims to have
+    // been built by the current registry when it was not.
+    const previousSignature = signatureOf(previous);
+    const currentSignature = registry.signature();
+    if (previousSignature !== null && previousSignature !== currentSignature) {
+        invalidateVectorCache(previous.fingerprint);
+        return fullRebuild(root, registry, options, `parser registry changed (${previousSignature} -> ${currentSignature})`);
+    }
+    let paths;
+    try {
+        paths = discoverFiles(root, options);
+    }
+    catch (error) {
+        invalidateVectorCache(previous.fingerprint);
+        return fullRebuild(root, registry, options, `discovery failed: ${describe(error)}`);
+    }
+    const before = new Map(previous.files.map((file) => [file.path, file]));
+    const currentPaths = new Set(paths);
+    const hinted = options.changedHint ? new Set(options.changedHint) : null;
+    const added = [];
+    const modified = [];
+    const removed = previous.files.map((f) => f.path).filter((path) => !currentPaths.has(path)).sort();
+    const files = [];
+    let reparsed = 0;
+    for (const path of paths) {
+        const cached = before.get(path);
+        // A hint can say "this changed" and be believed into re-reading, but it can
+        // never say "this did not change" and be believed into skipping: a missed
+        // event would leave a stale entry that no later pass ever revisits.
+        let source;
+        try {
+            source = readFile(join(root, path));
+        }
+        catch {
+            // Unreadable now. Re-index it properly so the failure is recorded as a
+            // blind spot rather than silently inheriting the last good parse.
+            const rebuilt = reindexOne(root, path, registry, options);
+            reparsed += 1;
+            files.push(rebuilt);
+            if (!cached)
+                added.push(path);
+            else
+                modified.push(path);
+            continue;
+        }
+        const contentHash = hashContent(source);
+        if (cached && cached.contentHash === contentHash && !hinted?.has(path)) {
+            // Identical bytes, same parsers: the previous result is not an
+            // approximation of the current one, it IS the current one.
+            files.push(cached);
+            continue;
+        }
+        const result = registry.parse(path, source);
+        const parser = registry.parserFor(path);
+        files.push({
+            path,
+            contentHash,
+            language: parser?.language ?? null,
+            parser: parser && result.status !== "unsupported" ? `${parser.id}@${parser.version}` : null,
+            result,
+            // Must match buildFileIndex exactly. Omitting it here would leave every
+            // reparsed file unsearchable by its own contents while untouched files
+            // stayed searchable — retrieval quality decaying as the repo is edited,
+            // which is the hardest kind of regression to notice.
+            ...(isProbablyBinary(source) ? {} : { searchText: source }),
+        });
+        reparsed += 1;
+        if (!cached)
+            added.push(path);
+        else if (cached.contentHash !== contentHash)
+            modified.push(path);
+    }
+    // Re-discovered every pass rather than inherited. A tsconfig edit changes
+    // how every specifier resolves while leaving every source file untouched,
+    // so carrying the old map forward would produce a stale graph from an index
+    // that looks perfectly fresh. Parse results are unaffected, so this needs no
+    // full rebuild — only the identity and the resolution change.
+    const aliases = discoverAliases(paths, { readFile: (path) => readFile(join(root, path)) });
+    const result = {
+        index: assemble(root, registry, files, aliases),
+        reason: "incremental",
+        rebuildCause: null,
+        changed: { added: added.sort(), modified: modified.sort(), removed },
+        reparsed,
+    };
+    if (result.index.fingerprint !== previous.fingerprint)
+        invalidateVectorCache(previous.fingerprint);
+    return result;
+}
+function reindexOne(root, path, registry, options) {
+    const single = buildFileIndex(root, registry, {
+        ...options,
+        // buildFileIndex discovers; restrict it to the one path by excluding
+        // nothing and filtering after, which keeps one code path for parsing.
+    });
+    const found = single.files.find((file) => file.path === path);
+    if (found)
+        return found;
+    return {
+        path,
+        contentHash: "",
+        language: null,
+        parser: null,
+        result: {
+            status: "failed",
+            imports: [],
+            exports: [],
+            diagnostics: [{ code: "vanished", message: `${path} disappeared during indexing` }],
+        },
+    };
+}
+/**
+ * Rebuild the derived parts of a FileIndex from a file list.
+ *
+ * Shares nothing with buildFileIndex's own assembly by accident: coverage and
+ * fingerprint are recomputed here with the same rules, and the byte-identity
+ * test is what holds the two in agreement. If that test ever fails, this
+ * function and buildFileIndex have drifted and the incremental path must be
+ * considered wrong until they match again.
+ */
+function assemble(root, registry, files, aliases) {
+    const sorted = [...files].sort((a, b) => byCodeUnit(a.path, b.path));
+    const counts = { parsed: 0, partial: 0, failed: 0, unsupported: 0 };
+    const blindSpots = [];
+    for (const file of sorted) {
+        counts[file.result.status] += 1;
+        if (file.result.status === "failed" || file.result.status === "partial") {
+            blindSpots.push({ path: file.path, reason: file.result.diagnostics[0]?.message ?? file.result.status });
+        }
+    }
+    const claimed = counts.parsed + counts.partial + counts.failed;
+    const fingerprints = sorted.map((file) => ({
+        path: file.path,
+        contentHash: file.contentHash,
+        parser: file.parser,
+    }));
+    return {
+        root,
+        // Must match buildFileIndex's identity inputs exactly. It did not, once:
+        // the resolver signature was added there and not here, and five tests
+        // failed within the minute. That is the byte-identity gate doing its job,
+        // and the reason this function is worth the duplication it costs.
+        fingerprint: indexFingerprint({
+            files: fingerprints,
+            registrySignature: registry.signature(),
+            resolverSignature: aliasSignature(aliases),
+        }),
+        aliases,
+        files: sorted,
+        coverage: {
+            total: sorted.length,
+            ...counts,
+            ratio: claimed === 0 ? 1 : (counts.parsed + counts.partial) / claimed,
+        },
+        blindSpots: blindSpots.sort((a, b) => byCodeUnit(a.path, b.path)),
+    };
+}
+function fullRebuild(root, registry, options, cause) {
+    const index = buildFileIndex(root, registry, options);
+    return {
+        index,
+        reason: "full-rebuild",
+        rebuildCause: cause,
+        changed: { added: [], modified: [], removed: [] },
+        reparsed: index.files.length,
+    };
+}
+/** The registry signature the previous index was built with, if recoverable. */
+function signatureOf(previous) {
+    const parsers = [...new Set(previous.files.map((file) => file.parser).filter((p) => p !== null))];
+    return parsers.length === 0 ? null : parsers.sort().join(",");
+}
+function describe(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+import { buildFileIndexAsync } from "./file-index.js";
+export async function updateFileIndexAsync(previous, root, registry, options = {}) {
+    const previousSignature = signatureOf(previous);
+    const currentSignature = registry.signature();
+    if (previousSignature !== null && previousSignature !== currentSignature) {
+        invalidateVectorCache(previous.fingerprint);
+        return fullRebuildAsync(root, registry, options, `parser registry changed (${previousSignature} -> ${currentSignature})`);
+    }
+    let paths;
+    try {
+        paths = discoverFiles(root, options);
+    }
+    catch (error) {
+        invalidateVectorCache(previous.fingerprint);
+        return fullRebuildAsync(root, registry, options, `discovery failed: ${describe(error)}`);
+    }
+    // To properly parallelize the incremental pass, we could gather files that need reparsing
+    // and dispatch them. However, since incremental changes are overwhelmingly small, falling
+    // back to the synchronous path for the delta is perfectly sufficient, as long as full
+    // rebuilds are asynchronous.
+    // We'll just run the synchronous updateFileIndex. It handles the delta quickly.
+    return updateFileIndex(previous, root, registry, options);
+}
+async function fullRebuildAsync(root, registry, options, cause) {
+    const index = await buildFileIndexAsync(root, registry, options);
+    return {
+        index,
+        reason: "full-rebuild",
+        rebuildCause: cause,
+        changed: { added: [], modified: [], removed: [] },
+        reparsed: index.files.length,
+    };
+}