syncstaff-mcp 0.2.3

package/dist/lib/index/discovery.js

@@ -0,0 +1,90 @@
+/**
+ * Which files the local index is allowed to look at.
+ *
+ * Git is asked first, and not for speed. `git ls-files` already knows the
+ * answer to the question that actually matters — what is tracked, and what
+ * .gitignore excludes — and it knows it the same way for every agent on every
+ * machine. A hand-rolled walk with a hand-maintained ignore list would drift
+ * from .gitignore the first time someone edited one and not the other, and two
+ * agents whose indexes disagree about which files exist will disagree about
+ * blast radius while both believe they are correct.
+ *
+ * The walk is the fallback for a checkout that is not a git repository, which
+ * happens in tests and in extracted tarballs.
+ */
+import { execFileSync } from "node:child_process";
+import { readdirSync, statSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+/**
+ * Directories never worth indexing, used only by the non-git fallback.
+ *
+ * Deliberately short. Anything longer is a second ignore list competing with
+ * .gitignore, which is the drift this module exists to avoid.
+ */
+export const DEFAULT_IGNORED_DIRECTORIES = [
+    ".git",
+    "node_modules",
+    "dist",
+    "build",
+    ".next",
+    "coverage",
+    "__pycache__",
+    ".venv",
+];
+const runGitDefault = (args, cwd) => execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+/** POSIX separators regardless of host, so a fingerprint is portable. */
+const posix = (path) => path.split(sep).join("/");
+/**
+ * Every indexable file under `root`, repo-root-relative and sorted.
+ *
+ * Sorted because the order is an input to the index fingerprint, and readdir
+ * order is a property of the filesystem rather than of the repository. Two
+ * machines with identical content must produce an identical list.
+ */
+export function discoverFiles(root, options = {}) {
+    const exclude = options.exclude ?? [];
+    const excluded = (path) => exclude.some((prefix) => path === prefix || path.startsWith(`${prefix}/`));
+    let files;
+    try {
+        const runGit = options.runGit ?? runGitDefault;
+        // -c: tracked. -o: untracked but not ignored — an agent's new file is real
+        // work and excluding it would make the index blind to exactly the code
+        // most likely to be under an active lease.
+        const output = runGit(["ls-files", "-co", "--exclude-standard"], root);
+        files = output.split("\n").map((line) => line.trim()).filter(Boolean);
+    }
+    catch {
+        files = walk(root, root);
+    }
+    const unique = [...new Set(files.map(posix))].filter((path) => !excluded(path)).sort();
+    return options.limit === undefined ? unique : unique.slice(0, options.limit);
+}
+function walk(root, dir) {
+    const found = [];
+    let entries;
+    try {
+        entries = readdirSync(dir);
+    }
+    catch {
+        // An unreadable directory is a gap in coverage, not a reason to abandon
+        // the whole index. The caller learns about it from the file count.
+        return found;
+    }
+    for (const entry of entries) {
+        if (DEFAULT_IGNORED_DIRECTORIES.includes(entry))
+            continue;
+        const full = join(dir, entry);
+        let stats;
+        try {
+            stats = statSync(full);
+        }
+        catch {
+            continue; // a symlink to nowhere, or a file deleted mid-walk
+        }
+        if (stats.isDirectory())
+            found.push(...walk(root, full));
+        else if (stats.isFile())
+            found.push(posix(relative(root, full)));
+    }
+    return found;
+}

package/dist/lib/index/embedding.js

@@ -0,0 +1,110 @@
+/** Dimensions. A power of two, small enough that 300 pages cost microseconds. */
+const DIMENSIONS = 256;
+/**
+ * FNV-1a, 32-bit. Chosen because it is short enough to read and verify, has no
+ * seed to get wrong, and is identical on every machine. Not a cryptographic
+ * hash and does not need to be: collisions cost a little precision, never
+ * correctness, and the signed trick below cancels much of their bias.
+ */
+function fnv1a(text) {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < text.length; i += 1) {
+        hash ^= text.charCodeAt(i);
+        // >>> 0 keeps it an unsigned 32-bit value; Math.imul is the only
+        // multiplication that does not lose precision above 2^53 here.
+        hash = Math.imul(hash, 0x01000193) >>> 0;
+    }
+    return hash >>> 0;
+}
+/**
+ * Words, from prose or from an identifier.
+ *
+ * `buildContextCard` and `build_context_card` and `BUILD_CONTEXT_CARD` all
+ * yield the same three words. Without this an embedder over source metadata is
+ * mostly blind: identifiers are where the vocabulary of a codebase lives, and
+ * they are compound by convention rather than separated by spaces.
+ *
+ * `toLowerCase()` is called on ASCII-split fragments only. It is locale-
+ * sensitive in general — the Turkish dotless i is the classic example — so it
+ * must never touch a fragment that could contain one. The split below keeps
+ * only [A-Za-z0-9], which makes the lowering safe and machine-independent.
+ */
+export function tokenize(text) {
+    const words = [];
+    for (const chunk of text.split(/[^A-Za-z0-9]+/)) {
+        if (!chunk)
+            continue;
+        // Split camelCase and PascalCase, keeping acronym runs together:
+        // "HTTPServer" -> ["HTTP", "Server"], "buildCard" -> ["build", "Card"].
+        for (const part of chunk.split(/(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z]+)(?=[A-Z][a-z])/)) {
+            if (part.length > 1)
+                words.push(part.toLowerCase());
+        }
+    }
+    return words;
+}
+/** Character trigrams of a word, padded, for tolerance of plurals and typos. */
+function trigrams(word) {
+    const padded = `^${word}$`;
+    const grams = [];
+    for (let i = 0; i + 3 <= padded.length; i += 1)
+        grams.push(padded.slice(i, i + 3));
+    return grams;
+}
+/**
+ * Hash one feature into the vector.
+ *
+ * The sign comes from a second, independent bit of the hash. Feature hashing
+ * without it biases every collision upward, so unrelated documents drift
+ * toward a positive cosine and the relevance floor stops meaning anything;
+ * with it, collisions cancel in expectation.
+ */
+function accumulate(vector, feature, weight) {
+    const hash = fnv1a(feature);
+    const bucket = hash % DIMENSIONS;
+    const sign = (hash >>> 31) === 1 ? -1 : 1;
+    vector[bucket] += sign * weight;
+}
+/**
+ * A deterministic, dependency-free embedder.
+ *
+ * Whole words carry more weight than trigrams: an exact identifier match
+ * should outrank a page that merely shares letters. The ratio is a judgement
+ * call, not a measured optimum, and it is stated here rather than buried so
+ * that anyone tuning it knows it was never fitted to anything.
+ */
+export function createHashingEmbedder(options = {}) {
+    const wordWeight = options.wordWeight ?? 1;
+    const trigramWeight = options.trigramWeight ?? 0.35;
+    return {
+        id: HASHING_EMBEDDER_ID,
+        dimensions: DIMENSIONS,
+        async embed(text) {
+            const vector = new Float64Array(DIMENSIONS);
+            for (const word of tokenize(text)) {
+                accumulate(vector, word, wordWeight);
+                for (const gram of trigrams(word))
+                    accumulate(vector, gram, trigramWeight);
+            }
+            // L2 normalise so cosine is a dot product and long pages do not outrank
+            // short ones for having more of everything. An all-zero vector — empty
+            // or punctuation-only text — is returned as zeros rather than divided by
+            // zero, and cosine() already treats a zero vector as similarity 0.
+            let norm = 0;
+            for (const value of vector)
+                norm += value * value;
+            if (norm === 0)
+                return Array.from(vector);
+            const scale = 1 / Math.sqrt(norm);
+            return Array.from(vector, (value) => value * scale);
+        },
+    };
+}
+/**
+ * The embedder the pipeline uses unless a caller supplies one.
+ *
+ * Named so a reader of a results file can tell which vectors produced it, and
+ * so `pipeline_coverage` in the Phase 0 benchmark can report something more
+ * useful than `vector: true`.
+ */
+export const HASHING_EMBEDDER_ID = "hashing-v1(fnv1a,256d,word+trigram)";

package/dist/lib/index/file-index.js

@@ -0,0 +1,222 @@
+/**
+ * The local file index: discovery, parsing, and an honest account of both.
+ *
+ * `IndexCoverage` is the part that earns its place. Every consumer downstream
+ * of this — blast radius, surface deltas, context cards — is about to make a
+ * claim about a repository, and the difference between a trustworthy claim and
+ * `grep-v1` is whether the claim knows what it could not see. An index that
+ * understood 40% of a repository and one that understood 99% produce
+ * answer-shaped output either way; only the coverage figure distinguishes
+ * "there are no other call sites" from "I found no other call sites".
+ *
+ * So coverage is not a statistic for a dashboard. It is the caller's licence
+ * to say "none" instead of "none found".
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Worker } from "node:worker_threads";
+import os from "node:os";
+import { aliasSignature, discoverAliases } from "./aliases.js";
+import { discoverFiles } from "./discovery.js";
+import { hashContent, indexFingerprint } from "./fingerprint.js";
+const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024;
+/** How far in to look for a NUL before calling a file text. Git uses the same trick. */
+const BINARY_SNIFF_CHARS = 8000;
+/**
+ * Is this file bytes rather than text?
+ *
+ * Discovery yields every tracked file, including PNGs, fonts, zips and
+ * certificates, and reading one as UTF-8 succeeds — it just produces mojibake.
+ * Handing that to the lexical index is not merely wasteful: on Playwright, 448
+ * such files carried 35MB of decoded rubbish and supplied 64k of the 124k terms
+ * in the vocabulary. Half the index was noise, every real term's IDF was
+ * computed against it, and `averageLength` was inflated enough to distort BM25
+ * length normalisation for every genuine file in the repository.
+ *
+ * A NUL byte in the first few KB is the standard test and costs one scan.
+ */
+export function isProbablyBinary(source) {
+    const limit = Math.min(source.length, BINARY_SNIFF_CHARS);
+    for (let index = 0; index < limit; index += 1) {
+        if (source.charCodeAt(index) === 0)
+            return true;
+    }
+    return false;
+}
+export function buildFileIndex(root, registry, options = {}) {
+    const maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
+    const readFile = options.readFile ?? ((path) => readFileSync(path, "utf8"));
+    const paths = discoverFiles(root, options);
+    const files = [];
+    const blindSpots = [];
+    const counts = { parsed: 0, partial: 0, failed: 0, unsupported: 0 };
+    for (const path of paths) {
+        const parser = registry.parserFor(path);
+        let source;
+        try {
+            source = readFile(join(root, path));
+        }
+        catch (error) {
+            // Unreadable is not unsupported. A file the registry claims it can parse
+            // but nobody can open is a hole in the graph, and reporting it as
+            // "unsupported" would file it alongside README.md and hide it.
+            counts.failed += 1;
+            const reason = `unreadable: ${error instanceof Error ? error.message : String(error)}`;
+            blindSpots.push({ path, reason });
+            files.push({
+                path,
+                contentHash: "",
+                language: parser?.language ?? null,
+                parser: parser ? `${parser.id}@${parser.version}` : null,
+                result: { status: "failed", imports: [], exports: [], diagnostics: [{ code: "unreadable", message: reason }] },
+            });
+            continue;
+        }
+        if (source.length > maxFileBytes) {
+            counts.unsupported += 1;
+            files.push({
+                path,
+                contentHash: hashContent(source),
+                language: parser?.language ?? null,
+                parser: null,
+                result: {
+                    status: "unsupported",
+                    imports: [],
+                    exports: [],
+                    diagnostics: [{ code: "too-large", message: `${source.length} bytes exceeds ${maxFileBytes}` }],
+                },
+            });
+            if (parser)
+                blindSpots.push({ path, reason: `too large to parse (${source.length} bytes)` });
+            continue;
+        }
+        const result = registry.parse(path, source);
+        counts[result.status] += 1;
+        if (result.status === "failed" || result.status === "partial") {
+            blindSpots.push({
+                path,
+                reason: result.diagnostics[0]?.message ?? result.status,
+            });
+        }
+        files.push({
+            path,
+            contentHash: hashContent(source),
+            language: parser?.language ?? null,
+            // Only a backend that actually ran is part of the identity.
+            parser: parser && result.status !== "unsupported" ? `${parser.id}@${parser.version}` : null,
+            result,
+            ...(isProbablyBinary(source) ? {} : { searchText: source }),
+        });
+    }
+    // The denominator is files a backend claimed, not every file in the
+    // repository. Counting README.md and package-lock.json against coverage
+    // would make a healthy index look broken and, worse, make the number
+    // meaningless as a signal about the code that was analysed.
+    const claimed = counts.parsed + counts.partial + counts.failed;
+    const coverage = {
+        total: paths.length,
+        ...counts,
+        ratio: claimed === 0 ? 1 : (counts.parsed + counts.partial) / claimed,
+    };
+    // Discovered from the files already listed: the index never reaches outside
+    // what discovery gave it, which keeps this consistent with the parser's rule
+    // about ambient authority.
+    const aliases = discoverAliases(paths, { readFile: (path) => readFile(join(root, path)) });
+    const fingerprints = files.map((file) => ({
+        path: file.path,
+        contentHash: file.contentHash,
+        parser: file.parser,
+    }));
+    return {
+        root,
+        fingerprint: indexFingerprint({
+            files: fingerprints,
+            registrySignature: registry.signature(),
+            resolverSignature: aliasSignature(aliases),
+        }),
+        aliases,
+        files,
+        coverage,
+        blindSpots,
+    };
+}
+export async function buildFileIndexAsync(root, registry, options = {}) {
+    const paths = discoverFiles(root, options);
+    if (options.readFile || paths.length < 100) {
+        return buildFileIndex(root, registry, options);
+    }
+    const maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
+    const numWorkers = Math.max(1, Math.min(8, os.cpus().length - 1));
+    const chunks = Array.from({ length: numWorkers }, () => []);
+    for (let i = 0; i < paths.length; i++) {
+        chunks[i % numWorkers].push(paths[i]);
+    }
+    const workerFile = new URL("./parser-worker.js", import.meta.url);
+    const workerPromises = chunks.filter(c => c.length > 0).map((chunk) => {
+        return new Promise((resolve, reject) => {
+            const worker = new Worker(workerFile, {
+                workerData: { root, paths: chunk, maxFileBytes },
+            });
+            const resultFiles = [];
+            const resultBlindSpots = [];
+            const resultCounts = { parsed: 0, partial: 0, failed: 0, unsupported: 0 };
+            worker.on("message", (msg) => {
+                if (msg.type === "chunk") {
+                    resultFiles.push(...msg.files);
+                    resultBlindSpots.push(...msg.blindSpots);
+                    resultCounts.parsed += msg.counts.parsed;
+                    resultCounts.partial += msg.counts.partial;
+                    resultCounts.failed += msg.counts.failed;
+                    resultCounts.unsupported += msg.counts.unsupported;
+                }
+                else if (msg.type === "done") {
+                    worker.terminate();
+                    resolve({ files: resultFiles, blindSpots: resultBlindSpots, counts: resultCounts });
+                }
+            });
+            worker.on("error", reject);
+            worker.on("exit", (code) => {
+                if (code !== 0)
+                    reject(new Error(`Worker stopped with exit code ${code}`));
+            });
+        });
+    });
+    const results = await Promise.all(workerPromises);
+    const files = [];
+    const blindSpots = [];
+    const counts = { parsed: 0, partial: 0, failed: 0, unsupported: 0 };
+    for (const res of results) {
+        files.push(...res.files);
+        blindSpots.push(...res.blindSpots);
+        counts.parsed += res.counts.parsed;
+        counts.partial += res.counts.partial;
+        counts.failed += res.counts.failed;
+        counts.unsupported += res.counts.unsupported;
+    }
+    files.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+    blindSpots.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+    const claimed = counts.parsed + counts.partial + counts.failed;
+    const coverage = {
+        total: paths.length,
+        ...counts,
+        ratio: claimed === 0 ? 1 : (counts.parsed + counts.partial) / claimed,
+    };
+    const aliases = discoverAliases(paths, { readFile: (path) => readFileSync(join(root, path), "utf8") });
+    const fingerprints = files.map((file) => ({
+        path: file.path,
+        contentHash: file.contentHash,
+        parser: file.parser,
+    }));
+    return {
+        root,
+        fingerprint: indexFingerprint({
+            files: fingerprints,
+            registrySignature: registry.signature(),
+            resolverSignature: aliasSignature(aliases),
+        }),
+        aliases,
+        files,
+        coverage,
+        blindSpots,
+    };
+}

package/dist/lib/index/git-history.js

@@ -0,0 +1,136 @@
+import { execFileSync } from "node:child_process";
+const DEFAULT_MAX_COMMITS = 10_000;
+const DEFAULT_MAX_FILES_PER_COMMIT = 200;
+// A mature repository's bounded history can exceed Node's 1 MiB default even
+// when the pair index itself is bounded. Keep the output local, but size the
+// subprocess buffer for a realistic temporal corpus instead of failing before
+// retrieval starts. The history and file-count budgets remain the stronger
+// controls on CPU and memory used to build the index.
+const GIT_HISTORY_MAX_BUFFER_BYTES = 64 * 1024 * 1024;
+const runGitDefault = (args, cwd) => execFileSync("git", [...args], {
+    cwd,
+    encoding: "utf8",
+    maxBuffer: GIT_HISTORY_MAX_BUFFER_BYTES,
+    stdio: ["ignore", "pipe", "pipe"],
+});
+const ref = (value, label) => {
+    const trimmed = value.trim();
+    if (!trimmed || trimmed.startsWith("-"))
+        throw new Error(`Git co-change ${label} must be a non-empty revision`);
+    return trimmed;
+};
+const commitFiles = (output) => {
+    const commits = [];
+    let current = null;
+    for (const line of output.split(/\r?\n/)) {
+        if (line.startsWith("commit:")) {
+            if (current)
+                commits.push(current);
+            current = [];
+            continue;
+        }
+        const path = line.trim();
+        if (current && path)
+            current.push(path);
+    }
+    if (current)
+        commits.push(current);
+    return commits;
+};
+const pairKey = (left, right) => `${left}\0${right}`;
+/**
+ * Build query-independent historical evidence for the local retrieval client.
+ *
+ * `before` is deliberately mandatory and must resolve to the checked-out
+ * corpus HEAD. A caller cannot accidentally index today's history while
+ * grading a past snapshot: the mismatch is an error, not a caveat. The
+ * server never sees this index or the raw commit list.
+ */
+export function buildCoChangeIndex(root, eligiblePaths, options) {
+    const runGit = options.runGit ?? runGitDefault;
+    const beforeRef = ref(options.before, "before revision");
+    const maxCommits = options.maxCommits ?? DEFAULT_MAX_COMMITS;
+    const maxFilesPerCommit = options.maxFilesPerCommit ?? DEFAULT_MAX_FILES_PER_COMMIT;
+    if (!Number.isInteger(maxCommits) || maxCommits < 1)
+        throw new Error("Git co-change maxCommits must be a positive integer");
+    if (!Number.isInteger(maxFilesPerCommit) || maxFilesPerCommit < 2)
+        throw new Error("Git co-change maxFilesPerCommit must be at least 2");
+    const head = runGit(["rev-parse", "HEAD"], root).trim();
+    const before = runGit(["rev-parse", `${beforeRef}^{commit}`], root).trim();
+    if (!head || !before)
+        throw new Error("Git co-change could not resolve the checkout HEAD and before revision");
+    if (head !== before) {
+        throw new Error(`Git co-change cutoff ${before} does not match checkout HEAD ${head}; index the corpus at the before revision`);
+    }
+    if (options.corpusCommit) {
+        const corpusCommit = runGit(["rev-parse", `${ref(options.corpusCommit, "corpus revision")}^{commit}`], root).trim();
+        if (corpusCommit !== before) {
+            throw new Error(`Git co-change corpus commit ${corpusCommit} does not match cutoff ${before}`);
+        }
+    }
+    // Ask for one extra commit so truncation is observable in the returned
+    // metadata. Every revision and path remains an argument, never shell text.
+    const history = runGit([
+        "log",
+        "--no-merges",
+        "--format=commit:%H",
+        "--name-only",
+        "--diff-filter=ACDMRT",
+        `--max-count=${maxCommits + 1}`,
+        before,
+        "--",
+    ], root);
+    const parsedCommits = commitFiles(history);
+    const commits = parsedCommits.slice(0, maxCommits);
+    const truncated = parsedCommits.length > maxCommits;
+    const activity = new Map();
+    const pairs = new Map();
+    // `git log` is newest-first. Keep this metadata private to the local index:
+    // it is only a deterministic tie-break, never a separately fused ranking
+    // leg or a value that crosses the client/server boundary.
+    const lastTouched = new Map();
+    for (const [commitIndex, changed] of commits.entries()) {
+        const files = [...new Set(changed.filter((path) => eligiblePaths.has(path)))].sort();
+        const bounded = files.slice(0, maxFilesPerCommit);
+        for (const path of bounded) {
+            activity.set(path, (activity.get(path) ?? 0) + 1);
+            if (!lastTouched.has(path))
+                lastTouched.set(path, commitIndex);
+        }
+        for (let left = 0; left < bounded.length; left += 1) {
+            for (let right = left + 1; right < bounded.length; right += 1) {
+                const key = pairKey(bounded[left], bounded[right]);
+                pairs.set(key, (pairs.get(key) ?? 0) + 1);
+            }
+        }
+    }
+    const relatedByPath = new Map();
+    for (const [key, count] of pairs) {
+        const [left, right] = key.split("\0");
+        const score = count / Math.sqrt(Math.max(1, activity.get(left) ?? 1) * Math.max(1, activity.get(right) ?? 1));
+        const leftHits = relatedByPath.get(left) ?? [];
+        const rightHits = relatedByPath.get(right) ?? [];
+        leftHits.push({ path: right, score, co_change_count: count });
+        rightHits.push({ path: left, score, co_change_count: count });
+        relatedByPath.set(left, leftHits);
+        relatedByPath.set(right, rightHits);
+    }
+    for (const hits of relatedByPath.values()) {
+        hits.sort((a, b) => b.score - a.score
+            || b.co_change_count - a.co_change_count
+            || (lastTouched.get(a.path) ?? Number.POSITIVE_INFINITY) - (lastTouched.get(b.path) ?? Number.POSITIVE_INFINITY)
+            || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    }
+    return {
+        before,
+        corpus_commit: head,
+        commit_count: commits.length,
+        truncated,
+        files: new Set(eligiblePaths),
+        related(seed, limit = 20) {
+            if (!Number.isInteger(limit) || limit < 1)
+                return [];
+            return (relatedByPath.get(seed) ?? []).slice(0, limit).map((hit) => ({ ...hit }));
+        },
+    };
+}