preflight-mcp 0.1.0 → 0.1.1

package/dist/bundle/service.js

@@ -3,7 +3,8 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import { logger } from '../logging/logger.js';
 import { getLocalHeadSha, getRemoteHeadSha, parseOwnerRepo, shallowClone, toCloneUrl, } from './github.js';
-import { ingestRepoToBundle } from './ingest.js';
+import { downloadAndExtractGitHubArchive } from './githubArchive.js';
+import { classifyIngestedFileKind, ingestRepoToBundle } from './ingest.js';
 import { writeManifest, readManifest } from './manifest.js';
 import { getBundlePaths, repoMetaPath, repoNormDir, repoRawDir, repoRootDir } from './paths.js';
 import { writeAgentsMd, writeStartHereMd } from './guides.js';
@@ -13,22 +14,261 @@ import { ingestContext7Libraries } from './context7.js';
 import { ingestDeepWikiRepo } from './deepwiki.js';
 import { analyzeBundleStatic } from './analysis.js';
 import { autoDetectTags, generateDisplayName, generateDescription } from './tagging.js';
+import { bundleCreationLimiter } from '../core/concurrency-limiter.js';
+const DEDUP_INDEX_FILE = '.preflight-dedup-index.json';
+function sha256Hex(text) {
+    return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
+}
+function normalizeList(values) {
+    return (values ?? [])
+        .map((s) => s.trim())
+        .filter(Boolean)
+        .map((s) => s.toLowerCase())
+        .sort();
+}
+function normalizeDeepWikiUrl(raw) {
+    const trimmed = raw.trim();
+    try {
+        const u = new URL(trimmed);
+        u.hash = '';
+        // Normalize host and strip trailing slash.
+        u.host = u.host.toLowerCase();
+        u.pathname = u.pathname.replace(/\/+$/g, '');
+        return u.toString();
+    }
+    catch {
+        return trimmed;
+    }
+}
+function canonicalizeCreateInput(input) {
+    const repos = input.repos
+        .map((r) => {
+        if (r.kind === 'github') {
+            const { owner, repo } = parseOwnerRepo(r.repo);
+            return {
+                kind: 'github',
+                repo: `${owner.toLowerCase()}/${repo.toLowerCase()}`,
+                ref: (r.ref ?? '').trim() || undefined,
+            };
+        }
+        if (r.kind === 'local') {
+            // For de-duplication, treat local imports as equivalent to github imports of the same logical repo/ref.
+            const { owner, repo } = parseOwnerRepo(r.repo);
+            return {
+                kind: 'github',
+                repo: `${owner.toLowerCase()}/${repo.toLowerCase()}`,
+                ref: (r.ref ?? '').trim() || undefined,
+            };
+        }
+        return {
+            kind: 'deepwiki',
+            url: normalizeDeepWikiUrl(r.url),
+        };
+    })
+        .sort((a, b) => {
+        const ka = a.kind === 'github' ? `github:${a.repo}:${a.ref ?? ''}` : `deepwiki:${a.url}`;
+        const kb = b.kind === 'github' ? `github:${b.repo}:${b.ref ?? ''}` : `deepwiki:${b.url}`;
+        return ka.localeCompare(kb);
+    });
+    return {
+        schemaVersion: 1,
+        repos,
+        libraries: normalizeList(input.libraries),
+        topics: normalizeList(input.topics),
+    };
+}
+export function computeCreateInputFingerprint(input) {
+    const canonical = canonicalizeCreateInput(input);
+    return sha256Hex(JSON.stringify(canonical));
+}
+function dedupIndexPath(storageDir) {
+    return path.join(storageDir, DEDUP_INDEX_FILE);
+}
+async function readDedupIndex(storageDir) {
+    const p = dedupIndexPath(storageDir);
+    try {
+        const raw = await fs.readFile(p, 'utf8');
+        const parsed = JSON.parse(raw);
+        if (parsed.schemaVersion !== 1 || typeof parsed.byFingerprint !== 'object' || !parsed.byFingerprint) {
+            return { schemaVersion: 1, updatedAt: nowIso(), byFingerprint: {} };
+        }
+        return parsed;
+    }
+    catch {
+        return { schemaVersion: 1, updatedAt: nowIso(), byFingerprint: {} };
+    }
+}
+async function writeDedupIndex(storageDir, idx) {
+    const p = dedupIndexPath(storageDir);
+    await ensureDir(path.dirname(p));
+    // Use atomic write (write to temp file, then rename) to prevent corruption
+    const tmpPath = `${p}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`;
+    try {
+        await fs.writeFile(tmpPath, JSON.stringify(idx, null, 2) + '\n', 'utf8');
+        // Atomic rename on POSIX; near-atomic on Windows
+        await fs.rename(tmpPath, p);
+    }
+    catch (err) {
+        // Clean up temp file on error
+        try {
+            await fs.unlink(tmpPath);
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+        throw err;
+    }
+}
+async function updateDedupIndexBestEffort(cfg, fingerprint, bundleId, bundleUpdatedAt) {
+    for (const storageDir of cfg.storageDirs) {
+        try {
+            const parentAvailable = await isParentAvailable(storageDir);
+            if (!parentAvailable)
+                continue;
+            await ensureDir(storageDir);
+            const idx = await readDedupIndex(storageDir);
+            idx.byFingerprint[fingerprint] = { bundleId, bundleUpdatedAt };
+            idx.updatedAt = nowIso();
+            await writeDedupIndex(storageDir, idx);
+        }
+        catch {
+            // best-effort
+        }
+    }
+}
+async function readBundleSummary(cfg, bundleId) {
+    const storageDir = (await findBundleStorageDir(cfg.storageDirs, bundleId)) ?? (await getEffectiveStorageDir(cfg));
+    const paths = getBundlePaths(storageDir, bundleId);
+    const manifest = await readManifest(paths.manifestPath);
+    return {
+        bundleId: manifest.bundleId,
+        createdAt: manifest.createdAt,
+        updatedAt: manifest.updatedAt,
+        repos: manifest.repos.map((r) => ({
+            kind: r.kind,
+            id: r.id,
+            source: r.source,
+            headSha: r.headSha,
+            notes: r.notes,
+        })),
+        libraries: manifest.libraries,
+    };
+}
+export async function findBundleByInputs(cfg, input) {
+    const fingerprint = computeCreateInputFingerprint(input);
+    return findExistingBundleByFingerprint(cfg, fingerprint);
+}
+async function findExistingBundleByFingerprint(cfg, fingerprint) {
+    // Fast path: consult any available dedup index.
+    for (const storageDir of cfg.storageDirs) {
+        try {
+            if (!(await isPathAvailable(storageDir)))
+                continue;
+            const idx = await readDedupIndex(storageDir);
+            const hit = idx.byFingerprint[fingerprint];
+            if (hit?.bundleId && (await bundleExistsMulti(cfg.storageDirs, hit.bundleId))) {
+                return hit.bundleId;
+            }
+        }
+        catch {
+            // ignore
+        }
+    }
+    // Slow path: scan manifests (works even for bundles created before fingerprints existed).
+    let best = null;
+    for (const storageDir of cfg.storageDirs) {
+        if (!(await isPathAvailable(storageDir)))
+            continue;
+        const ids = await listBundles(storageDir);
+        for (const id of ids) {
+            try {
+                const paths = getBundlePaths(storageDir, id);
+                const manifest = await readManifest(paths.manifestPath);
+                const fp = computeCreateInputFingerprint({
+                    repos: manifest.inputs.repos,
+                    libraries: manifest.inputs.libraries,
+                    topics: manifest.inputs.topics,
+                });
+                if (fp === fingerprint) {
+                    const updatedAt = manifest.updatedAt;
+                    if (!best || new Date(updatedAt) > new Date(best.updatedAt)) {
+                        best = { bundleId: manifest.bundleId, updatedAt };
+                    }
+                }
+            }
+            catch {
+                // ignore corrupt bundles
+            }
+        }
+    }
+    if (best) {
+        // Seed index for next time (best-effort).
+        await updateDedupIndexBestEffort(cfg, fingerprint, best.bundleId, best.updatedAt);
+        return best.bundleId;
+    }
+    return null;
+}
 async function ensureDir(p) {
     await fs.mkdir(p, { recursive: true });
 }
 function nowIso() {
     return new Date().toISOString();
 }
+function toPosix(p) {
+    return p.replaceAll('\\', '/');
+}
+function sha256Text(text) {
+    return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
+}
+async function statOrNull(p) {
+    try {
+        return await fs.stat(p);
+    }
+    catch {
+        return null;
+    }
+}
+async function readUtf8OrNull(p) {
+    try {
+        return await fs.readFile(p, 'utf8');
+    }
+    catch {
+        return null;
+    }
+}
+async function* walkFilesNoIgnore(rootDir) {
+    const stack = [rootDir];
+    while (stack.length) {
+        const dir = stack.pop();
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (const ent of entries) {
+            const abs = path.join(dir, ent.name);
+            const rel = toPosix(path.relative(rootDir, abs));
+            if (ent.isDirectory()) {
+                stack.push(abs);
+                continue;
+            }
+            if (!ent.isFile())
+                continue;
+            yield { absPath: abs, relPosix: rel };
+        }
+    }
+}
 async function writeRepoMeta(params) {
     await ensureDir(path.dirname(params.metaPath));
     const obj = {
         repoId: params.repoId,
         cloneUrl: params.cloneUrl,
-        headSha: params.headSha,
         fetchedAt: params.fetchedAt,
         ingestedFiles: params.ingestedFiles,
         skipped: params.skipped,
     };
+    if (params.headSha)
+        obj.headSha = params.headSha;
+    if (params.source)
+        obj.source = params.source;
+    if (params.ref)
+        obj.ref = params.ref;
     await fs.writeFile(params.metaPath, JSON.stringify(obj, null, 2) + '\n', 'utf8');
 }
 async function rmIfExists(p) {
@@ -205,26 +445,47 @@ async function mirrorBundleToBackups(primaryDir, backupDirs, bundleId) {
     const srcPath = path.join(primaryDir, bundleId);
     const mirrored = [];
     const failed = [];
-    for (const backupDir of backupDirs) {
-        if (backupDir === primaryDir)
-            continue; // Skip primary
+    // Mirror to all backup dirs in parallel for better performance
+    const mirrorPromises = backupDirs
+        .filter(dir => dir !== primaryDir) // Skip primary
+        .map(async (backupDir) => {
         const destPath = path.join(backupDir, bundleId);
         try {
             // Check if backup location is available
             const parentAvailable = await isParentAvailable(destPath);
             if (!parentAvailable) {
-                failed.push({ path: backupDir, error: 'Mount not available' });
-                continue;
+                return { success: false, path: backupDir, error: 'Mount not available' };
             }
             // Ensure backup dir exists
             await ensureDir(backupDir);
             // Remove old and copy new
             await rmIfExists(destPath);
             await copyDir(srcPath, destPath);
-            mirrored.push(backupDir);
+            return { success: true, path: backupDir };
         }
         catch (err) {
-            failed.push({ path: backupDir, error: err instanceof Error ? err.message : String(err) });
+            return {
+                success: false,
+                path: backupDir,
+                error: err instanceof Error ? err.message : String(err)
+            };
+        }
+    });
+    // Wait for all mirror operations to complete
+    const results = await Promise.allSettled(mirrorPromises);
+    for (const result of results) {
+        if (result.status === 'fulfilled') {
+            const { success, path: backupPath, error } = result.value;
+            if (success) {
+                mirrored.push(backupPath);
+            }
+            else {
+                failed.push({ path: backupPath, error: error ?? 'Unknown error' });
+            }
+        }
+        else {
+            // Promise rejection (shouldn't happen with try-catch, but handle it)
+            failed.push({ path: 'unknown', error: result.reason?.message ?? String(result.reason) });
         }
     }
     return { mirrored, failed };
@@ -330,13 +591,91 @@ async function syncStaleBackups(sourceDir, allDirs, bundleId) {
         }
     }
 }
+async function writeLocalRepoMeta(params) {
+    await ensureDir(path.dirname(params.metaPath));
+    const obj = {
+        repoId: params.repoId,
+        source: 'local',
+        localPath: params.localPath,
+        ref: params.ref,
+        fetchedAt: params.fetchedAt,
+        ingestedFiles: params.ingestedFiles,
+        skipped: params.skipped,
+    };
+    await fs.writeFile(params.metaPath, JSON.stringify(obj, null, 2) + '\n', 'utf8');
+}
+async function ingestLocalRepo(params) {
+    const repoId = `${params.owner}/${params.repo}`;
+    const repoRoot = path.resolve(params.localPath);
+    const st = await fs.stat(repoRoot);
+    if (!st.isDirectory()) {
+        throw new Error(`Local repo path is not a directory: ${repoRoot}`);
+    }
+    const bundlePaths = getBundlePaths(params.storageDir, params.bundleId);
+    const rawDest = repoRawDir(bundlePaths, params.owner, params.repo);
+    const normDest = repoNormDir(bundlePaths, params.owner, params.repo);
+    await rmIfExists(rawDest);
+    await rmIfExists(normDest);
+    await ensureDir(rawDest);
+    await ensureDir(normDest);
+    const bundleNormPrefixPosix = `repos/${params.owner}/${params.repo}/norm`;
+    const ingested = await ingestRepoToBundle({
+        repoId,
+        repoRoot,
+        rawDestRoot: rawDest,
+        normDestRoot: normDest,
+        bundleNormPrefixPosix,
+        options: {
+            maxFileBytes: params.cfg.maxFileBytes,
+            maxTotalBytes: params.cfg.maxTotalBytes,
+        },
+    });
+    const fetchedAt = nowIso();
+    await writeLocalRepoMeta({
+        metaPath: repoMetaPath(bundlePaths, params.owner, params.repo),
+        repoId,
+        localPath: repoRoot,
+        ref: params.ref,
+        fetchedAt,
+        ingestedFiles: ingested.files.length,
+        skipped: ingested.skipped,
+    });
+    return { files: ingested.files, skipped: ingested.skipped };
+}
 async function cloneAndIngestGitHubRepo(params) {
     const repoId = `${params.owner}/${params.repo}`;
     const cloneUrl = toCloneUrl({ owner: params.owner, repo: params.repo });
-    const tmpCheckout = path.join(params.cfg.tmpDir, 'checkouts', params.bundleId, `${params.owner}__${params.repo}`);
-    await rmIfExists(tmpCheckout);
-    await shallowClone(cloneUrl, tmpCheckout, { ref: params.ref });
-    const headSha = await getLocalHeadSha(tmpCheckout);
+    const tmpBase = path.join(params.cfg.tmpDir, 'checkouts', params.bundleId, `${params.owner}__${params.repo}`);
+    const tmpCheckoutGit = tmpBase;
+    const tmpArchiveDir = `${tmpBase}__archive`;
+    await rmIfExists(tmpCheckoutGit);
+    await rmIfExists(tmpArchiveDir);
+    let repoRootForIngest = tmpCheckoutGit;
+    let headSha;
+    const notes = [];
+    let source = 'git';
+    let fetchedAt = nowIso();
+    let refUsed = params.ref;
+    try {
+        await shallowClone(cloneUrl, tmpCheckoutGit, { ref: params.ref, timeoutMs: params.cfg.gitCloneTimeoutMs });
+        headSha = await getLocalHeadSha(tmpCheckoutGit);
+    }
+    catch (err) {
+        // Fallback: GitHub archive download (zipball) + extract.
+        source = 'archive';
+        const msg = err instanceof Error ? err.message : String(err);
+        notes.push(`git clone failed; used GitHub archive fallback: ${msg}`);
+        const archive = await downloadAndExtractGitHubArchive({
+            cfg: params.cfg,
+            owner: params.owner,
+            repo: params.repo,
+            ref: params.ref,
+            destDir: tmpArchiveDir,
+        });
+        repoRootForIngest = archive.repoRoot;
+        fetchedAt = archive.fetchedAt;
+        refUsed = archive.refUsed;
+    }
     const bundlePaths = getBundlePaths(params.storageDir, params.bundleId);
     const rawDest = repoRawDir(bundlePaths, params.owner, params.repo);
     const normDest = repoNormDir(bundlePaths, params.owner, params.repo);
@@ -347,7 +686,7 @@ async function cloneAndIngestGitHubRepo(params) {
     const bundleNormPrefixPosix = `repos/${params.owner}/${params.repo}/norm`;
     const ingested = await ingestRepoToBundle({
         repoId,
-        repoRoot: tmpCheckout,
+        repoRoot: repoRootForIngest,
         rawDestRoot: rawDest,
         normDestRoot: normDest,
         bundleNormPrefixPosix,
@@ -356,7 +695,6 @@ async function cloneAndIngestGitHubRepo(params) {
             maxTotalBytes: params.cfg.maxTotalBytes,
         },
     });
-    const fetchedAt = nowIso();
     await writeRepoMeta({
         metaPath: repoMetaPath(bundlePaths, params.owner, params.repo),
         repoId,
@@ -365,9 +703,12 @@ async function cloneAndIngestGitHubRepo(params) {
         fetchedAt,
         ingestedFiles: ingested.files.length,
         skipped: ingested.skipped,
+        source,
+        ref: refUsed,
     });
-    await rmIfExists(tmpCheckout);
-    return { headSha, files: ingested.files, skipped: ingested.skipped };
+    await rmIfExists(tmpCheckoutGit);
+    await rmIfExists(tmpArchiveDir);
+    return { headSha, files: ingested.files, skipped: ingested.skipped, notes, source };
 }
 function groupFilesByRepoId(files) {
     const byRepo = new Map();
@@ -401,7 +742,28 @@ async function generateFactsBestEffort(params) {
         logger.error('Static analysis exception', err instanceof Error ? err : undefined);
     }
 }
-export async function createBundle(cfg, input) {
+export async function createBundle(cfg, input, options) {
+    // Apply concurrency limiting to prevent DoS attacks
+    return await bundleCreationLimiter.run(async () => {
+        return await createBundleInternal(cfg, input, options);
+    });
+}
+async function createBundleInternal(cfg, input, options) {
+    const fingerprint = computeCreateInputFingerprint(input);
+    const ifExists = options?.ifExists ?? 'error';
+    if (ifExists !== 'createNew') {
+        const existing = await findExistingBundleByFingerprint(cfg, fingerprint);
+        if (existing) {
+            if (ifExists === 'returnExisting') {
+                return await readBundleSummary(cfg, existing);
+            }
+            if (ifExists === 'updateExisting') {
+                const { summary } = await updateBundle(cfg, existing);
+                return summary;
+            }
+            throw new Error(`Bundle already exists for these inputs: ${existing}`);
+        }
+    }
     const bundleId = crypto.randomUUID();
     const createdAt = nowIso();
     // Use effective storage dir (falls back if primary unavailable)
@@ -417,7 +779,7 @@ export async function createBundle(cfg, input) {
         for (const repoInput of input.repos) {
             if (repoInput.kind === 'github') {
                 const { owner, repo } = parseOwnerRepo(repoInput.repo);
-                const { headSha, files, skipped } = await cloneAndIngestGitHubRepo({
+                const { headSha, files, skipped, notes, source } = await cloneAndIngestGitHubRepo({
                     cfg,
                     bundleId,
                     storageDir: effectiveStorageDir,
@@ -426,7 +788,27 @@ export async function createBundle(cfg, input) {
                     ref: repoInput.ref,
                 });
                 allIngestedFiles.push(...files);
-                reposSummary.push({ kind: 'github', id: `${owner}/${repo}`, headSha, notes: skipped.slice(0, 50) });
+                reposSummary.push({
+                    kind: 'github',
+                    id: `${owner}/${repo}`,
+                    source,
+                    headSha,
+                    notes: [...notes, ...skipped].slice(0, 50),
+                });
+            }
+            else if (repoInput.kind === 'local') {
+                const { owner, repo } = parseOwnerRepo(repoInput.repo);
+                const { files, skipped } = await ingestLocalRepo({
+                    cfg,
+                    bundleId,
+                    storageDir: effectiveStorageDir,
+                    owner,
+                    repo,
+                    localPath: repoInput.path,
+                    ref: repoInput.ref,
+                });
+                allIngestedFiles.push(...files);
+                reposSummary.push({ kind: 'local', id: `${owner}/${repo}`, source: 'local', notes: skipped.slice(0, 50) });
             }
             else {
                 // DeepWiki integration: fetch and convert to Markdown.
@@ -439,6 +821,7 @@ export async function createBundle(cfg, input) {
                 reposSummary.push({
                     kind: 'deepwiki',
                     id: deepwikiResult.summary.repoId,
+                    source: 'deepwiki',
                     notes: deepwikiResult.summary.notes,
                 });
             }
@@ -482,6 +865,7 @@ export async function createBundle(cfg, input) {
             bundleId,
             createdAt,
             updatedAt: createdAt,
+            fingerprint,
             displayName,
             description,
             tags,
@@ -494,6 +878,7 @@ export async function createBundle(cfg, input) {
             repos: reposSummary.map((r) => ({
                 kind: r.kind,
                 id: r.id,
+                source: r.source,
                 headSha: r.headSha,
                 fetchedAt: createdAt,
                 notes: r.notes,
@@ -516,7 +901,7 @@ export async function createBundle(cfg, input) {
         });
         // Overview (S2: factual-only with evidence pointers).
         const perRepoOverviews = reposSummary
-            .filter((r) => r.kind === 'github')
+            .filter((r) => r.kind === 'github' || r.kind === 'local')
             .map((r) => {
             const repoId = r.id;
             const repoFiles = allIngestedFiles.filter((f) => f.repoId === repoId);
@@ -547,6 +932,8 @@ export async function createBundle(cfg, input) {
             logger.error(errorMsg);
             throw new Error(errorMsg);
         }
+        // Update de-duplication index (best-effort). This is intentionally after validation.
+        await updateDedupIndexBestEffort(cfg, fingerprint, bundleId, createdAt);
         const summary = {
             bundleId,
             createdAt,
@@ -592,6 +979,14 @@ export async function checkForUpdates(cfg, bundleId) {
                 hasUpdates = true;
             details.push({ repoId, currentSha: prev?.headSha, remoteSha, changed });
         }
+        else if (repoInput.kind === 'local') {
+            const { owner, repo } = parseOwnerRepo(repoInput.repo);
+            const repoId = `${owner}/${repo}`;
+            // We can't reliably detect whether local files changed without scanning; assume possible update.
+            const prev = manifest.repos.find((r) => r.id === repoId);
+            details.push({ repoId, currentSha: prev?.headSha, changed: true });
+            hasUpdates = true;
+        }
         else {
             // DeepWiki: can't easily detect changes, assume possible update
             details.push({ repoId: repoInput.url, changed: true });
@@ -600,6 +995,257 @@ export async function checkForUpdates(cfg, bundleId) {
     }
     return { hasUpdates, details };
 }
+async function scanBundleIndexableFiles(params) {
+    const files = [];
+    const skipped = [];
+    let totalBytes = 0;
+    const pushFile = async (f) => {
+        const st = await statOrNull(f.absPath);
+        if (!st?.isFile())
+            return;
+        if (st.size > params.cfg.maxFileBytes) {
+            skipped.push(`${f.bundleRelPosix} (too large: ${st.size} bytes)`);
+            return;
+        }
+        if (totalBytes + st.size > params.cfg.maxTotalBytes) {
+            skipped.push(`(bundle maxTotalBytes reached) stopped before: ${f.bundleRelPosix}`);
+            return;
+        }
+        const text = await readUtf8OrNull(f.absPath);
+        if (text === null) {
+            skipped.push(`${f.bundleRelPosix} (unreadable utf8)`);
+            return;
+        }
+        const normalized = text.replace(/\r\n/g, '\n');
+        const sha256 = sha256Text(normalized);
+        totalBytes += st.size;
+        files.push({
+            repoId: f.repoId,
+            kind: f.kind,
+            repoRelativePath: f.repoRelativePath,
+            bundleNormRelativePath: f.bundleRelPosix,
+            bundleNormAbsPath: f.absPath,
+            sha256,
+            bytes: st.size,
+        });
+    };
+    // 1) repos/<owner>/<repo>/norm/** (github/local)
+    try {
+        const owners = await fs.readdir(params.reposDir, { withFileTypes: true });
+        for (const ownerEnt of owners) {
+            if (!ownerEnt.isDirectory())
+                continue;
+            const owner = ownerEnt.name;
+            const ownerDir = path.join(params.reposDir, owner);
+            const repos = await fs.readdir(ownerDir, { withFileTypes: true });
+            for (const repoEnt of repos) {
+                if (!repoEnt.isDirectory())
+                    continue;
+                const repo = repoEnt.name;
+                const normDir = path.join(ownerDir, repo, 'norm');
+                const normSt = await statOrNull(normDir);
+                if (!normSt?.isDirectory())
+                    continue;
+                for await (const wf of walkFilesNoIgnore(normDir)) {
+                    const repoRel = wf.relPosix;
+                    const kind = classifyIngestedFileKind(repoRel);
+                    const bundleRel = `repos/${owner}/${repo}/norm/${repoRel}`;
+                    await pushFile({
+                        repoId: `${owner}/${repo}`,
+                        kind,
+                        repoRelativePath: repoRel,
+                        bundleRelPosix: bundleRel,
+                        absPath: wf.absPath,
+                    });
+                }
+            }
+        }
+    }
+    catch {
+        // ignore missing repos dir
+    }
+    // 2) libraries/context7/** (docs-only)
+    const context7Dir = path.join(params.librariesDir, 'context7');
+    const ctxSt = await statOrNull(context7Dir);
+    if (ctxSt?.isDirectory()) {
+        for await (const wf of walkFilesNoIgnore(context7Dir)) {
+            // Match original ingestion: only .md docs are indexed from Context7.
+            if (!wf.relPosix.toLowerCase().endsWith('.md'))
+                continue;
+            const relFromLibRoot = wf.relPosix; // relative to libraries/context7
+            const parts = relFromLibRoot.split('/').filter(Boolean);
+            const fileName = parts[parts.length - 1] ?? '';
+            const dirParts = parts.slice(0, -1);
+            let repoId = 'context7:unknown';
+            if (dirParts[0] === '_unresolved' && dirParts[1]) {
+                repoId = `context7:unresolved/${dirParts[1]}`;
+            }
+            else if (dirParts.length > 0) {
+                repoId = `context7:/${dirParts.join('/')}`;
+            }
+            const bundleRel = `libraries/context7/${relFromLibRoot}`;
+            await pushFile({
+                repoId,
+                kind: 'doc',
+                repoRelativePath: fileName,
+                bundleRelPosix: bundleRel,
+                absPath: wf.absPath,
+            });
+        }
+    }
+    // 3) deepwiki/<owner>/<repo>/norm/** (docs-only)
+    const deepwikiDir = path.join(params.bundleRootDir, 'deepwiki');
+    const dwSt = await statOrNull(deepwikiDir);
+    if (dwSt?.isDirectory()) {
+        // Only walk the norm subtrees.
+        const owners = await fs.readdir(deepwikiDir, { withFileTypes: true });
+        for (const ownerEnt of owners) {
+            if (!ownerEnt.isDirectory())
+                continue;
+            const owner = ownerEnt.name;
+            const ownerDir = path.join(deepwikiDir, owner);
+            const repos = await fs.readdir(ownerDir, { withFileTypes: true });
+            for (const repoEnt of repos) {
+                if (!repoEnt.isDirectory())
+                    continue;
+                const repo = repoEnt.name;
+                const normDir = path.join(ownerDir, repo, 'norm');
+                const normSt = await statOrNull(normDir);
+                if (!normSt?.isDirectory())
+                    continue;
+                for await (const wf of walkFilesNoIgnore(normDir)) {
+                    if (!wf.relPosix.toLowerCase().endsWith('.md'))
+                        continue;
+                    const bundleRel = `deepwiki/${owner}/${repo}/norm/${wf.relPosix}`;
+                    await pushFile({
+                        repoId: `deepwiki:${owner}/${repo}`,
+                        kind: 'doc',
+                        repoRelativePath: wf.relPosix,
+                        bundleRelPosix: bundleRel,
+                        absPath: wf.absPath,
+                    });
+                }
+            }
+        }
+    }
+    return { files, totalBytes, skipped };
+}
+export async function repairBundle(cfg, bundleId, options) {
+    const mode = options?.mode ?? 'repair';
+    const rebuildIndexOpt = options?.rebuildIndex ?? true;
+    const rebuildGuidesOpt = options?.rebuildGuides ?? true;
+    const rebuildOverviewOpt = options?.rebuildOverview ?? true;
+    const storageDir = await findBundleStorageDir(cfg.storageDirs, bundleId);
+    if (!storageDir) {
+        throw new Error(`Bundle not found: ${bundleId}`);
+    }
+    const paths = getBundlePaths(storageDir, bundleId);
+    const before = await validateBundleCompleteness(paths.rootDir);
+    if (mode === 'validate') {
+        return {
+            bundleId,
+            mode,
+            repaired: false,
+            actionsTaken: [],
+            before,
+            after: before,
+        };
+    }
+    // Manifest is required for safe repairs (no fetching/re-ingest).
+    const manifest = await readManifest(paths.manifestPath);
+    const actionsTaken = [];
+    // Determine what needs repair.
+    const stAgents = await statOrNull(paths.agentsPath);
+    const stStartHere = await statOrNull(paths.startHerePath);
+    const stOverview = await statOrNull(paths.overviewPath);
+    const stIndex = await statOrNull(paths.searchDbPath);
+    const needsAgents = !stAgents || stAgents.size === 0;
+    const needsStartHere = !stStartHere || stStartHere.size === 0;
+    const needsOverview = !stOverview || stOverview.size === 0;
+    const needsIndex = !stIndex || stIndex.size === 0;
+    // Scan bundle files once if needed for index/overview.
+    let scanned = null;
+    const needScan = (rebuildIndexOpt && needsIndex) || (rebuildOverviewOpt && needsOverview);
+    if (needScan) {
+        scanned = await scanBundleIndexableFiles({
+            cfg,
+            bundleRootDir: paths.rootDir,
+            reposDir: paths.reposDir,
+            librariesDir: paths.librariesDir,
+        });
+        if (scanned.skipped.length) {
+            actionsTaken.push(`scan: skipped ${scanned.skipped.length} file(s)`);
+        }
+    }
+    if (rebuildIndexOpt && needsIndex) {
+        const files = scanned?.files ?? [];
+        await rebuildIndex(paths.searchDbPath, files, { includeDocs: true, includeCode: true });
+        actionsTaken.push(`rebuildIndex: indexed ${files.length} file(s)`);
+    }
+    if (rebuildGuidesOpt && needsAgents) {
+        await writeAgentsMd(paths.agentsPath);
+        actionsTaken.push('writeAgentsMd');
+    }
+    if (rebuildGuidesOpt && needsStartHere) {
+        await writeStartHereMd({
+            targetPath: paths.startHerePath,
+            bundleId,
+            repos: (manifest.repos ?? []).map((r) => ({ id: r.id, headSha: r.headSha })),
+            libraries: manifest.libraries,
+        });
+        actionsTaken.push('writeStartHereMd');
+    }
+    if (rebuildOverviewOpt && needsOverview) {
+        const allFiles = scanned?.files ?? [];
+        const perRepoOverviews = (manifest.repos ?? [])
+            .filter((r) => r.kind === 'github' || r.kind === 'local')
+            .map((r) => {
+            const repoId = r.id;
+            const repoFiles = allFiles.filter((f) => f.repoId === repoId);
+            return { repoId, headSha: r.headSha, files: repoFiles };
+        });
+        const md = await generateOverviewMarkdown({
+            bundleId,
+            bundleRootDir: paths.rootDir,
+            repos: perRepoOverviews,
+            libraries: manifest.libraries,
+        });
+        await writeOverviewFile(paths.overviewPath, md);
+        actionsTaken.push('writeOverviewFile');
+    }
+    let updatedAt;
+    if (actionsTaken.length > 0) {
+        updatedAt = nowIso();
+        const fingerprint = manifest.fingerprint ??
+            computeCreateInputFingerprint({
+                repos: manifest.inputs.repos,
+                libraries: manifest.inputs.libraries,
+                topics: manifest.inputs.topics,
+            });
+        const newManifest = {
+            ...manifest,
+            updatedAt,
+            fingerprint,
+        };
+        await writeManifest(paths.manifestPath, newManifest);
+        // Keep the de-duplication index fresh (best-effort).
+        await updateDedupIndexBestEffort(cfg, fingerprint, bundleId, updatedAt);
+        // Mirror to backup storage directories (non-blocking on failures)
+        if (cfg.storageDirs.length > 1) {
+            await mirrorBundleToBackups(storageDir, cfg.storageDirs, bundleId);
+        }
+    }
+    const after = await validateBundleCompleteness(paths.rootDir);
+    return {
+        bundleId,
+        mode,
+        repaired: actionsTaken.length > 0,
+        actionsTaken,
+        before,
+        after,
+        updatedAt,
+    };
+}
 export async function updateBundle(cfg, bundleId, options) {
     // Use effective storage dir (falls back if primary unavailable)
     const effectiveStorageDir = await getEffectiveStorageDirForWrite(cfg);
@@ -626,7 +1272,7 @@ export async function updateBundle(cfg, bundleId, options) {
             if (remoteSha && prev?.headSha && remoteSha !== prev.headSha) {
                 changed = true;
             }
-            const { headSha, files, skipped } = await cloneAndIngestGitHubRepo({
+            const { headSha, files, skipped, notes, source } = await cloneAndIngestGitHubRepo({
                 cfg,
                 bundleId,
                 storageDir: effectiveStorageDir,
@@ -634,11 +1280,31 @@ export async function updateBundle(cfg, bundleId, options) {
                 repo,
                 ref: repoInput.ref,
             });
-            if (prev?.headSha && headSha !== prev.headSha) {
+            if (prev?.headSha && headSha && headSha !== prev.headSha) {
+                changed = true;
+            }
+            // If we had to fall back to an archive, treat as changed (we don't have git metadata).
+            if (source === 'archive') {
                 changed = true;
             }
             allIngestedFiles.push(...files);
-            reposSummary.push({ kind: 'github', id: repoId, headSha, notes: skipped.slice(0, 50) });
+            reposSummary.push({ kind: 'github', id: repoId, source, headSha, notes: [...notes, ...skipped].slice(0, 50) });
+        }
+        else if (repoInput.kind === 'local') {
+            const { owner, repo } = parseOwnerRepo(repoInput.repo);
+            const repoId = `${owner}/${repo}`;
+            const { files, skipped } = await ingestLocalRepo({
+                cfg,
+                bundleId,
+                storageDir: effectiveStorageDir,
+                owner,
+                repo,
+                localPath: repoInput.path,
+                ref: repoInput.ref,
+            });
+            allIngestedFiles.push(...files);
+            reposSummary.push({ kind: 'local', id: repoId, source: 'local', notes: skipped.slice(0, 50) });
+            changed = true;
         }
         else {
             // DeepWiki integration: fetch and convert to Markdown.
@@ -651,6 +1317,7 @@ export async function updateBundle(cfg, bundleId, options) {
             reposSummary.push({
                 kind: 'deepwiki',
                 id: deepwikiResult.summary.repoId,
+                source: 'deepwiki',
                 notes: deepwikiResult.summary.notes,
             });
             // Always mark as changed for DeepWiki since we can't easily detect content changes.
@@ -676,12 +1343,19 @@ export async function updateBundle(cfg, bundleId, options) {
         includeDocs: manifest.index.includeDocs,
         includeCode: manifest.index.includeCode,
     });
+    const fingerprint = computeCreateInputFingerprint({
+        repos: manifest.inputs.repos,
+        libraries: manifest.inputs.libraries,
+        topics: manifest.inputs.topics,
+    });
     const newManifest = {
         ...manifest,
         updatedAt,
+        fingerprint,
         repos: reposSummary.map((r) => ({
             kind: r.kind,
             id: r.id,
+            source: r.source,
             headSha: r.headSha,
             fetchedAt: updatedAt,
             notes: r.notes,
@@ -698,7 +1372,7 @@ export async function updateBundle(cfg, bundleId, options) {
         libraries: librariesSummary,
     });
     const perRepoOverviews = reposSummary
-        .filter((r) => r.kind === 'github')
+        .filter((r) => r.kind === 'github' || r.kind === 'local')
         .map((r) => {
         const repoId = r.id;
         const repoFiles = allIngestedFiles.filter((f) => f.repoId === repoId);
@@ -722,6 +1396,8 @@ export async function updateBundle(cfg, bundleId, options) {
     if (cfg.storageDirs.length > 1) {
         await mirrorBundleToBackups(effectiveStorageDir, cfg.storageDirs, bundleId);
     }
+    // Keep the de-duplication index fresh (best-effort).
+    await updateDedupIndexBestEffort(cfg, fingerprint, bundleId, updatedAt);
     const summary = {
         bundleId,
         createdAt: manifest.createdAt,