preflight-mcp 0.1.2 → 0.1.4

package/dist/bundle/github.js

@@ -13,8 +13,26 @@ export function parseOwnerRepo(input) {
 export function toCloneUrl(ref) {
     return `https://github.com/${ref.owner}/${ref.repo}.git`;
 }
+/**
+ * Parse git clone progress from stderr.
+ * Git outputs progress like:
+ * - "Receiving objects:  45% (1234/2741)"
+ * - "Resolving deltas:  60% (100/167)"
+ */
+function parseGitProgress(line) {
+    // Match patterns like "Receiving objects:  45% (1234/2741)"
+    const match = line.match(/(Receiving objects|Resolving deltas|Counting objects|Compressing objects):\s+(\d+)%/);
+    if (match) {
+        return {
+            phase: match[1],
+            percent: parseInt(match[2], 10),
+        };
+    }
+    return null;
+}
 async function runGit(args, opts) {
     const timeoutMs = opts?.timeoutMs ?? 5 * 60_000;
+    const onProgress = opts?.onProgress;
     return new Promise((resolve, reject) => {
         const child = spawn('git', args, {
             cwd: opts?.cwd,
@@ -65,7 +83,19 @@ async function runGit(args, opts) {
             stdout += data.toString('utf8');
         });
         child.stderr?.on('data', (data) => {
-            stderr += data.toString('utf8');
+            const chunk = data.toString('utf8');
+            stderr += chunk;
+            // Parse and report progress
+            if (onProgress) {
+                // Git progress can come in chunks, split by lines
+                const lines = chunk.split(/[\r\n]+/);
+                for (const line of lines) {
+                    const progress = parseGitProgress(line);
+                    if (progress) {
+                        onProgress(progress.phase, progress.percent, `${progress.phase}: ${progress.percent}%`);
+                    }
+                }
+            }
         });
         child.on('error', (err) => {
             cleanup();
@@ -125,14 +155,15 @@ export async function shallowClone(cloneUrl, destDir, opts) {
     await fs.mkdir(path.dirname(destDir), { recursive: true });
     // Clean dest if exists.
     await fs.rm(destDir, { recursive: true, force: true });
-    const args = ['-c', 'core.autocrlf=false', 'clone', '--depth', '1', '--no-tags', '--single-branch'];
+    // Use --progress to force progress output even when not attached to a terminal
+    const args = ['-c', 'core.autocrlf=false', 'clone', '--depth', '1', '--no-tags', '--single-branch', '--progress'];
     if (opts?.ref) {
         // Validate ref before using it in git command
         validateGitRef(opts.ref);
         args.push('--branch', opts.ref);
     }
     args.push(cloneUrl, destDir);
-    await runGit(args, { timeoutMs: opts?.timeoutMs ?? 15 * 60_000 });
+    await runGit(args, { timeoutMs: opts?.timeoutMs ?? 15 * 60_000, onProgress: opts?.onProgress });
 }
 export async function getLocalHeadSha(repoDir) {
     const { stdout } = await runGit(['-C', repoDir, 'rev-parse', 'HEAD']);

package/dist/bundle/githubArchive.js

@@ -1,12 +1,13 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import AdmZip from 'adm-zip';
+import { logger } from '../logging/logger.js';
 function nowIso() {
     return new Date().toISOString();
 }
 function githubHeaders(cfg) {
     const headers = {
-        'User-Agent': 'preflight-mcp/0.1.1',
+        'User-Agent': 'preflight-mcp/0.1.3',
         Accept: 'application/vnd.github+json',
     };
     if (cfg.githubToken) {
@@ -17,36 +18,106 @@ function githubHeaders(cfg) {
 async function ensureDir(p) {
     await fs.mkdir(p, { recursive: true });
 }
-async function fetchJson(url, headers) {
-    const res = await fetch(url, { headers });
-    if (!res.ok) {
-        throw new Error(`GitHub API error ${res.status}: ${res.statusText}`);
+/** Default timeout for GitHub API requests (30 seconds). */
+const DEFAULT_API_TIMEOUT_MS = 30_000;
+/** Default timeout for file downloads (5 minutes). */
+const DEFAULT_DOWNLOAD_TIMEOUT_MS = 5 * 60_000;
+async function fetchJson(url, headers, timeoutMs = DEFAULT_API_TIMEOUT_MS) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(url, { headers, signal: controller.signal });
+        if (!res.ok) {
+            throw new Error(`GitHub API error ${res.status}: ${res.statusText}`);
+        }
+        return (await res.json());
+    }
+    finally {
+        clearTimeout(timeoutId);
     }
-    return (await res.json());
 }
-async function downloadToFile(url, headers, destPath) {
-    const res = await fetch(url, { headers, redirect: 'follow' });
-    if (!res.ok) {
-        throw new Error(`Download error ${res.status}: ${res.statusText}`);
+async function downloadToFile(url, headers, destPath, timeoutMs = DEFAULT_DOWNLOAD_TIMEOUT_MS, onProgress) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(url, { headers, redirect: 'follow', signal: controller.signal });
+        if (!res.ok) {
+            throw new Error(`Download error ${res.status}: ${res.statusText}`);
+        }
+        // Get content length for progress reporting
+        const contentLengthHeader = res.headers.get('content-length');
+        const totalBytes = contentLengthHeader ? parseInt(contentLengthHeader, 10) : undefined;
+        await ensureDir(path.dirname(destPath));
+        // Use streaming to report progress
+        const anyRes = res;
+        const body = anyRes.body;
+        if (body && typeof body[Symbol.asyncIterator] === 'function') {
+            // Async iterator for progress tracking
+            const fsModule = await import('node:fs');
+            const ws = fsModule.createWriteStream(destPath);
+            let downloadedBytes = 0;
+            let lastReportTime = Date.now();
+            const reportIntervalMs = 500; // Report at most every 500ms
+            try {
+                for await (const chunk of body) {
+                    ws.write(chunk);
+                    downloadedBytes += chunk.length;
+                    // Throttle progress reports
+                    const now = Date.now();
+                    if (onProgress && (now - lastReportTime > reportIntervalMs)) {
+                        lastReportTime = now;
+                        const percent = totalBytes ? Math.round((downloadedBytes / totalBytes) * 100) : 0;
+                        const msg = totalBytes
+                            ? `Downloaded ${formatBytes(downloadedBytes)} / ${formatBytes(totalBytes)} (${percent}%)`
+                            : `Downloaded ${formatBytes(downloadedBytes)}`;
+                        onProgress(downloadedBytes, totalBytes, msg);
+                    }
+                }
+            }
+            finally {
+                ws.end();
+                await new Promise((resolve) => ws.on('finish', () => resolve()));
+            }
+            // Final progress report
+            if (onProgress) {
+                const msg = totalBytes
+                    ? `Downloaded ${formatBytes(downloadedBytes)} / ${formatBytes(totalBytes)} (100%)`
+                    : `Downloaded ${formatBytes(downloadedBytes)}`;
+                onProgress(downloadedBytes, totalBytes, msg);
+            }
+            return;
+        }
+        if (body && typeof body.pipe === 'function') {
+            // Node.js stream (fallback without progress)
+            const ws = (await import('node:fs')).createWriteStream(destPath);
+            await new Promise((resolve, reject) => {
+                body.pipe(ws);
+                body.on('error', reject);
+                ws.on('error', reject);
+                ws.on('finish', () => resolve());
+            });
+            return;
+        }
+        // Web stream or no stream support (fallback without progress)
+        const buf = Buffer.from(await res.arrayBuffer());
+        await fs.writeFile(destPath, buf);
+        if (onProgress) {
+            onProgress(buf.length, buf.length, `Downloaded ${formatBytes(buf.length)}`);
+        }
     }
-    // Use streaming if possible; otherwise fallback to arrayBuffer.
-    const anyRes = res;
-    const body = anyRes.body;
-    await ensureDir(path.dirname(destPath));
-    if (body && typeof body.pipe === 'function') {
-        // Node.js stream
-        const ws = (await import('node:fs')).createWriteStream(destPath);
-        await new Promise((resolve, reject) => {
-            body.pipe(ws);
-            body.on('error', reject);
-            ws.on('error', reject);
-            ws.on('finish', () => resolve());
-        });
-        return;
+    finally {
+        clearTimeout(timeoutId);
     }
-    // Web stream or no stream support.
-    const buf = Buffer.from(await res.arrayBuffer());
-    await fs.writeFile(destPath, buf);
+}
+/** Format bytes for display */
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes}B`;
+    if (bytes < 1024 * 1024)
+        return `${(bytes / 1024).toFixed(1)}KB`;
+    if (bytes < 1024 * 1024 * 1024)
+        return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+    return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)}GB`;
 }
 async function extractZip(zipPath, destDir) {
     await ensureDir(destDir);
@@ -72,11 +143,13 @@ export async function downloadAndExtractGitHubArchive(params) {
     // Use the API zipball endpoint so ref can be branch/tag/SHA (including slashes via URL-encoding).
     const zipballUrl = `https://api.github.com/repos/${params.owner}/${params.repo}/zipball/${encodeURIComponent(refUsed)}`;
     await ensureDir(params.destDir);
-    await downloadToFile(zipballUrl, headers, zipPath);
+    await downloadToFile(zipballUrl, headers, zipPath, DEFAULT_DOWNLOAD_TIMEOUT_MS, params.onProgress);
     const extractDir = path.join(params.destDir, `extracted-${Date.now()}`);
     await extractZip(zipPath, extractDir);
     const repoRoot = await findSingleTopLevelDir(extractDir);
     // Best-effort cleanup: remove zip file (keep extracted for caller to consume).
-    await fs.rm(zipPath, { force: true }).catch(() => undefined);
+    await fs.rm(zipPath, { force: true }).catch((err) => {
+        logger.debug(`Failed to cleanup zip file ${zipPath} (non-critical)`, err instanceof Error ? err : undefined);
+    });
     return { repoRoot, refUsed, fetchedAt: nowIso() };
 }

package/dist/bundle/overview.js

@@ -1,5 +1,6 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { readFacts } from './facts.js';
 function evidence(p, start, end) {
     return `(evidence: ${p}:${start}-${end})`;
 }
@@ -156,64 +157,241 @@ async function renderContext7LibraryFacts(bundleRootDir, lib) {
     }
     return out;
 }
+/**
+ * Phase 3: Extract project purpose from README.md
+ */
+async function extractProjectPurpose(files) {
+    const readme = files.find(f => f.repoRelativePath.toLowerCase() === 'readme.md');
+    if (!readme)
+        return null;
+    try {
+        const content = await fs.readFile(readme.bundleNormAbsPath, 'utf8');
+        const lines = content.split('\n');
+        // Skip title (first h1)
+        let startIdx = 0;
+        for (let i = 0; i < lines.length; i++) {
+            if (lines[i]?.startsWith('# ')) {
+                startIdx = i + 1;
+                break;
+            }
+        }
+        // Extract first paragraph (non-empty lines until empty line or next heading)
+        const paragraph = [];
+        for (let i = startIdx; i < Math.min(lines.length, startIdx + 20); i++) {
+            const line = lines[i]?.trim() || '';
+            if (!line || line.startsWith('#'))
+                break;
+            paragraph.push(line);
+        }
+        return paragraph.join(' ').trim() || null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Phase 3: Format module list for display
+ */
+function formatCoreModules(facts) {
+    if (!facts.modules || facts.modules.length === 0)
+        return [];
+    const coreModules = facts.modules
+        .filter(m => m.role === 'core')
+        .sort((a, b) => b.exports.length - a.exports.length)
+        .slice(0, 10);
+    if (coreModules.length === 0)
+        return [];
+    const lines = [];
+    for (const mod of coreModules) {
+        const shortPath = mod.path.replace(/^repos\/[^\/]+\/[^\/]+\/norm\//, '');
+        lines.push(`- **${shortPath}**`);
+        lines.push(`  - Exports: ${mod.exports.slice(0, 5).join(', ')}${mod.exports.length > 5 ? ` (+${mod.exports.length - 5} more)` : ''}`);
+        lines.push(`  - Complexity: ${mod.complexity}, LOC: ${mod.loc}`);
+        lines.push(`  - Evidence: ${mod.path}:1`);
+    }
+    return lines;
+}
+/**
+ * Phase 3: Format standalone modules for reuse guidance
+ */
+function formatStandaloneModules(facts) {
+    if (!facts.modules || facts.modules.length === 0)
+        return [];
+    const standalone = facts.modules
+        .filter(m => m.standalone && (m.role === 'core' || m.role === 'utility'))
+        .filter(m => m.exports.length > 0)
+        .slice(0, 5);
+    if (standalone.length === 0)
+        return [];
+    const lines = [];
+    for (const mod of standalone) {
+        const shortPath = mod.path.replace(/^repos\/[^\/]+\/[^\/]+\/norm\//, '');
+        lines.push(`- **${shortPath}**`);
+        lines.push(`  - Can be used independently`);
+        lines.push(`  - Exports: ${mod.exports.slice(0, 3).join(', ')}`);
+        lines.push(`  - External deps: ${mod.imports.filter(i => !i.startsWith('.')).slice(0, 3).join(', ') || 'None'}`);
+    }
+    return lines;
+}
 export async function generateOverviewMarkdown(params) {
-    const header = `# OVERVIEW.md - Preflight Bundle ${params.bundleId}
-
-This file is generated. It contains **only factual statements** with evidence pointers into bundle files.
-
-`;
-    const sections = [header];
-    for (const r of params.repos) {
-        sections.push(`## Repo: ${r.repoId}`);
-        const metaFacts = await renderRepoMetaFacts(params.bundleRootDir, r.repoId);
-        if (metaFacts.length) {
-            sections.push('### Snapshot facts');
-            sections.push(...metaFacts);
-        }
-        const nodeFacts = await renderNodePackageFacts(r.files);
-        if (nodeFacts.length) {
-            sections.push('### Node/JS facts');
-            sections.push(...nodeFacts);
-        }
-        const docs = getRepoDocFiles(r.files).slice(0, 50);
-        if (docs.length) {
-            sections.push('### Documentation files (first 50)');
-            for (const d of docs) {
-                sections.push(`- ${d.bundleNormRelativePath}. ${evidence(d.bundleNormRelativePath, 1, 1)}`);
-            }
-        }
-        // Give a small hint about where code lives, without guessing entry points.
-        const codeSamples = r.files
-            .filter((f) => f.kind === 'code')
-            .map((f) => f.repoRelativePath)
-            .filter((p) => p.startsWith('src/') || p.startsWith('lib/'))
-            .slice(0, 10);
-        if (codeSamples.length) {
-            sections.push('### Code paths spotted (sample)');
-            for (const p of codeSamples) {
-                const file = r.files.find((f) => f.repoRelativePath === p);
-                if (!file)
-                    continue;
-                sections.push(`- ${file.bundleNormRelativePath}. ${evidence(file.bundleNormRelativePath, 1, 1)}`);
+    // Load FACTS.json if available
+    const factsPath = path.join(params.bundleRootDir, 'analysis', 'FACTS.json');
+    const facts = await readFacts(factsPath);
+    const sections = [];
+    // Header
+    sections.push(`# ${params.repos[0]?.repoId || 'Project'} - Overview\r\n`);
+    // Phase 3: What is this?
+    if (facts) {
+        sections.push('## What is this?\r\n');
+        // Try to get project purpose from README
+        const allFiles = params.repos.flatMap(r => r.files);
+        const purpose = await extractProjectPurpose(allFiles);
+        if (purpose) {
+            sections.push(`**Purpose**: ${purpose}\r\n`);
+        }
+        // Primary language and frameworks
+        if (facts.languages && facts.languages.length > 0) {
+            const primaryLang = facts.languages[0];
+            if (primaryLang) {
+                sections.push(`**Language**: ${primaryLang.language} (${primaryLang.fileCount} files)\r\n`);
+            }
+        }
+        if (facts.frameworks && facts.frameworks.length > 0) {
+            sections.push(`**Frameworks**: ${facts.frameworks.join(', ')}\r\n`);
+        }
+        // Tech stack (Phase 2)
+        if (facts.techStack) {
+            if (facts.techStack.runtime) {
+                sections.push(`**Runtime**: ${facts.techStack.runtime}\r\n`);
+            }
+            if (facts.techStack.packageManager) {
+                sections.push(`**Package Manager**: ${facts.techStack.packageManager}\r\n`);
             }
         }
         sections.push('');
     }
-    const libs = params.libraries ?? [];
-    if (libs.length) {
-        sections.push('## Context7 libraries');
-        for (const lib of libs) {
-            const facts = await renderContext7LibraryFacts(params.bundleRootDir, lib);
-            sections.push(`### ${lib.input}`);
-            if (facts.length) {
-                sections.push(...facts);
+    // Phase 3: Architecture
+    if (facts) {
+        sections.push('## Architecture\r\n');
+        // Entry points
+        if (facts.entryPoints && facts.entryPoints.length > 0) {
+            sections.push('### Entry Points\r\n');
+            for (const ep of facts.entryPoints.slice(0, 5)) {
+                const shortPath = ep.file.replace(/^repos\/[^\/]+\/[^\/]+\/norm\//, '');
+                sections.push(`- \`${shortPath}\` (${ep.type}). ${evidence(ep.evidence, 1, 1)}\r\n`);
             }
-            else {
-                sections.push('- No library facts available.');
+            sections.push('');
+        }
+        // Phase 2: Architecture patterns
+        if (facts.patterns && facts.patterns.length > 0) {
+            sections.push('### Design Patterns\r\n');
+            for (const pattern of facts.patterns) {
+                sections.push(`- ${pattern}\r\n`);
+            }
+            sections.push('');
+        }
+        // Phase 2: Core modules
+        const coreModuleLines = formatCoreModules(facts);
+        if (coreModuleLines.length > 0) {
+            sections.push('### Core Modules\r\n');
+            sections.push(...coreModuleLines.map(l => l + '\r\n'));
+            sections.push('');
+        }
+    }
+    // Dependencies
+    if (facts && (facts.dependencies.runtime.length > 0 || facts.dependencies.dev.length > 0)) {
+        sections.push('## Dependencies\r\n');
+        if (facts.dependencies.runtime.length > 0) {
+            sections.push(`### Production (${facts.dependencies.runtime.length})\r\n`);
+            for (const dep of facts.dependencies.runtime.slice(0, 15)) {
+                sections.push(`- ${dep.name}${dep.version ? ` ${dep.version}` : ''}\r\n`);
+            }
+            if (facts.dependencies.runtime.length > 15) {
+                sections.push(`- ... and ${facts.dependencies.runtime.length - 15} more\r\n`);
+            }
+            sections.push('');
+        }
+        if (facts.dependencies.dev.length > 0) {
+            sections.push(`### Development (${facts.dependencies.dev.length})\r\n`);
+            for (const dep of facts.dependencies.dev.slice(0, 10)) {
+                sections.push(`- ${dep.name}${dep.version ? ` ${dep.version}` : ''}\r\n`);
+            }
+            if (facts.dependencies.dev.length > 10) {
+                sections.push(`- ... and ${facts.dependencies.dev.length - 10} more\r\n`);
             }
             sections.push('');
         }
     }
+    // Phase 3: How to Reuse
+    if (facts) {
+        const standaloneLines = formatStandaloneModules(facts);
+        if (standaloneLines.length > 0) {
+            sections.push('## How to Reuse\r\n');
+            sections.push('### Standalone Modules\r\n');
+            sections.push('These modules can be extracted and used independently:\r\n\r\n');
+            sections.push(...standaloneLines.map(l => l + '\r\n'));
+            sections.push('');
+        }
+        // Return Phase 3 format directly
+        return sections.join('\n') + '\n';
+    }
+    // Fallback to legacy format if no FACTS
+    {
+        const header = `# OVERVIEW.md - Preflight Bundle ${params.bundleId}\r\n\r\nThis file is generated. It contains **only factual statements** with evidence pointers into bundle files.\r\n\r\n`;
+        sections.splice(0, sections.length); // Clear Phase 3 sections
+        sections.push(header);
+        for (const r of params.repos) {
+            sections.push(`## Repo: ${r.repoId}`);
+            const metaFacts = await renderRepoMetaFacts(params.bundleRootDir, r.repoId);
+            if (metaFacts.length) {
+                sections.push('### Snapshot facts');
+                sections.push(...metaFacts);
+            }
+            const nodeFacts = await renderNodePackageFacts(r.files);
+            if (nodeFacts.length) {
+                sections.push('### Node/JS facts');
+                sections.push(...nodeFacts);
+            }
+            const docs = getRepoDocFiles(r.files).slice(0, 50);
+            if (docs.length) {
+                sections.push('### Documentation files (first 50)');
+                for (const d of docs) {
+                    sections.push(`- ${d.bundleNormRelativePath}. ${evidence(d.bundleNormRelativePath, 1, 1)}`);
+                }
+            }
+            // Give a small hint about where code lives, without guessing entry points.
+            const codeSamples = r.files
+                .filter((f) => f.kind === 'code')
+                .map((f) => f.repoRelativePath)
+                .filter((p) => p.startsWith('src/') || p.startsWith('lib/'))
+                .slice(0, 10);
+            if (codeSamples.length) {
+                sections.push('### Code paths spotted (sample)');
+                for (const p of codeSamples) {
+                    const file = r.files.find((f) => f.repoRelativePath === p);
+                    if (!file)
+                        continue;
+                    sections.push(`- ${file.bundleNormRelativePath}. ${evidence(file.bundleNormRelativePath, 1, 1)}`);
+                }
+            }
+            sections.push('');
+        }
+        const libs = params.libraries ?? [];
+        if (libs.length) {
+            sections.push('## Context7 libraries');
+            for (const lib of libs) {
+                const facts = await renderContext7LibraryFacts(params.bundleRootDir, lib);
+                sections.push(`### ${lib.input}`);
+                if (facts.length) {
+                    sections.push(...facts);
+                }
+                else {
+                    sections.push('- No library facts available.');
+                }
+                sections.push('');
+            }
+        }
+    }
     return sections.join('\n') + '\n';
 }
 export async function writeOverviewFile(targetPath, markdown) {