specweave 0.29.2 → 0.30.0

package/dist/src/core/living-docs/discovery.js

@@ -0,0 +1,656 @@
+/**
+ * Living Docs Builder - Discovery Phase
+ *
+ * Scans the codebase structure without LLM calls to:
+ * - Count files by type and extension
+ * - Detect tech stack from config files
+ * - Find existing documentation
+ * - Identify entry points
+ * - Group files into modules
+ * - Calculate sampling tier
+ */
+import * as fs from '../../utils/fs-native.js';
+import * as path from 'path';
+/**
+ * Skip patterns for directories that should not be scanned
+ */
+const SKIP_DIRS = new Set([
+    'node_modules',
+    '.git',
+    '.specweave',
+    'dist',
+    'build',
+    'out',
+    'coverage',
+    '.next',
+    '.nuxt',
+    '__pycache__',
+    '.pytest_cache',
+    '.venv',
+    'venv',
+    'env',
+    '.env',
+    'vendor',
+    'target', // Rust
+    'bin',
+    'obj', // .NET
+]);
+/**
+ * Extension to file type mapping
+ */
+const EXTENSION_TYPES = {
+    // Code
+    '.ts': 'code',
+    '.tsx': 'code',
+    '.js': 'code',
+    '.jsx': 'code',
+    '.mjs': 'code',
+    '.cjs': 'code',
+    '.py': 'code',
+    '.go': 'code',
+    '.rs': 'code',
+    '.java': 'code',
+    '.kt': 'code',
+    '.cs': 'code',
+    '.cpp': 'code',
+    '.c': 'code',
+    '.h': 'code',
+    '.hpp': 'code',
+    '.rb': 'code',
+    '.php': 'code',
+    '.swift': 'code',
+    '.scala': 'code',
+    '.vue': 'code',
+    '.svelte': 'code',
+    // Docs
+    '.md': 'docs',
+    '.mdx': 'docs',
+    '.rst': 'docs',
+    '.txt': 'docs',
+    '.adoc': 'docs',
+    // Config
+    '.json': 'config',
+    '.yaml': 'config',
+    '.yml': 'config',
+    '.toml': 'config',
+    '.xml': 'config',
+    '.ini': 'config',
+    '.env': 'config',
+    // Assets
+    '.png': 'assets',
+    '.jpg': 'assets',
+    '.jpeg': 'assets',
+    '.gif': 'assets',
+    '.svg': 'assets',
+    '.ico': 'assets',
+    '.woff': 'assets',
+    '.woff2': 'assets',
+    '.ttf': 'assets',
+    '.eot': 'assets',
+};
+/**
+ * Test file patterns
+ */
+const TEST_PATTERNS = [
+    /\.test\./,
+    /\.spec\./,
+    /_test\./,
+    /_spec\./,
+    /test_/,
+    /spec_/,
+    /__tests__/,
+    /tests\//,
+];
+/**
+ * Run the discovery phase
+ */
+export async function runDiscovery(projectPath, additionalSources, onProgress) {
+    const result = {
+        codebaseStats: {
+            totalFiles: 0,
+            totalDirs: 0,
+            byExtension: {},
+            byType: { code: 0, tests: 0, docs: 0, config: 0, assets: 0 },
+            estimatedLOC: 0,
+        },
+        techStack: {
+            languages: [],
+            frameworks: [],
+            buildTools: [],
+            testFrameworks: [],
+            detectedFrom: [],
+        },
+        existingDocs: {
+            readme: null,
+            contributing: null,
+            docsFolder: null,
+            wikiFolder: null,
+            mdFiles: [],
+        },
+        entryPoints: {
+            main: [],
+            index: [],
+            exports: [],
+        },
+        modules: [],
+        tier: 'small',
+        samplingConfig: calculateTier(0),
+        discoveredAt: new Date().toISOString(),
+    };
+    // First pass: count directories for progress
+    const dirs = await countDirectories(projectPath);
+    let processedDirs = 0;
+    // Scan project structure
+    await scanDirectory(projectPath, projectPath, result, (dir) => {
+        processedDirs++;
+        onProgress?.(processedDirs, dirs, dir);
+    });
+    // Scan additional sources
+    for (const source of additionalSources) {
+        const fullPath = path.isAbsolute(source) ? source : path.join(projectPath, source);
+        if (fs.existsSync(fullPath) && fs.statSync(fullPath).isDirectory()) {
+            await scanDirectory(fullPath, projectPath, result, (dir) => {
+                onProgress?.(processedDirs, dirs, `[additional] ${dir}`);
+            });
+        }
+    }
+    // Detect tech stack from config files
+    detectTechStack(projectPath, result);
+    // Find existing documentation
+    findExistingDocs(projectPath, result);
+    // Identify entry points
+    findEntryPoints(projectPath, result);
+    // Build modules from directory structure
+    buildModules(projectPath, result);
+    // Calculate sampling tier
+    result.tier = calculateTier(result.codebaseStats.totalFiles).tier;
+    result.samplingConfig = calculateTier(result.codebaseStats.totalFiles);
+    return result;
+}
+/**
+ * Count directories for progress estimation
+ */
+async function countDirectories(dirPath, count = 0) {
+    try {
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+        let total = 1;
+        for (const entry of entries) {
+            if (entry.isDirectory() && !SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
+                total += await countDirectories(path.join(dirPath, entry.name), total);
+            }
+        }
+        return total;
+    }
+    catch {
+        return count;
+    }
+}
+/**
+ * Recursively scan a directory
+ */
+async function scanDirectory(dirPath, projectRoot, result, onDir, depth = 0) {
+    if (depth > 15)
+        return; // Prevent infinite recursion
+    try {
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+        result.codebaseStats.totalDirs++;
+        onDir(path.relative(projectRoot, dirPath) || '.');
+        for (const entry of entries) {
+            const fullPath = path.join(dirPath, entry.name);
+            const relativePath = path.relative(projectRoot, fullPath);
+            if (entry.isDirectory()) {
+                if (!SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
+                    await scanDirectory(fullPath, projectRoot, result, onDir, depth + 1);
+                }
+            }
+            else if (entry.isFile()) {
+                result.codebaseStats.totalFiles++;
+                const ext = path.extname(entry.name).toLowerCase();
+                result.codebaseStats.byExtension[ext] = (result.codebaseStats.byExtension[ext] || 0) + 1;
+                // Categorize by type
+                const fileType = EXTENSION_TYPES[ext] || 'assets';
+                const isTest = TEST_PATTERNS.some(p => p.test(relativePath));
+                if (isTest) {
+                    result.codebaseStats.byType.tests++;
+                }
+                else {
+                    result.codebaseStats.byType[fileType]++;
+                }
+                // Estimate LOC for code files
+                if (fileType === 'code' && !isTest) {
+                    try {
+                        const stat = fs.statSync(fullPath);
+                        // Rough estimate: average 40 bytes per line of code
+                        result.codebaseStats.estimatedLOC += Math.round(stat.size / 40);
+                    }
+                    catch {
+                        // Ignore stat errors
+                    }
+                }
+            }
+        }
+    }
+    catch {
+        // Ignore permission errors
+    }
+}
+/**
+ * Detect tech stack from config files
+ */
+function detectTechStack(projectPath, result) {
+    // Check package.json (Node.js/JavaScript/TypeScript)
+    const packageJsonPath = path.join(projectPath, 'package.json');
+    if (fs.existsSync(packageJsonPath)) {
+        try {
+            const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+            result.techStack.detectedFrom.push('package.json');
+            const allDeps = {
+                ...(pkg.dependencies || {}),
+                ...(pkg.devDependencies || {}),
+            };
+            // Languages
+            if (allDeps.typescript || fs.existsSync(path.join(projectPath, 'tsconfig.json'))) {
+                result.techStack.languages.push('TypeScript');
+            }
+            else {
+                result.techStack.languages.push('JavaScript');
+            }
+            // Frameworks
+            if (allDeps.next)
+                result.techStack.frameworks.push('Next.js');
+            if (allDeps.react)
+                result.techStack.frameworks.push('React');
+            if (allDeps.vue)
+                result.techStack.frameworks.push('Vue');
+            if (allDeps.angular || allDeps['@angular/core'])
+                result.techStack.frameworks.push('Angular');
+            if (allDeps.svelte)
+                result.techStack.frameworks.push('Svelte');
+            if (allDeps.express)
+                result.techStack.frameworks.push('Express');
+            if (allDeps.fastify)
+                result.techStack.frameworks.push('Fastify');
+            if (allDeps.nestjs || allDeps['@nestjs/core'])
+                result.techStack.frameworks.push('NestJS');
+            // Build tools
+            if (allDeps.webpack)
+                result.techStack.buildTools.push('Webpack');
+            if (allDeps.vite)
+                result.techStack.buildTools.push('Vite');
+            if (allDeps.esbuild)
+                result.techStack.buildTools.push('esbuild');
+            if (allDeps.rollup)
+                result.techStack.buildTools.push('Rollup');
+            if (allDeps.parcel)
+                result.techStack.buildTools.push('Parcel');
+            // Test frameworks
+            if (allDeps.jest)
+                result.techStack.testFrameworks.push('Jest');
+            if (allDeps.vitest)
+                result.techStack.testFrameworks.push('Vitest');
+            if (allDeps.mocha)
+                result.techStack.testFrameworks.push('Mocha');
+            if (allDeps.playwright || allDeps['@playwright/test'])
+                result.techStack.testFrameworks.push('Playwright');
+            if (allDeps.cypress)
+                result.techStack.testFrameworks.push('Cypress');
+        }
+        catch {
+            // Ignore parse errors
+        }
+    }
+    // Check requirements.txt / pyproject.toml (Python)
+    const requirementsPath = path.join(projectPath, 'requirements.txt');
+    const pyprojectPath = path.join(projectPath, 'pyproject.toml');
+    if (fs.existsSync(requirementsPath) || fs.existsSync(pyprojectPath)) {
+        result.techStack.languages.push('Python');
+        result.techStack.detectedFrom.push(fs.existsSync(pyprojectPath) ? 'pyproject.toml' : 'requirements.txt');
+        // Try to detect frameworks
+        try {
+            const content = fs.existsSync(pyprojectPath)
+                ? fs.readFileSync(pyprojectPath, 'utf-8')
+                : fs.readFileSync(requirementsPath, 'utf-8');
+            if (content.includes('django'))
+                result.techStack.frameworks.push('Django');
+            if (content.includes('fastapi'))
+                result.techStack.frameworks.push('FastAPI');
+            if (content.includes('flask'))
+                result.techStack.frameworks.push('Flask');
+            if (content.includes('pytest'))
+                result.techStack.testFrameworks.push('pytest');
+        }
+        catch {
+            // Ignore read errors
+        }
+    }
+    // Check go.mod (Go)
+    const goModPath = path.join(projectPath, 'go.mod');
+    if (fs.existsSync(goModPath)) {
+        result.techStack.languages.push('Go');
+        result.techStack.detectedFrom.push('go.mod');
+        try {
+            const content = fs.readFileSync(goModPath, 'utf-8');
+            if (content.includes('gin-gonic'))
+                result.techStack.frameworks.push('Gin');
+            if (content.includes('echo'))
+                result.techStack.frameworks.push('Echo');
+            if (content.includes('fiber'))
+                result.techStack.frameworks.push('Fiber');
+        }
+        catch {
+            // Ignore read errors
+        }
+    }
+    // Check Cargo.toml (Rust)
+    const cargoPath = path.join(projectPath, 'Cargo.toml');
+    if (fs.existsSync(cargoPath)) {
+        result.techStack.languages.push('Rust');
+        result.techStack.detectedFrom.push('Cargo.toml');
+        try {
+            const content = fs.readFileSync(cargoPath, 'utf-8');
+            if (content.includes('actix'))
+                result.techStack.frameworks.push('Actix');
+            if (content.includes('axum'))
+                result.techStack.frameworks.push('Axum');
+            if (content.includes('rocket'))
+                result.techStack.frameworks.push('Rocket');
+        }
+        catch {
+            // Ignore read errors
+        }
+    }
+    // Check pom.xml / build.gradle (Java)
+    const pomPath = path.join(projectPath, 'pom.xml');
+    const gradlePath = path.join(projectPath, 'build.gradle');
+    if (fs.existsSync(pomPath) || fs.existsSync(gradlePath)) {
+        result.techStack.languages.push('Java');
+        result.techStack.detectedFrom.push(fs.existsSync(pomPath) ? 'pom.xml' : 'build.gradle');
+        if (fs.existsSync(gradlePath)) {
+            result.techStack.buildTools.push('Gradle');
+        }
+        else {
+            result.techStack.buildTools.push('Maven');
+        }
+    }
+    // Deduplicate
+    result.techStack.languages = [...new Set(result.techStack.languages)];
+    result.techStack.frameworks = [...new Set(result.techStack.frameworks)];
+    result.techStack.buildTools = [...new Set(result.techStack.buildTools)];
+    result.techStack.testFrameworks = [...new Set(result.techStack.testFrameworks)];
+}
+/**
+ * Find existing documentation
+ */
+function findExistingDocs(projectPath, result) {
+    // Look for README
+    const readmeVariants = ['README.md', 'README.MD', 'readme.md', 'Readme.md', 'README'];
+    for (const variant of readmeVariants) {
+        const readmePath = path.join(projectPath, variant);
+        if (fs.existsSync(readmePath)) {
+            result.existingDocs.readme = variant;
+            break;
+        }
+    }
+    // Look for CONTRIBUTING
+    const contributingVariants = ['CONTRIBUTING.md', 'CONTRIBUTING.MD', 'contributing.md'];
+    for (const variant of contributingVariants) {
+        const contribPath = path.join(projectPath, variant);
+        if (fs.existsSync(contribPath)) {
+            result.existingDocs.contributing = variant;
+            break;
+        }
+    }
+    // Look for docs folder
+    const docsFolders = ['docs', 'doc', 'documentation', 'Docs', 'Documentation'];
+    for (const folder of docsFolders) {
+        const docsPath = path.join(projectPath, folder);
+        if (fs.existsSync(docsPath) && fs.statSync(docsPath).isDirectory()) {
+            result.existingDocs.docsFolder = folder;
+            break;
+        }
+    }
+    // Look for wiki folder
+    const wikiPath = path.join(projectPath, 'wiki');
+    if (fs.existsSync(wikiPath) && fs.statSync(wikiPath).isDirectory()) {
+        result.existingDocs.wikiFolder = 'wiki';
+    }
+    // Find all markdown files (top level and in docs)
+    const mdFiles = [];
+    try {
+        const entries = fs.readdirSync(projectPath);
+        for (const entry of entries) {
+            if (entry.endsWith('.md') || entry.endsWith('.mdx')) {
+                mdFiles.push(entry);
+            }
+        }
+    }
+    catch {
+        // Ignore errors
+    }
+    result.existingDocs.mdFiles = mdFiles;
+}
+/**
+ * Find entry points
+ */
+function findEntryPoints(projectPath, result) {
+    const mainPatterns = ['main.ts', 'main.js', 'app.ts', 'app.js', 'server.ts', 'server.js', 'cli.ts', 'cli.js'];
+    const indexPatterns = ['index.ts', 'index.js', 'mod.ts', 'mod.js'];
+    const srcFolders = ['src', 'lib', 'app'];
+    // Check root level
+    for (const pattern of mainPatterns) {
+        const filePath = path.join(projectPath, pattern);
+        if (fs.existsSync(filePath)) {
+            result.entryPoints.main.push(pattern);
+        }
+    }
+    for (const pattern of indexPatterns) {
+        const filePath = path.join(projectPath, pattern);
+        if (fs.existsSync(filePath)) {
+            result.entryPoints.index.push(pattern);
+        }
+    }
+    // Check src folders
+    for (const folder of srcFolders) {
+        const folderPath = path.join(projectPath, folder);
+        if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
+            for (const pattern of mainPatterns) {
+                const filePath = path.join(folderPath, pattern);
+                if (fs.existsSync(filePath)) {
+                    result.entryPoints.main.push(path.join(folder, pattern));
+                }
+            }
+            for (const pattern of indexPatterns) {
+                const filePath = path.join(folderPath, pattern);
+                if (fs.existsSync(filePath)) {
+                    result.entryPoints.index.push(path.join(folder, pattern));
+                }
+            }
+        }
+    }
+    // Check package.json for main/exports
+    const packageJsonPath = path.join(projectPath, 'package.json');
+    if (fs.existsSync(packageJsonPath)) {
+        try {
+            const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+            if (pkg.main)
+                result.entryPoints.exports.push(pkg.main);
+            if (pkg.module)
+                result.entryPoints.exports.push(pkg.module);
+            if (pkg.bin) {
+                if (typeof pkg.bin === 'string') {
+                    result.entryPoints.exports.push(pkg.bin);
+                }
+                else {
+                    result.entryPoints.exports.push(...Object.values(pkg.bin));
+                }
+            }
+        }
+        catch {
+            // Ignore parse errors
+        }
+    }
+}
+/**
+ * Build modules from directory structure
+ */
+function buildModules(projectPath, result) {
+    const srcFolders = ['src', 'lib', 'app', 'packages'];
+    for (const srcFolder of srcFolders) {
+        const srcPath = path.join(projectPath, srcFolder);
+        if (fs.existsSync(srcPath) && fs.statSync(srcPath).isDirectory()) {
+            try {
+                const entries = fs.readdirSync(srcPath, { withFileTypes: true });
+                for (const entry of entries) {
+                    if (entry.isDirectory() && !SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
+                        const modulePath = path.join(srcFolder, entry.name);
+                        const fullModulePath = path.join(projectPath, modulePath);
+                        const moduleInfo = analyzeModule(fullModulePath, modulePath);
+                        result.modules.push(moduleInfo);
+                    }
+                }
+            }
+            catch {
+                // Ignore errors
+            }
+        }
+    }
+    // Sort modules by file count (largest first)
+    result.modules.sort((a, b) => b.fileCount - a.fileCount);
+}
+/**
+ * Analyze a module directory
+ */
+function analyzeModule(modulePath, relativePath) {
+    const info = {
+        name: path.basename(modulePath),
+        path: relativePath,
+        fileCount: 0,
+        estimatedLOC: 0,
+        hasTests: false,
+        hasReadme: false,
+        entryPoints: [],
+        extensions: {},
+    };
+    try {
+        const walk = (dir, depth = 0) => {
+            if (depth > 10)
+                return;
+            const entries = fs.readdirSync(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(dir, entry.name);
+                if (entry.isDirectory() && !SKIP_DIRS.has(entry.name)) {
+                    walk(fullPath, depth + 1);
+                }
+                else if (entry.isFile()) {
+                    info.fileCount++;
+                    const ext = path.extname(entry.name).toLowerCase();
+                    info.extensions[ext] = (info.extensions[ext] || 0) + 1;
+                    // Check for tests
+                    if (TEST_PATTERNS.some(p => p.test(entry.name))) {
+                        info.hasTests = true;
+                    }
+                    // Check for README
+                    if (entry.name.toLowerCase().includes('readme')) {
+                        info.hasReadme = true;
+                    }
+                    // Check for entry points
+                    if (entry.name === 'index.ts' || entry.name === 'index.js') {
+                        info.entryPoints.push(path.relative(modulePath, fullPath));
+                    }
+                    // Estimate LOC
+                    if (EXTENSION_TYPES[ext] === 'code') {
+                        try {
+                            const stat = fs.statSync(fullPath);
+                            info.estimatedLOC += Math.round(stat.size / 40);
+                        }
+                        catch {
+                            // Ignore stat errors
+                        }
+                    }
+                }
+            }
+        };
+        walk(modulePath);
+    }
+    catch {
+        // Ignore errors
+    }
+    return info;
+}
+/**
+ * Calculate sampling tier based on file count
+ */
+export function calculateTier(totalFiles) {
+    if (totalFiles < 500) {
+        return {
+            tier: 'small',
+            filesPerDir: 'all',
+            priorityPatterns: ['**/index.*', '**/main.*', '**/*.config.*', '**/types.*'],
+            skipPatterns: ['node_modules/**', 'dist/**', '**/*.test.*', '**/*.spec.*'],
+        };
+    }
+    else if (totalFiles < 2000) {
+        return {
+            tier: 'medium',
+            filesPerDir: 5,
+            priorityPatterns: ['**/index.*', '**/main.*', '**/*.config.*', '**/types.*', '**/README.md'],
+            skipPatterns: ['node_modules/**', 'dist/**', 'build/**', '**/*.test.*', '**/*.spec.*'],
+        };
+    }
+    else if (totalFiles < 10000) {
+        return {
+            tier: 'large',
+            filesPerDir: 3,
+            priorityPatterns: ['**/index.*', '**/main.*', '**/*.config.*', '**/types.*', '**/README.md'],
+            skipPatterns: ['node_modules/**', 'dist/**', 'build/**', 'coverage/**', '**/*.test.*', '**/*.spec.*', '**/__tests__/**'],
+        };
+    }
+    else {
+        return {
+            tier: 'massive',
+            filesPerDir: 1,
+            priorityPatterns: ['**/index.*', '**/main.*', '**/app.*', '**/*.config.*'],
+            skipPatterns: ['node_modules/**', 'dist/**', 'build/**', 'coverage/**', '**/*.test.*', '**/*.spec.*', '**/__tests__/**', '**/*.min.*', '**/*.bundle.*'],
+        };
+    }
+}
+/**
+ * Select representative files from a directory for analysis
+ */
+export function selectRepresentativeFiles(files, maxFiles, moduleName) {
+    if (maxFiles === 'all') {
+        return files;
+    }
+    if (files.length <= maxFiles) {
+        return files;
+    }
+    // Priority selection heuristics
+    const priority = [];
+    const regular = [];
+    for (const file of files) {
+        const basename = path.basename(file).toLowerCase();
+        const ext = path.extname(file);
+        // High priority files
+        if (basename.startsWith('index') ||
+            basename.startsWith('main') ||
+            basename.startsWith('app') ||
+            (moduleName && basename.startsWith(moduleName.toLowerCase())) ||
+            basename.includes('types') ||
+            basename.includes('config') ||
+            ext === '.d.ts') {
+            priority.push(file);
+        }
+        else {
+            regular.push(file);
+        }
+    }
+    // Take priority files first, then fill with regular
+    const selected = [...priority.slice(0, maxFiles)];
+    const remaining = maxFiles - selected.length;
+    if (remaining > 0) {
+        // Sort regular files by size (larger = more content)
+        selected.push(...regular.slice(0, remaining));
+    }
+    return selected;
+}
+//# sourceMappingURL=discovery.js.map