skrypt-ai 0.6.1 → 0.8.0

package/dist/commands/generate.js

@@ -1,445 +0,0 @@
-import { Command } from 'commander';
-import { existsSync, copyFileSync, mkdirSync, readFileSync } from 'fs';
-import { resolve, basename, dirname, join } from 'path';
-import { loadConfig, validateConfig, checkApiKey } from '../config/index.js';
-import { DEFAULT_MODELS } from '../config/types.js';
-import { scanDirectory } from '../scanner/index.js';
-import { createLLMClient } from '../llm/index.js';
-import { generateForElements, groupDocsByFile, writeDocsToDirectory, writeDocsByTopic, writeLlmsTxt } from '../generator/index.js';
-import { showSecurityNotice } from '../auth/notices.js';
-import { runQA, printQAReport, fixQAIssues, printFixReport } from '../qa/index.js';
-import { extractDependencyIds, isChubInstalled, fetchContextHubDocs, exportToContextHub } from '../context-hub/index.js';
-import * as readline from 'readline';
-/**
- * Read .skryptignore patterns from source directory
- */
-function readIgnorePatterns(sourcePath) {
-    const ignorePath = join(sourcePath, '.skryptignore');
-    if (!existsSync(ignorePath))
-        return [];
-    const content = readFileSync(ignorePath, 'utf-8');
-    return content
-        .split('\n')
-        .map(line => line.trim())
-        .filter(line => line && !line.startsWith('#'));
-}
-/**
- * Auto-detect OpenAPI spec file in source directory
- */
-function findOpenAPISpec(sourcePath) {
-    const candidates = [
-        'openapi.json',
-        'openapi.yaml',
-        'openapi.yml',
-        'swagger.json',
-        'swagger.yaml',
-        'swagger.yml',
-        'api.json',
-        'api.yaml',
-        'api.yml',
-    ];
-    for (const name of candidates) {
-        const specPath = join(sourcePath, name);
-        if (existsSync(specPath)) {
-            return specPath;
-        }
-    }
-    // Check common subdirectories
-    const subdirs = ['docs', 'api', 'spec', '.'];
-    for (const subdir of subdirs) {
-        for (const name of candidates) {
-            const specPath = join(sourcePath, subdir, name);
-            if (existsSync(specPath)) {
-                return specPath;
-            }
-        }
-    }
-    return null;
-}
-/**
- * Check if element should be excluded based on patterns
- */
-function shouldExcludeElement(element, patterns) {
-    for (const pattern of patterns) {
-        // Match by name
-        if (pattern.startsWith('name:')) {
-            const namePattern = pattern.slice(5);
-            if (element.name === namePattern) {
-                return true;
-            }
-            // Only use regex if the pattern contains regex metacharacters
-            // Reject patterns with nested quantifiers to prevent catastrophic backtracking (ReDoS)
-            if (/[*+?{}()|[\]\\^$.]/.test(namePattern)) {
-                if (/(\+|\*|\?)\{|\(\?[^:)]|\(\.[*+].*\)\+|\([^)]*[+*][^)]*\)[+*]/.test(namePattern)) {
-                    continue; // Skip patterns prone to catastrophic backtracking
-                }
-                try {
-                    if (new RegExp(namePattern).test(element.name))
-                        return true;
-                }
-                catch {
-                    // Invalid regex — treat as literal match (already checked above)
-                }
-            }
-        }
-        // Match by file path
-        else if (pattern.includes('/') || pattern.includes('*')) {
-            const filePath = element.filePath;
-            if (pattern.includes('**')) {
-                const parts = pattern.split('**');
-                const prefixMatch = !parts[0] || filePath.includes(parts[0].replace(/^\//, ''));
-                const suffixMatch = !parts[1] || filePath.includes(parts[1].replace(/^\//, ''));
-                if (prefixMatch && suffixMatch)
-                    return true;
-            }
-            else if (filePath.includes(pattern.replace(/\*/g, ''))) {
-                return true;
-            }
-        }
-        // Match by exact name
-        else if (element.name === pattern) {
-            return true;
-        }
-    }
-    return false;
-}
-export const generateCommand = new Command('generate')
-    .description('Generate documentation with code examples')
-    .argument('<source>', 'Source directory to scan')
-    .option('-o, --output <dir>', 'Output directory')
-    .option('-c, --config <file>', 'Config file path')
-    .option('--provider <name>', 'LLM provider (deepseek, openai, anthropic, google, ollama, openrouter)')
-    .option('--model <name>', 'LLM model name')
-    .option('--base-url <url>', 'Custom API base URL (for Ollama or proxies)')
-    .option('--dry-run', 'Scan only, do not generate docs')
-    .option('--multi-lang', 'Generate TypeScript + Python examples')
-    .option('--by-topic', 'Organize output by topic instead of file')
-    .option('--openapi <file>', 'Include OpenAPI spec file for API Playground')
-    .option('--public-only', 'Only document exported/public APIs')
-    .option('--exclude <patterns...>', 'Exclude patterns (files, names, or name:pattern)')
-    .option('--llms-txt', 'Generate llms.txt for Answer Engine Optimization (AEO)')
-    .option('--project-name <name>', 'Project name for llms.txt header')
-    .action(async (source, options) => {
-    try {
-        const startTime = Date.now();
-        // Load config (file or defaults)
-        const config = loadConfig(options.config);
-        // CLI flags override config
-        if (source)
-            config.source.path = source;
-        if (options.output)
-            config.output.path = options.output;
-        if (options.provider) {
-            const validProviders = ['deepseek', 'openai', 'anthropic', 'google', 'ollama', 'openrouter'];
-            if (!validProviders.includes(options.provider)) {
-                console.error(`Error: Unknown provider "${options.provider}". Valid: ${validProviders.join(', ')}`);
-                process.exit(1);
-            }
-            config.llm.provider = options.provider;
-            // Use provider's default model unless explicitly specified
-            if (!options.model) {
-                config.llm.model = DEFAULT_MODELS[config.llm.provider];
-            }
-        }
-        if (options.model)
-            config.llm.model = options.model;
-        if (options.baseUrl)
-            config.llm.baseUrl = options.baseUrl;
-        // Validate
-        const errors = validateConfig(config);
-        if (errors.length > 0) {
-            console.error('Config errors:');
-            errors.forEach(e => console.error(`  - ${e}`));
-            process.exit(1);
-        }
-        // Check for API key (not needed for Ollama or dry-run)
-        if (!options.dryRun) {
-            const { ok, envKey } = checkApiKey(config.llm.provider);
-            if (!ok && envKey) {
-                console.error(`Error: ${envKey} environment variable required for ${config.llm.provider}`);
-                process.exit(1);
-            }
-        }
-        // First-run security notice
-        showSecurityNotice();
-        console.log('skrypt generate');
-        console.log(`  source:   ${config.source.path}`);
-        console.log(`  output:   ${config.output.path}`);
-        console.log(`  provider: ${config.llm.provider}`);
-        console.log(`  model:    ${config.llm.model}`);
-        if (config.llm.baseUrl) {
-            console.log(`  base url: ${config.llm.baseUrl}`);
-        }
-        // Routing transparency
-        const providerEnvKeys = {
-            openai: 'OPENAI_API_KEY',
-            anthropic: 'ANTHROPIC_API_KEY',
-            google: 'GOOGLE_API_KEY',
-            deepseek: 'DEEPSEEK_API_KEY',
-        };
-        const providerKey = providerEnvKeys[config.llm.provider];
-        if (providerKey && process.env[providerKey]) {
-            console.log(`  routing:  direct to ${config.llm.provider} (BYOK — your key never touches Skrypt)`);
-        }
-        else if (config.llm.provider !== 'ollama') {
-            console.log('  routing:  via Skrypt API proxy');
-        }
-        console.log('');
-        // Check source exists
-        const sourcePath = resolve(config.source.path);
-        if (!existsSync(sourcePath)) {
-            console.error(`Error: Source directory not found: ${sourcePath}`);
-            process.exit(1);
-        }
-        // Step 1: Scan source code
-        console.log('Step 1: Scanning source code...');
-        const scanResult = await scanDirectory(sourcePath, {
-            include: config.source.include,
-            exclude: config.source.exclude,
-            onProgress: (current, total, file) => {
-                process.stdout.write(`\r  [${current}/${total}] ${file.slice(-50).padStart(50)}`);
-            }
-        });
-        console.log('');
-        if (scanResult.errors.length > 0) {
-            console.log('\n  Scan warnings:');
-            scanResult.errors.slice(0, 5).forEach(e => console.log(`    - ${e}`));
-            if (scanResult.errors.length > 5) {
-                console.log(`    ... and ${scanResult.errors.length - 5} more`);
-            }
-        }
-        console.log(`\n  Found ${scanResult.totalElements} API elements in ${scanResult.files.length} files`);
-        if (scanResult.totalElements === 0) {
-            console.log('  No API elements found. Nothing to generate.');
-            process.exit(0);
-        }
-        // Collect all elements
-        let allElements = [];
-        for (const file of scanResult.files) {
-            for (const el of file.elements) {
-                allElements.push(el);
-            }
-        }
-        // Apply privacy filters
-        const initialCount = allElements.length;
-        // 1. --public-only: filter to exported/public APIs only
-        if (options.publicOnly) {
-            allElements = allElements.filter(el => el.isExported === true || el.isPublic === true);
-            console.log(`  --public-only: filtered to ${allElements.length} exported APIs`);
-        }
-        // 2. Load .skryptignore patterns
-        const ignorePatterns = readIgnorePatterns(sourcePath);
-        if (ignorePatterns.length > 0) {
-            console.log(`  .skryptignore: loaded ${ignorePatterns.length} patterns`);
-        }
-        // 3. Combine with --exclude patterns
-        const excludePatterns = [...ignorePatterns, ...(options.exclude || [])];
-        if (excludePatterns.length > 0) {
-            allElements = allElements.filter(el => !shouldExcludeElement(el, excludePatterns));
-            if (options.exclude?.length) {
-                console.log(`  --exclude: applied ${options.exclude.length} additional patterns`);
-            }
-        }
-        if (initialCount !== allElements.length) {
-            console.log(`  Filtered: ${initialCount} -> ${allElements.length} elements`);
-        }
-        // Show summary by kind
-        const byKind = {};
-        for (const el of allElements) {
-            byKind[el.kind] = (byKind[el.kind] || 0) + 1;
-        }
-        const pluralize = (word, count) => {
-            if (count === 1)
-                return word;
-            if (word === 'class')
-                return 'classes';
-            return word + 's';
-        };
-        console.log('  ' + Object.entries(byKind).map(([k, v]) => `${v} ${pluralize(k, v)}`).join(', '));
-        // Dry run - stop here
-        if (options.dryRun) {
-            console.log('\n[dry run - stopping before generation]');
-            process.exit(0);
-        }
-        // Auto-read project context from README for richer doc generation
-        let projectContext;
-        const readmeCandidates = ['README.md', 'README.mdx', 'readme.md', 'README.rst', 'README.txt'];
-        for (const candidate of readmeCandidates) {
-            const readmePath = join(sourcePath, candidate);
-            if (existsSync(readmePath)) {
-                try {
-                    const raw = readFileSync(readmePath, 'utf-8');
-                    // Take the first ~1500 chars (intro/description section, not the whole file)
-                    projectContext = raw.slice(0, 1500);
-                    console.log(`  Project context: loaded from ${candidate}`);
-                }
-                catch {
-                    // Skip if unreadable
-                }
-                break;
-            }
-        }
-        // Also check parent directory (common for monorepos where source is a subdirectory)
-        if (!projectContext) {
-            const parentReadme = join(sourcePath, '..', 'README.md');
-            if (existsSync(parentReadme)) {
-                try {
-                    projectContext = readFileSync(parentReadme, 'utf-8').slice(0, 1500);
-                    console.log('  Project context: loaded from parent README.md');
-                }
-                catch {
-                    // Skip
-                }
-            }
-        }
-        // Context Hub enrichment: fetch third-party API docs if chub is installed
-        let externalContext;
-        const allImports = [];
-        for (const el of allElements) {
-            if (el.imports?.length) {
-                allImports.push(...el.imports);
-            }
-        }
-        const chubIds = extractDependencyIds(allImports);
-        if (chubIds.length > 0 && isChubInstalled()) {
-            console.log(`\n  Context Hub: fetching docs for ${chubIds.length} dependencies...`);
-            externalContext = fetchContextHubDocs(chubIds);
-            if (externalContext.size > 0) {
-                console.log(`  Context Hub: enriching with ${externalContext.size} API references`);
-            }
-        }
-        // Step 2: Generate docs
-        console.log('\nStep 2: Generating documentation...');
-        const client = createLLMClient({
-            provider: config.llm.provider,
-            model: config.llm.model,
-            baseUrl: config.llm.baseUrl
-        });
-        let lastElement = '';
-        const multiLanguage = options.multiLang ?? false;
-        if (multiLanguage) {
-            console.log('  mode: multi-language (TypeScript + Python)');
-        }
-        const docs = await generateForElements(allElements, client, {
-            multiLanguage,
-            externalContext,
-            projectContext,
-            onProgress: (progress) => {
-                if (progress.element !== lastElement) {
-                    if (lastElement)
-                        console.log('');
-                    lastElement = progress.element;
-                }
-                process.stdout.write(`\r  [${progress.current}/${progress.total}] ${progress.element}: ${progress.status}`.padEnd(80));
-            }
-        });
-        console.log('\n');
-        // Step 3: Write output
-        console.log('Step 3: Writing documentation...');
-        const outputPath = resolve(config.output.path);
-        let filesWritten;
-        let totalDocs;
-        if (options.byTopic) {
-            console.log('  mode: by-topic (grouped by concept)');
-            const result = await writeDocsByTopic(docs, outputPath);
-            filesWritten = result.filesWritten;
-            totalDocs = result.totalDocs;
-            console.log(`  topics: ${result.topics.map(t => t.name).join(', ')}`);
-        }
-        else {
-            // Default: file-based output
-            const fileResults = groupDocsByFile(docs);
-            const result = await writeDocsToDirectory(fileResults, outputPath, sourcePath);
-            filesWritten = result.filesWritten;
-            totalDocs = result.totalDocs;
-        }
-        const errorCount = docs.filter(d => d.error).length;
-        const duration = Math.round((Date.now() - startTime) / 1000);
-        console.log(`\n  Wrote ${filesWritten} documentation files to ${outputPath}`);
-        // Copy OpenAPI spec (provided or auto-detected)
-        let specPath = options.openapi ? resolve(options.openapi) : null;
-        // Auto-detect if not provided
-        if (!specPath) {
-            const detected = findOpenAPISpec(sourcePath);
-            if (detected) {
-                specPath = detected;
-                console.log(`\n  Auto-detected OpenAPI spec: ${basename(detected)}`);
-            }
-        }
-        if (specPath) {
-            if (existsSync(specPath)) {
-                const specFilename = basename(specPath);
-                const contentDir = dirname(outputPath);
-                const destPath = resolve(contentDir, specFilename);
-                mkdirSync(dirname(destPath), { recursive: true });
-                copyFileSync(specPath, destPath);
-                console.log(`  Copied OpenAPI spec: ${specFilename} -> ${destPath}`);
-                console.log('  API Playground will be available at /reference');
-            }
-            else if (options.openapi) {
-                console.log(`\n  Warning: OpenAPI spec not found: ${specPath}`);
-            }
-        }
-        // Always generate llms.txt for AEO (Answer Engine Optimization)
-        await writeLlmsTxt(docs, outputPath, {
-            projectName: options.projectName,
-            description: `API documentation for ${options.projectName || basename(sourcePath)}`
-        });
-        console.log(`\n  Generated llms.txt and llms-full.md for AEO`);
-        // Step 4: Embedded QA (auto-fix then check)
-        console.log('\nStep 4: Running QA checks...');
-        const fixReport = fixQAIssues(outputPath);
-        printFixReport(fixReport);
-        const qaReport = runQA(outputPath);
-        printQAReport(qaReport);
-        // Context Hub export prompt (TTY only — skip in CI/piped mode)
-        if (process.stdin.isTTY) {
-            console.log('');
-            console.log('  Context Hub: Make your docs discoverable by AI coding agents.');
-            console.log('  Context Hub is a curated registry by Andrew Ng (7K+ stars)');
-            console.log('  https://github.com/andrewyng/context-hub');
-            console.log('');
-            const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
-            const answer = await new Promise((resolve) => {
-                rl.question('  Export for Context Hub? (y/N) ', (ans) => {
-                    rl.close();
-                    resolve(ans.trim().toLowerCase());
-                });
-            });
-            if (answer === 'y' || answer === 'yes') {
-                const languages = multiLanguage ? ['typescript', 'python'] : ['typescript'];
-                const projName = options.projectName || basename(sourcePath);
-                const result = exportToContextHub(docs, outputPath, {
-                    projectName: projName,
-                    languages,
-                    description: `API documentation for ${projName}`,
-                });
-                console.log(`\n  Exported ${result.filesWritten} files to ${result.outputDir}`);
-                console.log('  See context-hub/README.md for submission instructions');
-            }
-        }
-        console.log('\n=== Summary ===');
-        console.log(`  Total elements: ${totalDocs}`);
-        console.log(`  Generated:      ${totalDocs - errorCount}`);
-        if (errorCount > 0) {
-            console.log(`  Errors:         ${errorCount}`);
-        }
-        console.log(`  Duration:       ${duration}s`);
-        console.log(`  Output:         ${outputPath}`);
-        if (errorCount > 0) {
-            console.log('\n  Elements with errors:');
-            docs.filter(d => d.error).slice(0, 10).forEach(d => {
-                console.log(`    - ${d.element.name}: ${d.error?.slice(0, 50)}`);
-            });
-            if (errorCount > 10) {
-                console.log(`    ... and ${errorCount - 10} more`);
-            }
-        }
-        console.log('\nDone!');
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        console.error(`Error: ${message}`);
-        process.exit(1);
-    }
-});