skrypt-ai 0.6.1 → 0.8.0

package/dist/commands/generate/index.js

@@ -0,0 +1,393 @@
+import { Command } from 'commander';
+import { existsSync, readFileSync, rmSync } from 'fs';
+import { resolve, basename, join } from 'path';
+import { loadConfig, validateConfig, checkApiKey } from '../../config/index.js';
+import { DEFAULT_MODELS } from '../../config/types.js';
+import { createLLMClient } from '../../llm/index.js';
+import { generateForElements } from '../../generator/index.js';
+import { showSecurityNotice } from '../../auth/notices.js';
+import { requirePro, getAuthConfigAsync } from '../../auth/index.js';
+import { ProxyClient, startProxySession, completeProxySession } from '../../llm/proxy-client.js';
+import { runQA, printQAReport, fixQAIssues, printFixReport } from '../../qa/index.js';
+import { extractDependencyIds, isChubInstalled, fetchContextHubDocs, exportToContextHub } from '../../context-hub/index.js';
+import { planSmartStructure, generateWithStructure } from '../../structure/index.js';
+import * as readline from 'readline';
+import { scanSources } from './scan.js';
+import { writeDocs, writeAssets } from './write.js';
+import { verifyCodeExamples } from './verify.js';
+function promptUser(question) {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    return new Promise(resolve => {
+        rl.question(question, answer => {
+            rl.close();
+            resolve(answer.trim());
+        });
+    });
+}
+export { parseSourceArgs } from './scan.js';
+export const generateCommand = new Command('generate')
+    .description('Generate documentation with code examples')
+    .argument('[sources...]', 'Source directories to scan (use dir:Label for labels)')
+    .option('-o, --output <dir>', 'Output directory')
+    .option('-c, --config <file>', 'Config file path')
+    .option('--provider <name>', 'LLM provider (deepseek, openai, anthropic, google, ollama, openrouter)')
+    .option('--model <name>', 'LLM model name')
+    .option('--base-url <url>', 'Custom API base URL (for Ollama or proxies)')
+    .option('--dry-run', 'Scan only, do not generate docs')
+    .option('--multi-lang', 'Generate TypeScript + Python examples')
+    .option('--by-topic', 'Organize output by topic instead of file')
+    .option('--openapi <file>', 'Include OpenAPI spec file for API Playground')
+    .option('--public-only', 'Only document exported/public APIs')
+    .option('--exclude <patterns...>', 'Exclude patterns (files, names, or name:pattern)')
+    .option('--llms-txt', 'Generate llms.txt for Answer Engine Optimization (AEO)')
+    .option('--project-name <name>', 'Project name for llms.txt header')
+    .option('--org <name>', 'GitHub organization to discover repos from')
+    .option('--repos <list>', 'Comma-separated list of repos to include (with --org)')
+    .option('--exclude-repos <list>', 'Comma-separated list of repos to exclude (with --org)')
+    .option('--verify', 'Verify generated code examples by running them (Pro)')
+    .option('--env-file <file>', 'Load environment variables for code verification')
+    .option('--smart-structure', 'Organize docs by user journey instead of file structure (Pro)')
+    .option('--max-verify-iterations <n>', 'Max re-generation attempts for failing snippets', '2')
+    .option('--no-agents-md', 'Skip AGENTS.md / CLAUDE.md generation')
+    .action(async (sources = [], options) => {
+    try {
+        const startTime = Date.now();
+        // Require at least one source or --org
+        if (sources.length === 0 && !options.org) {
+            console.error('Error: At least one source directory is required (or use --org)');
+            process.exit(1);
+        }
+        // Load config (file or defaults)
+        const config = loadConfig(options.config);
+        // CLI flags override config — first source is used for single-source compat
+        if (sources.length > 0)
+            config.source.path = sources[0];
+        if (options.output)
+            config.output.path = options.output;
+        if (options.provider) {
+            const validProviders = ['deepseek', 'openai', 'anthropic', 'google', 'ollama', 'openrouter'];
+            if (!validProviders.includes(options.provider)) {
+                console.error(`Error: Unknown provider "${options.provider}". Valid: ${validProviders.join(', ')}`);
+                process.exit(1);
+            }
+            config.llm.provider = options.provider;
+            // Use provider's default model unless explicitly specified
+            if (!options.model) {
+                config.llm.model = DEFAULT_MODELS[config.llm.provider];
+            }
+        }
+        if (options.model)
+            config.llm.model = options.model;
+        if (options.baseUrl)
+            config.llm.baseUrl = options.baseUrl;
+        // Validate
+        const errors = validateConfig(config);
+        if (errors.length > 0) {
+            console.error('Config errors:');
+            errors.forEach(e => console.error(`  - ${e}`));
+            process.exit(1);
+        }
+        // Determine LLM routing: own key (direct) or Skrypt proxy
+        let useProxy = false;
+        let proxySessionId;
+        let proxyApiKey;
+        if (!options.dryRun && config.llm.provider !== 'ollama') {
+            const { ok } = checkApiKey(config.llm.provider);
+            if (!ok) {
+                // No own API key — check if user is logged into Skrypt
+                const authConfig = await getAuthConfigAsync();
+                if (!authConfig.apiKey) {
+                    console.error('\n  No API key found. Choose one:\n');
+                    console.error(`  1. Set your own key:  export ${config.llm.provider === 'anthropic' ? 'ANTHROPIC_API_KEY' : 'OPENAI_API_KEY'}=sk-...`);
+                    console.error('  2. Use Skrypt proxy:  skrypt login\n');
+                    process.exit(1);
+                }
+                // User is logged in — use Skrypt proxy
+                proxyApiKey = authConfig.apiKey;
+                useProxy = true;
+                // Prompt for free tier confirmation
+                try {
+                    const session = await startProxySession(proxyApiKey);
+                    proxySessionId = session.sessionId;
+                    if (session.plan === 'free') {
+                        const limit = session.remaining + session.used;
+                        const answer = await promptUser(`\n  This will use your free generation for this month (${session.used}/${limit} used). Continue? (Y/n) `);
+                        if (answer.toLowerCase() === 'n') {
+                            // Complete the session so it doesn't count as abandoned
+                            await completeProxySession(proxyApiKey, proxySessionId);
+                            console.log('\n  Cancelled. Set your own API key or upgrade to Pro: https://skrypt.sh/pro\n');
+                            process.exit(0);
+                        }
+                    }
+                }
+                catch (err) {
+                    console.error(`\n  ${err instanceof Error ? err.message : 'Failed to start proxy session'}`);
+                    process.exit(1);
+                }
+            }
+        }
+        // Pro-gated flags
+        if (options.verify) {
+            if (!await requirePro('generate --verify')) {
+                process.exit(1);
+            }
+        }
+        if (options.smartStructure) {
+            if (!await requirePro('generate --smart-structure')) {
+                process.exit(1);
+            }
+        }
+        // First-run security notice
+        showSecurityNotice();
+        console.log('skrypt generate');
+        if (options.org) {
+            console.log(`  source:   org:${options.org}`);
+        }
+        else if (sources.length > 1) {
+            console.log(`  sources:  ${sources.join(', ')}`);
+        }
+        else {
+            console.log(`  source:   ${config.source.path}`);
+        }
+        console.log(`  output:   ${config.output.path}`);
+        console.log(`  provider: ${config.llm.provider}`);
+        console.log(`  model:    ${config.llm.model}`);
+        if (config.llm.baseUrl) {
+            console.log(`  base url: ${config.llm.baseUrl}`);
+        }
+        // Routing transparency
+        const providerEnvKeys = {
+            openai: 'OPENAI_API_KEY',
+            anthropic: 'ANTHROPIC_API_KEY',
+            google: 'GOOGLE_API_KEY',
+            deepseek: 'DEEPSEEK_API_KEY',
+        };
+        const providerKey = providerEnvKeys[config.llm.provider];
+        if (providerKey && process.env[providerKey]) {
+            console.log(`  routing:  direct to ${config.llm.provider} (BYOK — your key never touches Skrypt)`);
+        }
+        else if (config.llm.provider !== 'ollama') {
+            console.log('  routing:  via Skrypt API proxy');
+        }
+        console.log('');
+        // Scan sources
+        const scanResult = await scanSources(sources, {
+            org: options.org,
+            repos: options.repos,
+            excludeRepos: options.excludeRepos,
+            publicOnly: options.publicOnly,
+            exclude: options.exclude,
+        }, config);
+        const { allElements, primarySourcePath, tempDirs, isMultiSource } = scanResult;
+        try {
+            if (allElements.length === 0) {
+                return;
+            }
+            // Dry run - stop here
+            if (options.dryRun) {
+                console.log('\n[dry run - stopping before generation]');
+                return;
+            }
+            // Auto-read project context from README for richer doc generation
+            let projectContext;
+            const readmeCandidates = ['README.md', 'README.mdx', 'readme.md', 'README.rst', 'README.txt'];
+            for (const candidate of readmeCandidates) {
+                const readmePath = join(primarySourcePath, candidate);
+                if (existsSync(readmePath)) {
+                    try {
+                        const raw = readFileSync(readmePath, 'utf-8');
+                        // Take the first ~1500 chars (intro/description section, not the whole file)
+                        projectContext = raw.slice(0, 1500);
+                        console.log(`  Project context: loaded from ${candidate}`);
+                    }
+                    catch {
+                        // Skip if unreadable
+                    }
+                    break;
+                }
+            }
+            // Also check parent directory (common for monorepos where source is a subdirectory)
+            if (!projectContext) {
+                const parentReadme = join(primarySourcePath, '..', 'README.md');
+                if (existsSync(parentReadme)) {
+                    try {
+                        projectContext = readFileSync(parentReadme, 'utf-8').slice(0, 1500);
+                        console.log('  Project context: loaded from parent README.md');
+                    }
+                    catch {
+                        // Skip
+                    }
+                }
+            }
+            // Context Hub enrichment: fetch third-party API docs if chub is installed
+            let externalContext;
+            const allImports = [];
+            for (const el of allElements) {
+                if (el.imports?.length) {
+                    allImports.push(...el.imports);
+                }
+            }
+            const chubIds = extractDependencyIds(allImports);
+            if (chubIds.length > 0 && isChubInstalled()) {
+                console.log(`\n  Context Hub: fetching docs for ${chubIds.length} dependencies...`);
+                externalContext = fetchContextHubDocs(chubIds);
+                if (externalContext.size > 0) {
+                    console.log(`  Context Hub: enriching with ${externalContext.size} API references`);
+                }
+            }
+            // Step 2: Generate docs
+            console.log('\nStep 2: Generating documentation...');
+            const client = useProxy && proxyApiKey && proxySessionId
+                ? new ProxyClient({ apiKey: proxyApiKey, sessionId: proxySessionId })
+                : createLLMClient({
+                    provider: config.llm.provider,
+                    model: config.llm.model,
+                    baseUrl: config.llm.baseUrl
+                });
+            let lastElement = '';
+            const multiLanguage = options.multiLang ?? false;
+            if (multiLanguage) {
+                console.log('  mode: multi-language (TypeScript + Python)');
+            }
+            const genOptions = {
+                multiLanguage,
+                externalContext,
+                projectContext,
+                onProgress: (progress) => {
+                    if (progress.element !== lastElement) {
+                        if (lastElement)
+                            console.log('');
+                        lastElement = progress.element;
+                    }
+                    process.stdout.write(`\r  [${progress.current}/${progress.total}] ${progress.element}: ${progress.status}`.padEnd(80));
+                }
+            };
+            // Step 3: Write output
+            const outputPath = resolve(config.output.path);
+            let filesWritten;
+            let totalDocs;
+            let docs;
+            let errorCount;
+            if (options.smartStructure) {
+                // Smart structure: LLM-planned page organization
+                console.log('  mode: smart-structure (user-journey organization)');
+                console.log('\n  Planning documentation structure...');
+                const structure = await planSmartStructure(allElements, client);
+                console.log(`  Planned ${structure.pages.length} page(s)`);
+                for (const page of structure.pages) {
+                    console.log(`    ${page.category}/${page.slug}: ${page.elements.length} elements`);
+                }
+                console.log('\nStep 3: Generating & writing structured documentation...');
+                const result = await generateWithStructure(structure, client, outputPath, genOptions);
+                filesWritten = result.filesWritten;
+                totalDocs = result.totalDocs;
+                docs = result.docs;
+                errorCount = docs.filter(d => d.error).length;
+            }
+            else {
+                docs = await generateForElements(allElements, client, genOptions);
+                console.log('\n');
+                console.log('Step 3: Writing documentation...');
+                const writeResult = await writeDocs(docs, outputPath, primarySourcePath, isMultiSource, {
+                    byTopic: options.byTopic,
+                });
+                filesWritten = writeResult.filesWritten;
+                totalDocs = writeResult.totalDocs;
+                errorCount = writeResult.errorCount;
+            }
+            const duration = Math.round((Date.now() - startTime) / 1000);
+            // Post-write assets: OpenAPI, llms.txt, AGENTS.md, manifest
+            await writeAssets(docs, allElements, outputPath, primarySourcePath, config.output.path, filesWritten, {
+                openapi: options.openapi,
+                projectName: options.projectName,
+                baseUrl: options.baseUrl,
+                agentsMd: options.agentsMd,
+            });
+            // Step 4: Verify code examples (if --verify)
+            if (options.verify) {
+                await verifyCodeExamples(outputPath, allElements, client, primarySourcePath, {
+                    envFile: options.envFile,
+                    maxVerifyIterations: options.maxVerifyIterations,
+                    multiLanguage,
+                    externalContext,
+                    projectContext,
+                });
+            }
+            // Step 5: Embedded QA (auto-fix then check)
+            console.log(`\nStep ${options.verify ? '5' : '4'}: Running QA checks...`);
+            const fixReport = fixQAIssues(outputPath);
+            printFixReport(fixReport);
+            const qaReport = runQA(outputPath);
+            printQAReport(qaReport);
+            // Context Hub export prompt (TTY only — skip in CI/piped mode)
+            if (process.stdin.isTTY) {
+                console.log('');
+                console.log('  Context Hub: Make your docs discoverable by AI coding agents.');
+                console.log('  Context Hub is a curated registry by Andrew Ng (7K+ stars)');
+                console.log('  https://github.com/andrewyng/context-hub');
+                console.log('');
+                const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+                const answer = await new Promise((resolve) => {
+                    rl.question('  Export for Context Hub? (y/N) ', (ans) => {
+                        rl.close();
+                        resolve(ans.trim().toLowerCase());
+                    });
+                });
+                if (answer === 'y' || answer === 'yes') {
+                    const languages = multiLanguage ? ['typescript', 'python'] : ['typescript'];
+                    const projName = options.projectName || basename(primarySourcePath);
+                    const result = exportToContextHub(docs, outputPath, {
+                        projectName: projName,
+                        languages,
+                        description: `API documentation for ${projName}`,
+                    });
+                    console.log(`\n  Exported ${result.filesWritten} files to ${result.outputDir}`);
+                    console.log('  See context-hub/README.md for submission instructions');
+                }
+            }
+            console.log('\n=== Summary ===');
+            console.log(`  Total elements: ${totalDocs}`);
+            console.log(`  Generated:      ${totalDocs - errorCount}`);
+            if (errorCount > 0) {
+                console.log(`  Errors:         ${errorCount}`);
+            }
+            console.log(`  Duration:       ${duration}s`);
+            console.log(`  Output:         ${outputPath}`);
+            if (errorCount > 0) {
+                console.log('\n  Elements with errors:');
+                docs.filter(d => d.error).slice(0, 10).forEach(d => {
+                    console.log(`    - ${d.element.name}: ${d.error?.slice(0, 50)}`);
+                });
+                if (errorCount > 10) {
+                    console.log(`    ... and ${errorCount - 10} more`);
+                }
+            }
+            console.log('\nDone!');
+        }
+        finally {
+            // Complete proxy session (increments monthly usage counter)
+            if (useProxy && proxyApiKey && proxySessionId) {
+                try {
+                    await completeProxySession(proxyApiKey, proxySessionId);
+                }
+                catch {
+                    // Don't mask the original error
+                }
+            }
+            // Clean up temp directories from --org clones
+            for (const dir of tempDirs) {
+                try {
+                    rmSync(dir, { recursive: true, force: true });
+                }
+                catch {
+                    // Ignore cleanup errors
+                }
+            }
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(`Error: ${message}`);
+        process.exit(1);
+    }
+});

package/dist/commands/generate/scan.d.ts

@@ -0,0 +1,41 @@
+import { SourceEntry } from '../../config/types.js';
+import { APIElement } from '../../scanner/index.js';
+import type { Config } from '../../config/types.js';
+interface ScanOptions {
+    org?: string;
+    repos?: string;
+    excludeRepos?: string;
+    publicOnly?: boolean;
+    exclude?: string[];
+    dryRun?: boolean;
+}
+export interface ScanResult {
+    allElements: APIElement[];
+    sourceEntries: SourceEntry[];
+    primarySourcePath: string;
+    tempDirs: string[];
+    isMultiSource: boolean;
+}
+/**
+ * Parse source arguments with optional labels.
+ * e.g. "./api:API" -> { path: "./api", label: "API" }
+ * e.g. "./src" -> { path: "./src" }
+ */
+export declare function parseSourceArgs(args: string[]): SourceEntry[];
+/**
+ * Read .skryptignore patterns from source directory
+ */
+export declare function readIgnorePatterns(sourcePath: string): string[];
+/**
+ * Auto-detect OpenAPI spec file in source directory
+ */
+export declare function findOpenAPISpec(sourcePath: string): string | null;
+/**
+ * Check if element should be excluded based on patterns
+ */
+export declare function shouldExcludeElement(element: APIElement, patterns: string[]): boolean;
+/**
+ * Scan all sources (CLI args, --org, or config), apply filters, return elements.
+ */
+export declare function scanSources(sources: string[], options: ScanOptions, config: Config): Promise<ScanResult>;
+export {};