skrypt-ai 0.6.0 → 0.7.0

package/dist/commands/test.js

@@ -1,229 +1,136 @@
 import { Command } from 'commander';
-import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
-import { resolve, join, extname, relative } from 'path';
-import { spawn } from 'child_process';
-import { writeFileSync, mkdirSync, rmSync } from 'fs';
-import { tmpdir } from 'os';
-import { randomUUID } from 'crypto';
+import { existsSync, statSync } from 'fs';
+import { resolve, relative } from 'path';
 import { requirePro } from '../auth/index.js';
-const SUPPORTED_LANGUAGES = ['typescript', 'ts', 'javascript', 'js', 'python', 'py'];
+import { findDocFiles, extractSnippets, runLocally, isDockerAvailable, parseEnvironments, getCompatibleEnvironments, runInDocker, loadEnvFile, } from '../testing/index.js';
 /**
- * Find all MDX/MD files in a directory recursively
+ * Format duration in human-readable format
  */
-function findDocFiles(dir) {
-    const files = [];
-    function walk(currentDir) {
-        const entries = readdirSync(currentDir);
-        for (const entry of entries) {
-            const fullPath = join(currentDir, entry);
-            const stat = statSync(fullPath);
-            if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') {
-                walk(fullPath);
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    return `${(ms / 1000).toFixed(2)}s`;
+}
+/**
+ * Print a test result line
+ */
+function printResult(result, basePath, verbose) {
+    const relPath = relative(basePath, result.snippet.filePath);
+    const location = `${relPath}:${result.snippet.lineNumber}`;
+    const lang = result.snippet.language;
+    const env = result.environment !== 'local' ? ` [${result.environment}]` : '';
+    switch (result.status) {
+        case 'pass':
+            console.log(`  \x1b[32m✓\x1b[0m ${location} [${lang}]${env} (${formatDuration(result.duration)})`);
+            if (result.outputMatch) {
+                console.log(`    \x1b[32moutput: ✓ matches\x1b[0m`);
             }
-            else if (stat.isFile() && (extname(entry) === '.mdx' || extname(entry) === '.md')) {
-                files.push(fullPath);
+            if (verbose && result.stdout) {
+                console.log(`    \x1b[90mOutput: ${result.stdout.trim().slice(0, 100)}${result.stdout.length > 100 ? '...' : ''}\x1b[0m`);
             }
-        }
+            break;
+        case 'output_mismatch':
+            console.log(`  \x1b[33m≠\x1b[0m ${location} [${lang}]${env} (${formatDuration(result.duration)}) output mismatch`);
+            if (result.diff) {
+                const diffLines = result.diff.split('\n').slice(0, 6);
+                for (const line of diffLines) {
+                    console.log(`    \x1b[33m${line}\x1b[0m`);
+                }
+            }
+            break;
+        case 'skip':
+            console.log(`  \x1b[33m⊘\x1b[0m ${location} [${lang}]${env} — ${result.stderr}`);
+            break;
+        case 'timeout':
+            console.log(`  \x1b[31m⏱\x1b[0m ${location} [${lang}]${env} — timeout`);
+            break;
+        case 'fail':
+            console.log(`  \x1b[31m✗\x1b[0m ${location} [${lang}]${env} (${formatDuration(result.duration)})`);
+            if (result.stderr) {
+                const errorLines = result.stderr.trim().split('\n').slice(0, 3);
+                for (const line of errorLines) {
+                    console.log(`    \x1b[31m${line}\x1b[0m`);
+                }
+            }
+            break;
     }
-    walk(dir);
-    return files;
 }
 /**
- * Extract code blocks from a markdown/MDX file
+ * Print test summary
  */
-function extractCodeBlocks(filePath, languageFilter) {
-    const content = readFileSync(filePath, 'utf-8');
-    const blocks = [];
-    const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
-    let match;
-    while ((match = codeBlockRegex.exec(content)) !== null) {
-        const language = match[1] || '';
-        const code = match[2] || '';
-        // Skip if language filter is set and doesn't match
-        if (languageFilter && !language.toLowerCase().startsWith(languageFilter.toLowerCase())) {
-            continue;
-        }
-        // Skip unsupported languages
-        if (!SUPPORTED_LANGUAGES.includes(language.toLowerCase())) {
-            continue;
-        }
-        // Calculate line number
-        const beforeMatch = content.substring(0, match.index);
-        const lineNumber = beforeMatch.split('\n').length;
-        blocks.push({
-            code: code.trim(),
-            language: language.toLowerCase(),
-            file: filePath,
-            line: lineNumber,
-            index: blocks.length,
-        });
+function printSummary(report) {
+    console.log('');
+    console.log('=== Test Summary ===');
+    console.log(`  Total:             ${report.total}`);
+    console.log(`  \x1b[32mPassed:\x1b[0m            ${report.passed}`);
+    console.log(`  \x1b[31mFailed:\x1b[0m            ${report.failed}`);
+    if (report.outputMismatches > 0) {
+        console.log(`  \x1b[33mOutput mismatch:\x1b[0m   ${report.outputMismatches}`);
     }
-    return blocks;
+    if (report.skipped > 0) {
+        console.log(`  \x1b[33mSkipped:\x1b[0m           ${report.skipped}`);
+    }
+    console.log(`  Duration:          ${formatDuration(report.duration)}`);
 }
 /**
- * Run a code block in an isolated environment with timeout
+ * Run snippets through Docker multi-environment matrix
  */
-async function runCodeBlock(block, timeoutMs) {
-    const startTime = Date.now();
-    const tempDir = join(tmpdir(), `skrypt-test-${randomUUID()}`);
-    mkdirSync(tempDir, { recursive: true });
-    try {
-        const isTypeScript = ['typescript', 'ts'].includes(block.language);
-        const isPython = ['python', 'py'].includes(block.language);
-        const isJavaScript = ['javascript', 'js'].includes(block.language);
-        let tempFile;
-        let command;
-        let args;
-        if (isTypeScript) {
-            tempFile = join(tempDir, 'test.ts');
-            writeFileSync(tempFile, block.code);
-            // Use tsx for TypeScript execution
-            command = 'npx';
-            args = ['tsx', tempFile];
-        }
-        else if (isJavaScript) {
-            tempFile = join(tempDir, 'test.js');
-            writeFileSync(tempFile, block.code);
-            command = 'node';
-            args = [tempFile];
-        }
-        else if (isPython) {
-            tempFile = join(tempDir, 'test.py');
-            writeFileSync(tempFile, block.code);
-            command = 'python3';
-            args = [tempFile];
-        }
-        else {
-            return {
-                block,
-                passed: false,
-                error: `Unsupported language: ${block.language}`,
-                duration: Date.now() - startTime,
-            };
+async function runMatrix(snippets, envString, config, basePath, verbose) {
+    if (!isDockerAvailable()) {
+        console.log('\n  \x1b[33m⚠ Docker not found. Running with local runtimes only.\x1b[0m');
+        console.log('    Install Docker for multi-environment testing.\n');
+        // Fall back to local execution
+        const results = [];
+        for (const snippet of snippets) {
+            const result = await runLocally(snippet, config);
+            results.push(result);
+            printResult(result, basePath, verbose);
         }
-        const result = await executeWithTimeout(command, args, timeoutMs, tempDir);
-        const duration = Date.now() - startTime;
-        return {
-            block,
-            passed: result.exitCode === 0,
-            output: result.stdout,
-            error: result.stderr || (result.exitCode !== 0 ? `Exit code: ${result.exitCode}` : undefined),
-            duration,
-        };
+        return results;
     }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        return {
-            block,
-            passed: false,
-            error: message,
-            duration: Date.now() - startTime,
-        };
-    }
-    finally {
-        // Cleanup temp directory
-        try {
-            rmSync(tempDir, { recursive: true, force: true });
-        }
-        catch {
-            // Ignore cleanup errors — OS will clean tmpdir
-        }
+    const environments = parseEnvironments(envString);
+    if (environments.length === 0) {
+        console.error('  No valid environments specified.');
+        return [];
     }
-}
-/**
- * Execute a command with timeout
- */
-function executeWithTimeout(command, args, timeoutMs, cwd) {
-    return new Promise((resolve) => {
-        const proc = spawn(command, args, {
-            cwd,
-            stdio: ['pipe', 'pipe', 'pipe'],
-            env: { ...process.env, NODE_NO_WARNINGS: '1' },
-        });
-        let stdout = '';
-        let stderr = '';
-        let killed = false;
-        const timeout = setTimeout(() => {
-            killed = true;
-            proc.kill('SIGKILL');
-        }, timeoutMs);
-        proc.stdout?.on('data', (data) => {
-            stdout += data.toString();
-        });
-        proc.stderr?.on('data', (data) => {
-            stderr += data.toString();
-        });
-        proc.on('close', (code) => {
-            clearTimeout(timeout);
-            if (killed) {
-                resolve({
-                    exitCode: 1,
-                    stdout,
-                    stderr: `Timeout: code execution exceeded ${timeoutMs}ms`,
-                });
-            }
-            else {
-                resolve({
-                    exitCode: code ?? 1,
-                    stdout,
-                    stderr,
-                });
-            }
-        });
-        proc.on('error', (err) => {
-            clearTimeout(timeout);
-            resolve({
-                exitCode: 1,
-                stdout,
-                stderr: err.message,
+    console.log(`  Environments: ${environments.map(e => e.name).join(', ')}\n`);
+    const results = [];
+    for (const snippet of snippets) {
+        const compatible = getCompatibleEnvironments(snippet.language, environments);
+        if (compatible.length === 0) {
+            // No compatible environment — skip
+            results.push({
+                snippet,
+                status: 'skip',
+                stdout: '',
+                stderr: `No compatible environment for ${snippet.language}`,
+                exitCode: 0,
+                duration: 0,
+                environment: 'none',
             });
-        });
-    });
-}
-/**
- * Format duration in human-readable format
- */
-function formatDuration(ms) {
-    if (ms < 1000)
-        return `${ms}ms`;
-    return `${(ms / 1000).toFixed(2)}s`;
-}
-/**
- * Print test result
- */
-function printResult(result, basePath, verbose) {
-    const relPath = relative(basePath, result.block.file);
-    const location = `${relPath}:${result.block.line}`;
-    const lang = result.block.language;
-    if (result.passed) {
-        console.log(`  \x1b[32m✓\x1b[0m ${location} [${lang}] (${formatDuration(result.duration)})`);
-        if (verbose && result.output) {
-            console.log(`    \x1b[90mOutput: ${result.output.trim().slice(0, 100)}${result.output.length > 100 ? '...' : ''}\x1b[0m`);
+            continue;
         }
-    }
-    else {
-        console.log(`  \x1b[31m✗\x1b[0m ${location} [${lang}] (${formatDuration(result.duration)})`);
-        if (result.error) {
-            const errorLines = result.error.trim().split('\n').slice(0, 3);
-            for (const line of errorLines) {
-                console.log(`    \x1b[31m${line}\x1b[0m`);
+        const relPath = relative(basePath, snippet.filePath);
+        console.log(`  ${relPath}:${snippet.lineNumber} [${snippet.language}]`);
+        for (const env of compatible) {
+            const result = await runInDocker(snippet, env, config);
+            results.push(result);
+            const statusIcon = result.status === 'pass' ? '\x1b[32m✓\x1b[0m'
+                : result.status === 'output_mismatch' ? '\x1b[33m≠\x1b[0m'
+                    : result.status === 'skip' ? '\x1b[33m⊘\x1b[0m'
+                        : '\x1b[31m✗\x1b[0m';
+            let outputInfo = '';
+            if (result.outputMatch !== undefined) {
+                outputInfo = result.outputMatch ? '  output: \x1b[32m✓ matches\x1b[0m' : '  output: \x1b[31m✗ mismatch\x1b[0m';
+            }
+            console.log(`    ${result.environment.padEnd(20)} ${statusIcon} (${formatDuration(result.duration)})${outputInfo}`);
+            if (result.status === 'fail' && result.stderr) {
+                const firstLine = result.stderr.trim().split('\n')[0] || '';
+                console.log(`      \x1b[31m${firstLine.slice(0, 80)}\x1b[0m`);
             }
         }
     }
-}
-/**
- * Print summary
- */
-function printSummary(summary) {
-    console.log('');
-    console.log('=== Test Summary ===');
-    console.log(`  Total:    ${summary.total}`);
-    console.log(`  \x1b[32mPassed:\x1b[0m   ${summary.passed}`);
-    console.log(`  \x1b[31mFailed:\x1b[0m   ${summary.failed}`);
-    if (summary.skipped > 0) {
-        console.log(`  \x1b[33mSkipped:\x1b[0m  ${summary.skipped}`);
-    }
-    console.log(`  Duration: ${formatDuration(summary.duration)}`);
+    return results;
 }
 export const testCommand = new Command('test')
     .description('Test code examples in documentation files')
@@ -233,28 +140,52 @@ export const testCommand = new Command('test')
     .option('--fix', 'Auto-fix failing examples using autofix command')
     .option('-t, --timeout <ms>', 'Timeout per code block in milliseconds', '10000')
     .option('-v, --verbose', 'Show detailed output')
+    .option('--verify-output', 'Compare stdout against // Output: comments')
+    .option('--env-file <file>', 'Load environment variables from file')
+    .option('--environments <list>', 'Comma-separated Docker environments (e.g. node-20,python-3.12)')
     .action(async (path, options) => {
     try {
-        // Pro feature - requires subscription
-        if (!await requirePro('test')) {
-            process.exit(1);
+        // Docker multi-env testing is Pro-only
+        if (options.environments) {
+            if (!await requirePro('test --environments')) {
+                process.exit(1);
+            }
         }
         const targetPath = resolve(options.file || path);
         if (!existsSync(targetPath)) {
             console.error(`Error: Path not found: ${targetPath}`);
             process.exit(1);
         }
-        const timeoutMs = parseInt(options.timeout);
+        const timeoutMs = parseInt(options.timeout, 10);
         if (isNaN(timeoutMs) || timeoutMs <= 0) {
             console.error(`Error: Invalid timeout value: ${options.timeout}`);
             process.exit(1);
         }
+        // Load env vars
+        let envVars = {};
+        if (options.envFile) {
+            const envPath = resolve(options.envFile);
+            try {
+                envVars = loadEnvFile(envPath);
+                console.log(`  Loaded ${Object.keys(envVars).length} env var(s) from ${options.envFile}`);
+            }
+            catch (err) {
+                console.error(`Error loading env file: ${err instanceof Error ? err.message : err}`);
+                process.exit(1);
+            }
+        }
         console.log('skrypt test');
         console.log(`  path: ${targetPath}`);
         if (options.language) {
             console.log(`  language: ${options.language}`);
         }
         console.log(`  timeout: ${timeoutMs}ms`);
+        if (options.verifyOutput) {
+            console.log('  verify-output: enabled');
+        }
+        if (options.environments) {
+            console.log(`  environments: ${options.environments}`);
+        }
         console.log('');
         // Find all doc files
         const files = statSync(targetPath).isDirectory()
@@ -264,54 +195,64 @@ export const testCommand = new Command('test')
             console.log('No .md or .mdx files found.');
             process.exit(0);
         }
-        // Extract all code blocks
-        const allBlocks = [];
+        // Extract all snippets
+        const allSnippets = [];
         for (const file of files) {
-            const blocks = extractCodeBlocks(file, options.language);
-            allBlocks.push(...blocks);
+            const snippets = extractSnippets(file, options.language);
+            allSnippets.push(...snippets);
         }
-        if (allBlocks.length === 0) {
+        if (allSnippets.length === 0) {
             console.log('No testable code blocks found.');
             if (options.language) {
                 console.log(`  (filtered by language: ${options.language})`);
             }
-            console.log(`  Supported languages: ${SUPPORTED_LANGUAGES.join(', ')}`);
+            console.log('  Supported languages: typescript, ts, javascript, js, python, py');
             process.exit(0);
         }
-        console.log(`Found ${allBlocks.length} code block(s) in ${files.length} file(s)`);
+        console.log(`Found ${allSnippets.length} code block(s) in ${files.length} file(s)`);
         console.log('');
         console.log('Running tests...\n');
-        const results = [];
-        const failedBlocks = [];
+        const runnerConfig = {
+            timeout: timeoutMs,
+            envVars,
+            installDeps: true,
+        };
+        let results;
         const startTime = Date.now();
-        for (let i = 0; i < allBlocks.length; i++) {
-            const block = allBlocks[i];
-            if (!block)
-                continue;
-            const result = await runCodeBlock(block, timeoutMs);
-            results.push(result);
-            printResult(result, targetPath, options.verbose || false);
-            if (!result.passed) {
-                failedBlocks.push(block);
+        if (options.environments) {
+            // Docker multi-environment matrix
+            results = await runMatrix(allSnippets, options.environments, runnerConfig, targetPath, options.verbose || false);
+        }
+        else {
+            // Local execution (default)
+            results = [];
+            for (const snippet of allSnippets) {
+                // If --verify-output is not set, clear expectedOutput so we only check exit code
+                const snippetToRun = options.verifyOutput ? snippet : { ...snippet, expectedOutput: undefined };
+                const result = await runLocally(snippetToRun, runnerConfig);
+                results.push(result);
+                printResult(result, targetPath, options.verbose || false);
             }
         }
         const totalDuration = Date.now() - startTime;
-        const summary = {
-            total: allBlocks.length,
-            passed: results.filter(r => r.passed).length,
-            failed: results.filter(r => !r.passed).length,
-            skipped: 0,
+        const report = {
+            total: results.length,
+            passed: results.filter(r => r.status === 'pass').length,
+            failed: results.filter(r => r.status === 'fail' || r.status === 'timeout').length,
+            skipped: results.filter(r => r.status === 'skip').length,
+            outputMismatches: results.filter(r => r.status === 'output_mismatch').length,
             duration: totalDuration,
+            results,
         };
-        printSummary(summary);
+        printSummary(report);
         // Handle --fix flag
-        if (options.fix && failedBlocks.length > 0) {
+        const failedSnippets = results.filter(r => r.status === 'fail' || r.status === 'output_mismatch');
+        if (options.fix && failedSnippets.length > 0) {
             console.log('');
             console.log('Attempting to auto-fix failing examples...');
             console.log('  Run: skrypt autofix <file> for each failing file');
             console.log('');
-            // Get unique files with failures
-            const failedFiles = [...new Set(failedBlocks.map(b => b.file))];
+            const failedFiles = [...new Set(failedSnippets.map(r => r.snippet.filePath))];
             for (const file of failedFiles) {
                 console.log(`  → ${relative(targetPath, file)}`);
             }
@@ -322,7 +263,7 @@ export const testCommand = new Command('test')
             }
         }
         // Exit with error code if tests failed
-        if (summary.failed > 0) {
+        if (report.failed > 0 || report.outputMismatches > 0) {
             process.exit(1);
         }
         console.log('\nAll tests passed!');

package/dist/commands/watch.js

@@ -10,17 +10,22 @@ import { generateForElements, groupDocsByFile, writeDocsToDirectory } from '../g
 import { runQA, printQAReport, fixQAIssues, printFixReport } from '../qa/index.js';
 export const watchCommand = new Command('watch')
     .description('Watch source files and regenerate docs on changes')
-    .argument('<source>', 'Source directory to watch')
+    .argument('[sources...]', 'Source directories to watch')
     .option('-o, --output <dir>', 'Output directory')
     .option('-c, --config <file>', 'Config file path')
     .option('--provider <name>', 'LLM provider')
     .option('--model <name>', 'LLM model name')
     .option('--debounce <ms>', 'Debounce time in milliseconds', '1000')
-    .action(async (source, options) => {
+    .action(async (sources = [], options) => {
     try {
         const config = loadConfig(options.config);
-        if (source)
-            config.source.path = source;
+        if (sources.length === 0) {
+            // Fall back to config source path
+            sources = [config.source.path];
+        }
+        else {
+            config.source.path = sources[0];
+        }
         if (options.output)
             config.output.path = options.output;
         if (options.provider) {
@@ -41,14 +46,16 @@ export const watchCommand = new Command('watch')
             console.error(`Error: ${envKey} required`);
             process.exit(1);
         }
-        const sourcePath = resolve(config.source.path);
+        const sourcePaths = sources.map(s => resolve(s));
         const outputPath = resolve(config.output.path);
-        if (!existsSync(sourcePath)) {
-            console.error(`Source not found: ${sourcePath}`);
-            process.exit(1);
+        for (const sp of sourcePaths) {
+            if (!existsSync(sp)) {
+                console.error(`Source not found: ${sp}`);
+                process.exit(1);
+            }
         }
         console.log('skrypt watch');
-        console.log(`  source: ${sourcePath}`);
+        console.log(`  source: ${sourcePaths.join(', ')}`);
         console.log(`  output: ${outputPath}`);
         console.log(`  provider: ${config.llm.provider}`);
         console.log('');
@@ -73,25 +80,27 @@ export const watchCommand = new Command('watch')
             const startTime = Date.now();
             try {
                 console.log('\n[' + new Date().toLocaleTimeString() + '] Regenerating docs...');
-                const scanResult = await scanDirectory(sourcePath, {
-                    include: config.source.include,
-                    exclude: config.source.exclude,
-                });
-                if (scanResult.totalElements === 0) {
+                const allElements = [];
+                for (const sp of sourcePaths) {
+                    const scanResult = await scanDirectory(sp, {
+                        include: config.source.include,
+                        exclude: config.source.exclude,
+                    });
+                    for (const file of scanResult.files) {
+                        allElements.push(...file.elements);
+                    }
+                }
+                if (allElements.length === 0) {
                     console.log('  No elements found');
                     return;
                 }
-                const allElements = [];
-                for (const file of scanResult.files) {
-                    allElements.push(...file.elements);
-                }
                 const docs = await generateForElements(allElements, client, {
                     onProgress: (p) => {
                         process.stdout.write(`\r  [${p.current}/${p.total}] ${p.element}`.padEnd(60));
                     }
                 });
                 const fileResults = groupDocsByFile(docs);
-                await writeDocsToDirectory(fileResults, outputPath, sourcePath);
+                await writeDocsToDirectory(fileResults, outputPath, sourcePaths[0]);
                 // Auto-fix then run QA on generated output
                 const fixReport = fixQAIssues(outputPath);
                 printFixReport(fixReport);
@@ -116,7 +125,7 @@ export const watchCommand = new Command('watch')
         // Watch for changes
         console.log('\nWatching for changes... (Ctrl+C to stop)');
         let debounceTimer = null;
-        const watcher = watch(sourcePath, {
+        const watcher = watch(sourcePaths, {
             ignored: [
                 '**/node_modules/**',
                 '**/.git/**',

package/dist/config/loader.d.ts

@@ -1,7 +1,12 @@
-import { Config, LLMProvider } from './types.js';
+import { Config, LLMProvider, SourceEntry } from './types.js';
 export declare function findConfigFile(dir: string): string | null;
 export declare function loadConfig(configPath?: string): Config;
 export declare function validateConfig(config: Config): string[];
+/**
+ * Resolve v1 `source` or v2 `sources` into a normalized array of SourceEntry.
+ * Falls back to the single source config if no sources array is defined.
+ */
+export declare function resolveSourceEntries(config: Config): SourceEntry[];
 export declare function checkApiKey(provider: LLMProvider): {
     ok: boolean;
     envKey: string | null;