skrypt-ai 0.6.1 → 0.8.0

package/dist/testing/extractor.js

@@ -0,0 +1,195 @@
+import { readFileSync, readdirSync, statSync } from 'fs';
+import { join, extname } from 'path';
+const SUPPORTED_LANGUAGES = ['typescript', 'ts', 'javascript', 'js', 'python', 'py'];
+/**
+ * Normalize language to canonical form for comparison
+ */
+const LANGUAGE_CANONICAL = {
+    typescript: 'typescript',
+    ts: 'typescript',
+    javascript: 'javascript',
+    js: 'javascript',
+    python: 'python',
+    py: 'python',
+};
+/**
+ * Find all .md/.mdx files in a directory recursively
+ */
+export function findDocFiles(dir) {
+    const files = [];
+    function walk(currentDir) {
+        const entries = readdirSync(currentDir);
+        for (const entry of entries) {
+            const fullPath = join(currentDir, entry);
+            const stat = statSync(fullPath);
+            if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') {
+                walk(fullPath);
+            }
+            else if (stat.isFile() && (extname(entry) === '.mdx' || extname(entry) === '.md')) {
+                files.push(fullPath);
+            }
+        }
+    }
+    walk(dir);
+    return files;
+}
+/**
+ * Parse a multi-line expected output block.
+ * Supports:
+ *   // Output: single line
+ *   // Output (contains): partial match
+ *   // Output:
+ *   //   line1
+ *   //   line2
+ */
+function parseExpectedOutput(code) {
+    const lines = code.split('\n');
+    let output;
+    let mode = 'exact';
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+        // Check for "// Output (contains): ..."
+        const containsMatch = line.match(/^\/\/\s*Output\s*\(contains\)\s*:\s*(.*)$/);
+        if (containsMatch) {
+            mode = 'contains';
+            const value = containsMatch[1].trim();
+            if (value) {
+                output = value;
+            }
+            else {
+                // Multi-line: collect subsequent // prefixed lines
+                output = collectMultiLineOutput(lines, i + 1);
+            }
+            break;
+        }
+        // Check for "// Output: ..."
+        const exactMatch = line.match(/^\/\/\s*Output\s*:\s*(.*)$/);
+        if (exactMatch) {
+            const value = exactMatch[1].trim();
+            if (value) {
+                output = value;
+            }
+            else {
+                output = collectMultiLineOutput(lines, i + 1);
+            }
+            break;
+        }
+        // Python: # Output: ...
+        const pyContainsMatch = line.match(/^#\s*Output\s*\(contains\)\s*:\s*(.*)$/);
+        if (pyContainsMatch) {
+            mode = 'contains';
+            const value = pyContainsMatch[1].trim();
+            if (value) {
+                output = value;
+            }
+            else {
+                output = collectMultiLineOutput(lines, i + 1, '#');
+            }
+            break;
+        }
+        const pyMatch = line.match(/^#\s*Output\s*:\s*(.*)$/);
+        if (pyMatch) {
+            const value = pyMatch[1].trim();
+            if (value) {
+                output = value;
+            }
+            else {
+                output = collectMultiLineOutput(lines, i + 1, '#');
+            }
+            break;
+        }
+    }
+    return { output, mode };
+}
+function collectMultiLineOutput(lines, startIdx, commentChar = '//') {
+    const outputLines = [];
+    for (let j = startIdx; j < lines.length; j++) {
+        const l = lines[j].trim();
+        if (commentChar === '//') {
+            if (l.startsWith('//')) {
+                outputLines.push(l.replace(/^\/\/\s?/, ''));
+            }
+            else {
+                break;
+            }
+        }
+        else {
+            if (l.startsWith('#')) {
+                outputLines.push(l.replace(/^#\s?/, ''));
+            }
+            else {
+                break;
+            }
+        }
+    }
+    return outputLines.join('\n');
+}
+/**
+ * Parse directive comments from a code snippet
+ */
+function parseDirectives(code, language) {
+    const commentPrefix = ['python', 'py'].includes(language) ? '#' : '//';
+    const lines = code.split('\n');
+    let requiredEnv;
+    let skipReason;
+    let dependencies;
+    for (const line of lines) {
+        const trimmed = line.trim();
+        // Requires: ENV_VAR1, ENV_VAR2
+        const reqMatch = trimmed.match(new RegExp(`^${commentPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*Requires:\\s*(.+)$`));
+        if (reqMatch) {
+            requiredEnv = reqMatch[1].split(',').map(s => s.trim()).filter(Boolean);
+        }
+        // Skip: reason
+        const skipMatch = trimmed.match(new RegExp(`^${commentPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*Skip:\\s*(.+)$`));
+        if (skipMatch) {
+            skipReason = skipMatch[1].trim();
+        }
+        // Deps: pkg1, pkg2
+        const depsMatch = trimmed.match(new RegExp(`^${commentPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*Deps:\\s*(.+)$`));
+        if (depsMatch) {
+            dependencies = depsMatch[1].split(',').map(s => s.trim()).filter(Boolean);
+        }
+    }
+    return { requiredEnv, skipReason, dependencies };
+}
+/**
+ * Extract code blocks from a markdown/MDX file
+ */
+export function extractSnippets(filePath, languageFilter) {
+    const content = readFileSync(filePath, 'utf-8');
+    const snippets = [];
+    const codeBlockRegex = /```(\w+)?[^\n]*\n([\s\S]*?)```/g;
+    let match;
+    while ((match = codeBlockRegex.exec(content)) !== null) {
+        const language = (match[1] || '').toLowerCase();
+        const code = (match[2] || '').trim();
+        if (languageFilter) {
+            const filterCanonical = LANGUAGE_CANONICAL[languageFilter.toLowerCase()] || languageFilter.toLowerCase();
+            const langCanonical = LANGUAGE_CANONICAL[language] || language;
+            if (langCanonical !== filterCanonical) {
+                continue;
+            }
+        }
+        if (!SUPPORTED_LANGUAGES.includes(language)) {
+            continue;
+        }
+        const beforeMatch = content.substring(0, match.index);
+        const lineNumber = beforeMatch.split('\n').length;
+        const { output, mode } = parseExpectedOutput(code);
+        const directives = parseDirectives(code, language);
+        snippets.push({
+            code,
+            language,
+            filePath,
+            lineNumber,
+            index: snippets.length,
+            expectedOutput: output,
+            expectedOutputMode: mode,
+            requiredEnv: directives.requiredEnv,
+            skipReason: directives.skipReason,
+            dependencies: directives.dependencies,
+        });
+    }
+    return snippets;
+}

package/dist/testing/index.d.ts

@@ -0,0 +1,6 @@
+export * from './types.js';
+export * from './extractor.js';
+export * from './comparator.js';
+export * from './env.js';
+export * from './runner.js';
+export * from './docker.js';

package/dist/testing/index.js

@@ -0,0 +1,6 @@
+export * from './types.js';
+export * from './extractor.js';
+export * from './comparator.js';
+export * from './env.js';
+export * from './runner.js';
+export * from './docker.js';

package/dist/testing/runner.d.ts

@@ -0,0 +1,5 @@
+import { ExtractedSnippet, TestResult, RunnerConfig } from './types.js';
+/**
+ * Run a snippet locally in an isolated temp directory with a clean environment
+ */
+export declare function runLocally(snippet: ExtractedSnippet, config: RunnerConfig): Promise<TestResult>;

package/dist/testing/runner.js

@@ -0,0 +1,225 @@
+import { spawn } from 'child_process';
+import { writeFileSync, mkdirSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { randomUUID } from 'crypto';
+import { compareOutput } from './comparator.js';
+import { buildCleanEnv, checkRequiredEnv } from './env.js';
+/**
+ * Run a snippet locally in an isolated temp directory with a clean environment
+ */
+export async function runLocally(snippet, config) {
+    const startTime = Date.now();
+    // Check for skip directive
+    if (snippet.skipReason) {
+        return {
+            snippet,
+            status: 'skip',
+            stdout: '',
+            stderr: `Skipped: ${snippet.skipReason}`,
+            exitCode: 0,
+            duration: 0,
+            environment: 'local',
+        };
+    }
+    // Check required env vars
+    if (snippet.requiredEnv && snippet.requiredEnv.length > 0) {
+        const { ok, missing } = checkRequiredEnv(snippet.requiredEnv, config.envVars);
+        if (!ok) {
+            return {
+                snippet,
+                status: 'skip',
+                stdout: '',
+                stderr: `Missing required env vars: ${missing.join(', ')}`,
+                exitCode: 0,
+                duration: 0,
+                environment: 'local',
+            };
+        }
+    }
+    const tempDir = join(tmpdir(), `skrypt-test-${randomUUID()}`);
+    mkdirSync(tempDir, { recursive: true });
+    try {
+        // Install dependencies if needed
+        if (config.installDeps && snippet.dependencies && snippet.dependencies.length > 0) {
+            const isPython = ['python', 'py'].includes(snippet.language);
+            if (isPython) {
+                await installPythonDeps(snippet.dependencies, tempDir, config.envVars);
+            }
+            else {
+                await installNodeDeps(snippet.dependencies, tempDir, config.envVars);
+            }
+        }
+        const { command, args, tempFile } = prepareExecution(snippet, tempDir);
+        if (!command) {
+            return {
+                snippet,
+                status: 'fail',
+                stdout: '',
+                stderr: `Unsupported language: ${snippet.language}`,
+                exitCode: 1,
+                duration: Date.now() - startTime,
+                environment: 'local',
+            };
+        }
+        writeFileSync(tempFile, snippet.code);
+        const cleanEnv = buildCleanEnv(config.envVars);
+        // Set TMPDIR to the snippet's temp dir for extra isolation
+        cleanEnv.TMPDIR = tempDir;
+        const result = await executeWithTimeout(command, args, config.timeout, tempDir, cleanEnv);
+        const duration = Date.now() - startTime;
+        // Check output if expected
+        if (snippet.expectedOutput !== undefined) {
+            const comparison = compareOutput(result.stdout, snippet.expectedOutput, snippet.expectedOutputMode || 'exact');
+            if (!comparison.match) {
+                return {
+                    snippet,
+                    status: 'output_mismatch',
+                    stdout: result.stdout,
+                    stderr: result.stderr,
+                    exitCode: result.exitCode,
+                    duration,
+                    environment: 'local',
+                    outputMatch: false,
+                    diff: comparison.diff,
+                };
+            }
+            return {
+                snippet,
+                status: result.exitCode === 0 ? 'pass' : 'fail',
+                stdout: result.stdout,
+                stderr: result.stderr,
+                exitCode: result.exitCode,
+                duration,
+                environment: 'local',
+                outputMatch: true,
+            };
+        }
+        return {
+            snippet,
+            status: result.timedOut ? 'timeout' : (result.exitCode === 0 ? 'pass' : 'fail'),
+            stdout: result.stdout,
+            stderr: result.timedOut
+                ? `Timeout: execution exceeded ${config.timeout}ms`
+                : result.stderr,
+            exitCode: result.exitCode,
+            duration,
+            environment: 'local',
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            snippet,
+            status: 'fail',
+            stdout: '',
+            stderr: message,
+            exitCode: 1,
+            duration: Date.now() - startTime,
+            environment: 'local',
+        };
+    }
+    finally {
+        try {
+            rmSync(tempDir, { recursive: true, force: true });
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+}
+/**
+ * Determine the command and args to execute a snippet
+ */
+function prepareExecution(snippet, tempDir) {
+    const isTS = ['typescript', 'ts'].includes(snippet.language);
+    const isJS = ['javascript', 'js'].includes(snippet.language);
+    const isPy = ['python', 'py'].includes(snippet.language);
+    if (isTS) {
+        const tempFile = join(tempDir, 'test.ts');
+        return { command: 'npx', args: ['tsx', tempFile], tempFile };
+    }
+    if (isJS) {
+        const tempFile = join(tempDir, 'test.js');
+        return { command: 'node', args: [tempFile], tempFile };
+    }
+    if (isPy) {
+        const tempFile = join(tempDir, 'test.py');
+        return { command: 'python3', args: [tempFile], tempFile };
+    }
+    return { command: null, args: [], tempFile: join(tempDir, 'test.txt') };
+}
+/**
+ * Install Node.js dependencies in a temp directory
+ */
+async function installNodeDeps(deps, tempDir, envVars) {
+    const pkg = { name: 'skrypt-test', private: true, dependencies: {} };
+    for (const dep of deps) {
+        pkg.dependencies[dep] = '*';
+    }
+    writeFileSync(join(tempDir, 'package.json'), JSON.stringify(pkg));
+    const env = buildCleanEnv(envVars);
+    await executeWithTimeout('npm', ['install', '--prefix', tempDir, '--no-audit', '--no-fund'], 30000, tempDir, env);
+}
+/**
+ * Install Python dependencies in a temp directory using pip
+ */
+async function installPythonDeps(deps, tempDir, envVars) {
+    const env = buildCleanEnv(envVars);
+    env.PIP_TARGET = join(tempDir, 'pip_packages');
+    env.PYTHONPATH = join(tempDir, 'pip_packages');
+    mkdirSync(join(tempDir, 'pip_packages'), { recursive: true });
+    await executeWithTimeout('python3', ['-m', 'pip', 'install', '--target', join(tempDir, 'pip_packages'), '--quiet', ...deps], 60000, tempDir, env);
+}
+/**
+ * Execute a command with timeout
+ */
+function executeWithTimeout(command, args, timeoutMs, cwd, env) {
+    return new Promise((resolve) => {
+        const proc = spawn(command, args, {
+            cwd,
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env,
+        });
+        let stdout = '';
+        let stderr = '';
+        let timedOut = false;
+        const MAX_BUFFER = 1024 * 1024; // 1MB cap to prevent OOM
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            proc.kill('SIGKILL');
+        }, timeoutMs);
+        proc.stdout?.on('data', (data) => {
+            if (stdout.length < MAX_BUFFER) {
+                stdout += data.toString();
+                if (stdout.length > MAX_BUFFER)
+                    stdout = stdout.slice(0, MAX_BUFFER);
+            }
+        });
+        proc.stderr?.on('data', (data) => {
+            if (stderr.length < MAX_BUFFER) {
+                stderr += data.toString();
+                if (stderr.length > MAX_BUFFER)
+                    stderr = stderr.slice(0, MAX_BUFFER);
+            }
+        });
+        proc.on('close', (code) => {
+            clearTimeout(timeout);
+            resolve({
+                exitCode: timedOut ? 1 : (code ?? 1),
+                stdout,
+                stderr,
+                timedOut,
+            });
+        });
+        proc.on('error', (err) => {
+            clearTimeout(timeout);
+            resolve({
+                exitCode: 1,
+                stdout,
+                stderr: err.message,
+                timedOut: false,
+            });
+        });
+    });
+}

package/dist/testing/types.d.ts

@@ -0,0 +1,58 @@
+/**
+ * A code snippet extracted from a documentation file
+ */
+export interface ExtractedSnippet {
+    code: string;
+    language: string;
+    filePath: string;
+    lineNumber: number;
+    index: number;
+    expectedOutput?: string;
+    expectedOutputMode?: 'exact' | 'contains';
+    requiredEnv?: string[];
+    skipReason?: string;
+    dependencies?: string[];
+}
+/**
+ * Result of running a single snippet
+ */
+export interface TestResult {
+    snippet: ExtractedSnippet;
+    status: 'pass' | 'fail' | 'skip' | 'timeout' | 'output_mismatch';
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+    duration: number;
+    environment: string;
+    outputMatch?: boolean;
+    diff?: string;
+}
+/**
+ * Aggregate report for a test run
+ */
+export interface VerificationReport {
+    total: number;
+    passed: number;
+    failed: number;
+    skipped: number;
+    outputMismatches: number;
+    duration: number;
+    results: TestResult[];
+}
+/**
+ * Configuration for the local runner
+ */
+export interface RunnerConfig {
+    timeout: number;
+    envVars: Record<string, string>;
+    installDeps: boolean;
+}
+/**
+ * Configuration for Docker-based multi-environment testing
+ */
+export interface DockerEnvironment {
+    name: string;
+    image: string;
+    command: string;
+    languages: string[];
+}

package/dist/utils/files.d.ts

@@ -7,11 +7,3 @@ export declare function findMdxFiles(dir: string): string[];
  * Convert string to URL-safe slug
  */
 export declare function slugify(str: string): string;
-/**
- * Simple YAML frontmatter parser — splits on --- markers.
- * Returns parsed key-value data and remaining content body.
- */
-export declare function parseFrontmatter(content: string): {
-    data: Record<string, unknown>;
-    content: string;
-};

package/dist/utils/files.js

@@ -47,36 +47,3 @@ export function slugify(str) {
         .replace(/[^a-z0-9]+/g, '-')
         .replace(/^-|-$/g, '');
 }
-/**
- * Simple YAML frontmatter parser — splits on --- markers.
- * Returns parsed key-value data and remaining content body.
- */
-export function parseFrontmatter(content) {
-    const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
-    if (!match)
-        return { data: {}, content };
-    const yamlStr = match[1];
-    const body = match[2];
-    const data = {};
-    for (const line of yamlStr.split('\n')) {
-        const kvMatch = line.match(/^(\w[\w-]*)\s*:\s*(.*)$/);
-        if (!kvMatch)
-            continue;
-        const key = kvMatch[1];
-        let value = kvMatch[2].trim();
-        if (typeof value === 'string') {
-            if ((value.startsWith('"') && value.endsWith('"')) ||
-                (value.startsWith("'") && value.endsWith("'"))) {
-                value = value.slice(1, -1);
-            }
-            else if (value === 'true')
-                value = true;
-            else if (value === 'false')
-                value = false;
-            else if (/^\d+$/.test(value))
-                value = parseInt(value, 10);
-        }
-        data[key] = value;
-    }
-    return { data, content: body };
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skrypt-ai",
-  "version": "0.6.1",
+  "version": "0.8.0",
   "description": "AI-powered documentation generator with code examples",
   "type": "module",
   "main": "dist/cli.js",