skrypt-ai 0.6.0 → 0.7.0

package/dist/testing/runner.js

@@ -0,0 +1,225 @@
+import { spawn } from 'child_process';
+import { writeFileSync, mkdirSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { randomUUID } from 'crypto';
+import { compareOutput } from './comparator.js';
+import { buildCleanEnv, checkRequiredEnv } from './env.js';
+/**
+ * Run a snippet locally in an isolated temp directory with a clean environment
+ */
+export async function runLocally(snippet, config) {
+    const startTime = Date.now();
+    // Check for skip directive
+    if (snippet.skipReason) {
+        return {
+            snippet,
+            status: 'skip',
+            stdout: '',
+            stderr: `Skipped: ${snippet.skipReason}`,
+            exitCode: 0,
+            duration: 0,
+            environment: 'local',
+        };
+    }
+    // Check required env vars
+    if (snippet.requiredEnv && snippet.requiredEnv.length > 0) {
+        const { ok, missing } = checkRequiredEnv(snippet.requiredEnv, config.envVars);
+        if (!ok) {
+            return {
+                snippet,
+                status: 'skip',
+                stdout: '',
+                stderr: `Missing required env vars: ${missing.join(', ')}`,
+                exitCode: 0,
+                duration: 0,
+                environment: 'local',
+            };
+        }
+    }
+    const tempDir = join(tmpdir(), `skrypt-test-${randomUUID()}`);
+    mkdirSync(tempDir, { recursive: true });
+    try {
+        // Install dependencies if needed
+        if (config.installDeps && snippet.dependencies && snippet.dependencies.length > 0) {
+            const isPython = ['python', 'py'].includes(snippet.language);
+            if (isPython) {
+                await installPythonDeps(snippet.dependencies, tempDir, config.envVars);
+            }
+            else {
+                await installNodeDeps(snippet.dependencies, tempDir, config.envVars);
+            }
+        }
+        const { command, args, tempFile } = prepareExecution(snippet, tempDir);
+        if (!command) {
+            return {
+                snippet,
+                status: 'fail',
+                stdout: '',
+                stderr: `Unsupported language: ${snippet.language}`,
+                exitCode: 1,
+                duration: Date.now() - startTime,
+                environment: 'local',
+            };
+        }
+        writeFileSync(tempFile, snippet.code);
+        const cleanEnv = buildCleanEnv(config.envVars);
+        // Set TMPDIR to the snippet's temp dir for extra isolation
+        cleanEnv.TMPDIR = tempDir;
+        const result = await executeWithTimeout(command, args, config.timeout, tempDir, cleanEnv);
+        const duration = Date.now() - startTime;
+        // Check output if expected
+        if (snippet.expectedOutput !== undefined) {
+            const comparison = compareOutput(result.stdout, snippet.expectedOutput, snippet.expectedOutputMode || 'exact');
+            if (!comparison.match) {
+                return {
+                    snippet,
+                    status: 'output_mismatch',
+                    stdout: result.stdout,
+                    stderr: result.stderr,
+                    exitCode: result.exitCode,
+                    duration,
+                    environment: 'local',
+                    outputMatch: false,
+                    diff: comparison.diff,
+                };
+            }
+            return {
+                snippet,
+                status: result.exitCode === 0 ? 'pass' : 'fail',
+                stdout: result.stdout,
+                stderr: result.stderr,
+                exitCode: result.exitCode,
+                duration,
+                environment: 'local',
+                outputMatch: true,
+            };
+        }
+        return {
+            snippet,
+            status: result.timedOut ? 'timeout' : (result.exitCode === 0 ? 'pass' : 'fail'),
+            stdout: result.stdout,
+            stderr: result.timedOut
+                ? `Timeout: execution exceeded ${config.timeout}ms`
+                : result.stderr,
+            exitCode: result.exitCode,
+            duration,
+            environment: 'local',
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            snippet,
+            status: 'fail',
+            stdout: '',
+            stderr: message,
+            exitCode: 1,
+            duration: Date.now() - startTime,
+            environment: 'local',
+        };
+    }
+    finally {
+        try {
+            rmSync(tempDir, { recursive: true, force: true });
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+}
+/**
+ * Determine the command and args to execute a snippet
+ */
+function prepareExecution(snippet, tempDir) {
+    const isTS = ['typescript', 'ts'].includes(snippet.language);
+    const isJS = ['javascript', 'js'].includes(snippet.language);
+    const isPy = ['python', 'py'].includes(snippet.language);
+    if (isTS) {
+        const tempFile = join(tempDir, 'test.ts');
+        return { command: 'npx', args: ['tsx', tempFile], tempFile };
+    }
+    if (isJS) {
+        const tempFile = join(tempDir, 'test.js');
+        return { command: 'node', args: [tempFile], tempFile };
+    }
+    if (isPy) {
+        const tempFile = join(tempDir, 'test.py');
+        return { command: 'python3', args: [tempFile], tempFile };
+    }
+    return { command: null, args: [], tempFile: join(tempDir, 'test.txt') };
+}
+/**
+ * Install Node.js dependencies in a temp directory
+ */
+async function installNodeDeps(deps, tempDir, envVars) {
+    const pkg = { name: 'skrypt-test', private: true, dependencies: {} };
+    for (const dep of deps) {
+        pkg.dependencies[dep] = '*';
+    }
+    writeFileSync(join(tempDir, 'package.json'), JSON.stringify(pkg));
+    const env = buildCleanEnv(envVars);
+    await executeWithTimeout('npm', ['install', '--prefix', tempDir, '--no-audit', '--no-fund'], 30000, tempDir, env);
+}
+/**
+ * Install Python dependencies in a temp directory using pip
+ */
+async function installPythonDeps(deps, tempDir, envVars) {
+    const env = buildCleanEnv(envVars);
+    env.PIP_TARGET = join(tempDir, 'pip_packages');
+    env.PYTHONPATH = join(tempDir, 'pip_packages');
+    mkdirSync(join(tempDir, 'pip_packages'), { recursive: true });
+    await executeWithTimeout('python3', ['-m', 'pip', 'install', '--target', join(tempDir, 'pip_packages'), '--quiet', ...deps], 60000, tempDir, env);
+}
+/**
+ * Execute a command with timeout
+ */
+function executeWithTimeout(command, args, timeoutMs, cwd, env) {
+    return new Promise((resolve) => {
+        const proc = spawn(command, args, {
+            cwd,
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env,
+        });
+        let stdout = '';
+        let stderr = '';
+        let timedOut = false;
+        const MAX_BUFFER = 1024 * 1024; // 1MB cap to prevent OOM
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            proc.kill('SIGKILL');
+        }, timeoutMs);
+        proc.stdout?.on('data', (data) => {
+            if (stdout.length < MAX_BUFFER) {
+                stdout += data.toString();
+                if (stdout.length > MAX_BUFFER)
+                    stdout = stdout.slice(0, MAX_BUFFER);
+            }
+        });
+        proc.stderr?.on('data', (data) => {
+            if (stderr.length < MAX_BUFFER) {
+                stderr += data.toString();
+                if (stderr.length > MAX_BUFFER)
+                    stderr = stderr.slice(0, MAX_BUFFER);
+            }
+        });
+        proc.on('close', (code) => {
+            clearTimeout(timeout);
+            resolve({
+                exitCode: timedOut ? 1 : (code ?? 1),
+                stdout,
+                stderr,
+                timedOut,
+            });
+        });
+        proc.on('error', (err) => {
+            clearTimeout(timeout);
+            resolve({
+                exitCode: 1,
+                stdout,
+                stderr: err.message,
+                timedOut: false,
+            });
+        });
+    });
+}

package/dist/testing/types.d.ts

@@ -0,0 +1,58 @@
+/**
+ * A code snippet extracted from a documentation file
+ */
+export interface ExtractedSnippet {
+    code: string;
+    language: string;
+    filePath: string;
+    lineNumber: number;
+    index: number;
+    expectedOutput?: string;
+    expectedOutputMode?: 'exact' | 'contains';
+    requiredEnv?: string[];
+    skipReason?: string;
+    dependencies?: string[];
+}
+/**
+ * Result of running a single snippet
+ */
+export interface TestResult {
+    snippet: ExtractedSnippet;
+    status: 'pass' | 'fail' | 'skip' | 'timeout' | 'output_mismatch';
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+    duration: number;
+    environment: string;
+    outputMatch?: boolean;
+    diff?: string;
+}
+/**
+ * Aggregate report for a test run
+ */
+export interface VerificationReport {
+    total: number;
+    passed: number;
+    failed: number;
+    skipped: number;
+    outputMismatches: number;
+    duration: number;
+    results: TestResult[];
+}
+/**
+ * Configuration for the local runner
+ */
+export interface RunnerConfig {
+    timeout: number;
+    envVars: Record<string, string>;
+    installDeps: boolean;
+}
+/**
+ * Configuration for Docker-based multi-environment testing
+ */
+export interface DockerEnvironment {
+    name: string;
+    image: string;
+    command: string;
+    languages: string[];
+}

package/dist/testing/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skrypt-ai",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "AI-powered documentation generator with code examples",
   "type": "module",
   "main": "dist/cli.js",