stepproof 0.2.0

package/dist/cli.js

@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import * as path from 'node:path';
+import { parseScenario } from './core/scenario-parser.js';
+import { runScenario } from './core/scenario-runner.js';
+import { writeJsonReport } from './reporters/json-reporter.js';
+import { printReport } from './reporters/terminal-reporter.js';
+import { formatSarif } from './reporters/sarif-reporter.js';
+import { formatJunit } from './reporters/junit-reporter.js';
+import * as fs from 'node:fs';
+import { guard } from '@preflight/license';
+import { runInit } from './commands/init.js';
+const program = new Command();
+program
+    .name('stepproof')
+    .description('Regression testing for multi-step AI workflows. Not observability — a CI gate.')
+    .version('0.2.0')
+    .addHelpText('after', `
+Examples:
+  stepproof init                                        scaffold a starter scenario
+  stepproof run ./scenarios/first-test.yaml             run one scenario
+  stepproof run ./scenarios/                            run all scenarios in a directory
+  stepproof run test.yaml --format sarif --output results.sarif  SARIF output for CI`);
+program
+    .command('init [dir]')
+    .description('Scaffold a starter scenario in ./scenarios/first-test.yaml')
+    .action((dir) => {
+    runInit(dir);
+});
+program
+    .command('run <scenario>')
+    .description('Run a scenario YAML file and report pass rates per step')
+    .option('-n, --iterations <number>', 'Number of iterations to run (overrides scenario file)', parseInt)
+    .option('-o, --output <file>', 'Path for output file (JSON by default; SARIF or JUnit when --format is set)', 'stepproof-report.json')
+    .option('--no-json', 'Skip JSON report output')
+    .option('--quiet', 'Suppress terminal output (use with --output for CI)')
+    .option('--format <format>', 'Output format: sarif, junit')
+    .option('--report <format>', '(deprecated: use --format)')
+    .action(async (scenarioPath, opts) => {
+    // --report is deprecated; normalize to --format
+    if (opts.report && !opts.format) {
+        process.stderr.write('Warning: --report is deprecated, use --format instead\n');
+        opts.format = opts.report;
+    }
+    if (opts.format && opts.format !== 'sarif' && opts.format !== 'junit') {
+        console.error(`\nError: --format must be "sarif" or "junit", got "${opts.format}"`);
+        process.exit(2);
+    }
+    // License gate — check before running the scenario (avoid wasted API calls)
+    if (opts.format === 'sarif' || opts.format === 'junit') {
+        guard('team', { feature: `--format ${opts.format}` });
+    }
+    // --format implies quiet (suppress terminal output) unless --quiet already set
+    const isQuiet = opts.quiet || !!opts.format;
+    const resolvedPath = path.resolve(process.cwd(), scenarioPath);
+    let scenario;
+    try {
+        scenario = parseScenario(resolvedPath);
+    }
+    catch (e) {
+        console.error(`\nError parsing scenario: ${e.message}`);
+        process.exit(2);
+    }
+    if (!isQuiet) {
+        console.log(`\nLoading: ${scenario.name}`);
+        if (opts.iterations) {
+            console.log(`Overriding iterations: ${scenario.iterations ?? 10} → ${opts.iterations}`);
+        }
+    }
+    let currentIteration = 0;
+    const totalIterations = opts.iterations ?? scenario.iterations ?? 10;
+    let report;
+    try {
+        report = await runScenario(scenario, resolvedPath, {
+            iterations: opts.iterations,
+            onIterationComplete: (iteration, total) => {
+                currentIteration = iteration;
+                if (!isQuiet) {
+                    process.stdout.write(`\r  Completed iteration ${iteration}/${total}...`);
+                    if (iteration === total) {
+                        process.stdout.write('\r' + ' '.repeat(50) + '\r');
+                    }
+                }
+            },
+        });
+    }
+    catch (e) {
+        console.error(`\nError running scenario: ${e.message}`);
+        process.exit(2);
+    }
+    // Handle --format sarif / --format junit
+    if (opts.format === 'sarif' || opts.format === 'junit') {
+        const formatted = opts.format === 'sarif' ? formatSarif(report) : formatJunit(report);
+        const hasExplicitOutput = process.argv.includes('--output') || process.argv.includes('-o');
+        if (hasExplicitOutput) {
+            try {
+                fs.writeFileSync(opts.output, formatted, 'utf-8');
+            }
+            catch (e) {
+                console.error(`Warning: Could not write ${opts.format} report: ${e.message}`);
+            }
+        }
+        else {
+            process.stdout.write(formatted + '\n');
+        }
+    }
+    const reportPath = opts.json ? opts.output : undefined;
+    if (!isQuiet) {
+        printReport(report, reportPath);
+    }
+    if (opts.json) {
+        try {
+            writeJsonReport(report, opts.output);
+        }
+        catch (e) {
+            console.error(`Warning: Could not write JSON report: ${e.message}`);
+        }
+    }
+    // Exit 1 if any step below threshold — this is the CI gate
+    if (!report.allPassed) {
+        process.exit(1);
+    }
+    process.exit(0);
+});
+program.parse(process.argv);
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,OAAO,EAAE;;;;;qFAK6D,CAAC,CAAC;AAEvF,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,CAAC,GAAY,EAAE,EAAE;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC;AACf,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,IAOpC,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;IAE/D,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,mBAAmB,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACxC,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}

package/dist/commands/init.d.ts

@@ -0,0 +1,2 @@
+export declare function runInit(outputDir?: string): void;
+//# sourceMappingURL=init.d.ts.map

package/dist/commands/init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAoBA,wBAAgB,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAsBhD"}

package/dist/commands/init.js

@@ -0,0 +1,39 @@
+import { writeFileSync, existsSync, mkdirSync } from 'fs';
+import { resolve } from 'path';
+const SCENARIO_SCAFFOLD = `# stepproof scenario — edit this, then run: stepproof run ./scenarios/first-test.yaml
+name: First Test
+description: "Test your AI agent's response quality"
+iterations: 5
+
+steps:
+  - id: step-1
+    prompt: "Explain what you do in one sentence."
+    threshold: 0.8     # 80% of runs must pass
+    checks:
+      - type: contains
+        value: "AI"    # replace with text you expect in the response
+
+# More check types: contains, not_contains, regex, length_gt, length_lt
+# Full docs: https://github.com/StanislavBG/stepproof
+`;
+export function runInit(outputDir) {
+    const dir = resolve(outputDir ?? './scenarios');
+    const dest = resolve(dir, 'first-test.yaml');
+    if (existsSync(dest)) {
+        console.log(`Scenario already exists: ${dest}`);
+        console.log('Edit it, then run: stepproof run ./scenarios/first-test.yaml');
+        process.exit(0);
+    }
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(dest, SCENARIO_SCAFFOLD, 'utf-8');
+    console.log(`\n✔ Created ${dest}`);
+    console.log('');
+    console.log('Next:');
+    console.log('  1. Edit the scenario — replace the prompt and checks with your actual test');
+    console.log('  2. stepproof run ./scenarios/first-test.yaml');
+    console.log('');
+    console.log('Add to CI: stepproof run ./scenarios/ --format sarif --output results.sarif');
+    console.log('');
+    console.log('Ready for a deploy gate? Try: npx agent-gate init');
+}
+//# sourceMappingURL=init.js.map

package/dist/commands/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;CAezB,CAAC;AAEF,MAAM,UAAU,OAAO,CAAC,SAAkB;IACxC,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAE7C,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,8EAA8E,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;AACnE,CAAC"}

package/dist/core/scenario-parser.d.ts

@@ -0,0 +1,4 @@
+import type { Scenario } from './types.js';
+export declare function parseScenario(filePath: string): Scenario;
+export declare function substituteVariables(template: string, variables: Record<string, string>, stepOutputs: Record<string, string>): string;
+//# sourceMappingURL=scenario-parser.d.ts.map

package/dist/core/scenario-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-parser.d.ts","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAQ,MAAM,YAAY,CAAC;AAEjD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAuCxD;AA0CD,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,MAAM,CAyBR"}

package/dist/core/scenario-parser.js

@@ -0,0 +1,92 @@
+import * as fs from 'node:fs';
+import * as yaml from 'js-yaml';
+export function parseScenario(filePath) {
+    let content;
+    try {
+        content = fs.readFileSync(filePath, 'utf-8');
+    }
+    catch (e) {
+        throw new Error(`Cannot read scenario file: ${filePath}`);
+    }
+    let raw;
+    try {
+        raw = yaml.load(content);
+    }
+    catch (e) {
+        throw new Error(`Invalid YAML in scenario file: ${e.message}`);
+    }
+    if (!raw || typeof raw !== 'object') {
+        throw new Error('Scenario file must be a YAML object');
+    }
+    const scenario = raw;
+    if (!scenario.name || typeof scenario.name !== 'string') {
+        throw new Error('Scenario must have a "name" field (string)');
+    }
+    if (!Array.isArray(scenario.steps) || scenario.steps.length === 0) {
+        throw new Error('Scenario must have a "steps" array with at least one step');
+    }
+    const steps = scenario.steps.map((rawStep, i) => validateStep(rawStep, i));
+    return {
+        name: scenario.name,
+        iterations: typeof scenario.iterations === 'number' ? scenario.iterations : 10,
+        variables: typeof scenario.variables === 'object' && scenario.variables !== null
+            ? scenario.variables
+            : {},
+        steps,
+    };
+}
+function validateStep(raw, index) {
+    if (!raw || typeof raw !== 'object') {
+        throw new Error(`Step ${index + 1} must be an object`);
+    }
+    const step = raw;
+    const pos = step.id ? `step "${step.id}"` : `step ${index + 1}`;
+    if (!step.id || typeof step.id !== 'string') {
+        throw new Error(`Step ${index + 1} must have an "id" field (string)`);
+    }
+    if (!step.provider || !['openai', 'anthropic'].includes(step.provider)) {
+        throw new Error(`${pos}: "provider" must be "openai" or "anthropic"`);
+    }
+    if (!step.model || typeof step.model !== 'string') {
+        throw new Error(`${pos}: "model" field is required (string)`);
+    }
+    if (!step.prompt || typeof step.prompt !== 'string') {
+        throw new Error(`${pos}: "prompt" field is required (string)`);
+    }
+    const minPassRate = typeof step.min_pass_rate === 'number' ? step.min_pass_rate : 0.8;
+    if (minPassRate < 0 || minPassRate > 1) {
+        throw new Error(`${pos}: "min_pass_rate" must be between 0.0 and 1.0`);
+    }
+    return {
+        id: step.id,
+        provider: step.provider,
+        model: step.model,
+        prompt: step.prompt,
+        system: typeof step.system === 'string' ? step.system : undefined,
+        min_pass_rate: minPassRate,
+        assertions: Array.isArray(step.assertions) ? step.assertions : [],
+    };
+}
+export function substituteVariables(template, variables, stepOutputs) {
+    return template.replace(/\{\{([^}]+)\}\}/g, (match, key) => {
+        const trimmed = key.trim();
+        // Check for step output reference: {{step_id.output}}
+        if (trimmed.includes('.')) {
+            const [stepId, field] = trimmed.split('.', 2);
+            if (field === 'output' && stepOutputs[stepId] !== undefined) {
+                return stepOutputs[stepId];
+            }
+        }
+        // Check global variables
+        if (variables[trimmed] !== undefined) {
+            return variables[trimmed];
+        }
+        // Check environment variables
+        if (process.env[trimmed] !== undefined) {
+            return process.env[trimmed];
+        }
+        // Leave unresolved — caller decides if this is an error
+        return match;
+    });
+}
+//# sourceMappingURL=scenario-parser.js.map

package/dist/core/scenario-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-parser.js","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAGhC,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAG,GAA8B,CAAC;IAEhD,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAE5F,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,UAAU,EAAE,OAAO,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC9E,SAAS,EAAE,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI;YAC9E,CAAC,CAAC,QAAQ,CAAC,SAAmC;YAC9C,CAAC,CAAC,EAAE;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC;IAEhE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAkB,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,8CAA8C,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sCAAsC,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uCAAuC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;IACtF,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,+CAA+C,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,QAAQ,EAAE,IAAI,CAAC,QAAkC;QACjD,KAAK,EAAE,IAAI,CAAC,KAAe;QAC3B,MAAM,EAAE,IAAI,CAAC,MAAgB;QAC7B,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACjE,aAAa,EAAE,WAAW;QAC1B,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,SAAiC,EACjC,WAAmC;IAEnC,OAAO,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAE3B,sDAAsD;QACtD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC5D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAC/B,CAAC;QAED,wDAAwD;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/core/scenario-runner.d.ts

@@ -0,0 +1,11 @@
+import type { Scenario, ScenarioReport } from './types.js';
+export interface RunOptions {
+    /** Override iterations from scenario file */
+    iterations?: number;
+    /** Called after each iteration completes */
+    onIterationComplete?: (iteration: number, total: number) => void;
+    /** Called after each step within an iteration */
+    onStepComplete?: (stepId: string, passed: boolean) => void;
+}
+export declare function runScenario(scenario: Scenario, scenarioFilePath: string, options?: RunOptions): Promise<ScenarioReport>;
+//# sourceMappingURL=scenario-runner.d.ts.map

package/dist/core/scenario-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-runner.d.ts","sourceRoot":"","sources":["../../src/core/scenario-runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAA2B,MAAM,YAAY,CAAC;AAEpF,MAAM,WAAW,UAAU;IACzB,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACjE,iDAAiD;IACjD,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;CAC5D;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,MAAM,EACxB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CA8FzB"}

package/dist/core/scenario-runner.js

@@ -0,0 +1,85 @@
+import * as path from 'node:path';
+import { getAdapter } from '../adapters/index.js';
+import { runAssertions } from '../assertions/engine.js';
+import { substituteVariables } from './scenario-parser.js';
+export async function runScenario(scenario, scenarioFilePath, options = {}) {
+    const iterations = options.iterations ?? scenario.iterations ?? 10;
+    const scenarioDir = path.dirname(path.resolve(scenarioFilePath));
+    const variables = scenario.variables ?? {};
+    const startedAt = new Date().toISOString();
+    const startMs = Date.now();
+    const allResults = [];
+    for (let i = 1; i <= iterations; i++) {
+        const stepOutputs = {};
+        for (const step of scenario.steps) {
+            const resolvedPrompt = substituteVariables(step.prompt, variables, stepOutputs);
+            const resolvedSystem = step.system
+                ? substituteVariables(step.system, variables, stepOutputs)
+                : undefined;
+            const stepStartMs = Date.now();
+            let output = '';
+            let error;
+            try {
+                const adapter = getAdapter(step.provider, step.model);
+                output = await adapter.call(resolvedPrompt, resolvedSystem);
+                stepOutputs[step.id] = output;
+            }
+            catch (e) {
+                error = e.message;
+                stepOutputs[step.id] = '';
+            }
+            const durationMs = Date.now() - stepStartMs;
+            let assertionResults = [];
+            let assertionsPassed = false;
+            if (!error) {
+                const { results, allPassed } = await runAssertions(output, step.assertions, scenarioDir);
+                assertionResults = results;
+                assertionsPassed = allPassed;
+            }
+            const stepPassed = !error && assertionsPassed;
+            const result = {
+                stepId: step.id,
+                iteration: i,
+                output,
+                passed: stepPassed,
+                assertionResults,
+                error,
+                durationMs,
+            };
+            allResults.push(result);
+            options.onStepComplete?.(step.id, stepPassed);
+        }
+        options.onIterationComplete?.(i, iterations);
+    }
+    // Aggregate per-step summaries
+    const steps = scenario.steps.map((step) => {
+        const stepResults = allResults.filter((r) => r.stepId === step.id);
+        const passes = stepResults.filter((r) => r.passed).length;
+        const failures = stepResults.length - passes;
+        const passRate = stepResults.length > 0 ? passes / stepResults.length : 0;
+        const minPassRate = step.min_pass_rate ?? 0.8;
+        return {
+            stepId: step.id,
+            totalRuns: stepResults.length,
+            passes,
+            failures,
+            passRate,
+            minPassRate,
+            belowThreshold: passRate < minPassRate,
+        };
+    });
+    const allPassed = steps.every((s) => !s.belowThreshold);
+    const completedAt = new Date().toISOString();
+    const durationMs = Date.now() - startMs;
+    return {
+        scenarioName: scenario.name,
+        iterations,
+        startedAt,
+        completedAt,
+        durationMs,
+        steps,
+        allPassed,
+        results: allResults,
+    };
+}
+//# sourceMappingURL=scenario-runner.js.map

package/dist/core/scenario-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-runner.js","sourceRoot":"","sources":["../../src/core/scenario-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAkB,EAClB,gBAAwB,EACxB,UAAsB,EAAE;IAExB,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IACnE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;IAE3C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAiB,EAAE,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,WAAW,GAA2B,EAAE,CAAC;QAE/C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;YAChF,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM;gBAChC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC;gBAC1D,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,KAAyB,CAAC;YAE9B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBACtD,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;gBAC5D,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,KAAK,GAAI,CAAW,CAAC,OAAO,CAAC;gBAC7B,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;YAE5C,IAAI,gBAAgB,GAA0D,EAAE,CAAC;YACjF,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACzF,gBAAgB,GAAG,OAAO,CAAC;gBAC3B,gBAAgB,GAAG,SAAS,CAAC;YAC/B,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,gBAAgB,CAAC;YAE9C,MAAM,MAAM,GAAe;gBACzB,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,SAAS,EAAE,CAAC;gBACZ,MAAM;gBACN,MAAM,EAAE,UAAU;gBAClB,gBAAgB;gBAChB,KAAK;gBACL,UAAU;aACX,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,GAAkB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvD,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC1D,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC;QAE9C,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,cAAc,EAAE,QAAQ,GAAG,WAAW;SACvC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAExC,OAAO;QACL,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,UAAU;QACV,SAAS;QACT,WAAW;QACX,UAAU;QACV,KAAK;QACL,SAAS;QACT,OAAO,EAAE,UAAU;KACpB,CAAC;AACJ,CAAC"}

package/dist/core/types.d.ts

@@ -0,0 +1,71 @@
+export type Provider = 'openai' | 'anthropic';
+export type AssertionType = 'contains' | 'not_contains' | 'regex' | 'json_schema' | 'llm_judge';
+export interface Assertion {
+    type: AssertionType;
+    /** For contains, not_contains, regex */
+    value?: string;
+    /** For json_schema: path to JSON schema file (relative to scenario file) */
+    schema?: string;
+    /** For llm_judge: the evaluation prompt */
+    prompt?: string;
+    /** For llm_judge: the expected response prefix (default: "yes") */
+    pass_on?: string;
+    /** For llm_judge: override provider (default: anthropic) */
+    provider?: Provider;
+    /** For llm_judge: override model (default: claude-haiku or gpt-4o-mini) */
+    model?: string;
+}
+export interface Step {
+    id: string;
+    provider: Provider;
+    model: string;
+    /** Prompt template. Use {{variable}} for variables, {{step_id.output}} for prior step outputs. */
+    prompt: string;
+    /** Optional system prompt */
+    system?: string;
+    /** Minimum pass rate threshold (0.0–1.0). Default: 0.8 */
+    min_pass_rate?: number;
+    assertions: Assertion[];
+}
+export interface Scenario {
+    name: string;
+    /** Number of iterations to run. Default: 10 */
+    iterations?: number;
+    /** Global variables for template substitution */
+    variables?: Record<string, string>;
+    steps: Step[];
+}
+export interface AssertionResult {
+    type: string;
+    passed: boolean;
+    message?: string;
+}
+export interface StepResult {
+    stepId: string;
+    iteration: number;
+    output: string;
+    passed: boolean;
+    assertionResults: AssertionResult[];
+    error?: string;
+    durationMs: number;
+}
+export interface StepSummary {
+    stepId: string;
+    totalRuns: number;
+    passes: number;
+    failures: number;
+    passRate: number;
+    minPassRate: number;
+    belowThreshold: boolean;
+}
+export interface ScenarioReport {
+    scenarioName: string;
+    iterations: number;
+    startedAt: string;
+    completedAt: string;
+    durationMs: number;
+    steps: StepSummary[];
+    allPassed: boolean;
+    results: StepResult[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE9C,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,GAAG,OAAO,GAAG,aAAa,GAAG,WAAW,CAAC;AAEhG,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,SAAS,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB"}

package/dist/core/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":""}

package/dist/reporters/json-reporter.d.ts

@@ -0,0 +1,4 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function writeJsonReport(report: ScenarioReport, outputPath: string): void;
+export declare function formatJsonReport(report: ScenarioReport): string;
+//# sourceMappingURL=json-reporter.d.ts.map

package/dist/reporters/json-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/json-reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAGhF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAE/D"}

package/dist/reporters/json-reporter.js

@@ -0,0 +1,9 @@
+import * as fs from 'node:fs';
+export function writeJsonReport(report, outputPath) {
+    const json = JSON.stringify(report, null, 2);
+    fs.writeFileSync(outputPath, json, 'utf-8');
+}
+export function formatJsonReport(report) {
+    return JSON.stringify(report, null, 2);
+}
+//# sourceMappingURL=json-reporter.js.map

package/dist/reporters/json-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-reporter.js","sourceRoot":"","sources":["../../src/reporters/json-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAG9B,MAAM,UAAU,eAAe,CAAC,MAAsB,EAAE,UAAkB;IACxE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAsB;IACrD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}

package/dist/reporters/junit-reporter.d.ts

@@ -0,0 +1,3 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function formatJunit(report: ScenarioReport): string;
+//# sourceMappingURL=junit-reporter.d.ts.map

package/dist/reporters/junit-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/junit-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAWvD,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CA8B1D"}

package/dist/reporters/junit-reporter.js

@@ -0,0 +1,34 @@
+function escapeXml(str) {
+    return str
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+export function formatJunit(report) {
+    const totalTests = report.steps.length;
+    const totalFailures = report.steps.filter((s) => s.belowThreshold).length;
+    const timeSeconds = (report.durationMs / 1000).toFixed(3);
+    const testCases = report.steps.map((step) => {
+        const passRatePct = (step.passRate * 100).toFixed(1);
+        const thresholdPct = (step.minPassRate * 100).toFixed(0);
+        const openTag = `    <testcase name="${escapeXml(step.stepId)}" classname="stepproof.steps" time="0">`;
+        if (step.belowThreshold) {
+            const failureMessage = `Pass rate ${passRatePct}% below threshold ${thresholdPct}%`;
+            const failureBody = `${escapeXml(step.stepId)}: ${step.passes}/${step.totalRuns} iterations passed`;
+            return `${openTag}\n      <failure message="${escapeXml(failureMessage)}">${escapeXml(failureBody)}</failure>\n    </testcase>`;
+        }
+        return `${openTag}\n    </testcase>`;
+    });
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites name="stepproof" tests="${totalTests}" failures="${totalFailures}" time="${timeSeconds}">`,
+        `  <testsuite name="${escapeXml(report.scenarioName)}" tests="${totalTests}" failures="${totalFailures}" time="${timeSeconds}">`,
+        ...testCases,
+        '  </testsuite>',
+        '</testsuites>',
+    ];
+    return lines.join('\n');
+}
+//# sourceMappingURL=junit-reporter.js.map

package/dist/reporters/junit-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit-reporter.js","sourceRoot":"","sources":["../../src/reporters/junit-reporter.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,GAAG;SACP,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAsB;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;IACvC,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;IAC1E,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEzD,MAAM,OAAO,GAAG,uBAAuB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,yCAAyC,CAAC;QAEvG,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,cAAc,GAAG,aAAa,WAAW,qBAAqB,YAAY,GAAG,CAAC;YACpF,MAAM,WAAW,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,oBAAoB,CAAC;YACpG,OAAO,GAAG,OAAO,6BAA6B,SAAS,CAAC,cAAc,CAAC,KAAK,SAAS,CAAC,WAAW,CAAC,6BAA6B,CAAC;QAClI,CAAC;QAED,OAAO,GAAG,OAAO,mBAAmB,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG;QACZ,wCAAwC;QACxC,uCAAuC,UAAU,eAAe,aAAa,WAAW,WAAW,IAAI;QACvG,sBAAsB,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,UAAU,eAAe,aAAa,WAAW,WAAW,IAAI;QAChI,GAAG,SAAS;QACZ,gBAAgB;QAChB,eAAe;KAChB,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/reporters/sarif-reporter.d.ts

@@ -0,0 +1,3 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function formatSarif(report: ScenarioReport): string;
+//# sourceMappingURL=sarif-reporter.d.ts.map

package/dist/reporters/sarif-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarif-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/sarif-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAmD1D"}

package/dist/reporters/sarif-reporter.js

@@ -0,0 +1,47 @@
+export function formatSarif(report) {
+    const rules = report.steps.map((step) => ({
+        id: step.stepId,
+        name: step.stepId,
+        shortDescription: {
+            text: `Step: ${step.stepId} — min pass rate ${(step.minPassRate * 100).toFixed(0)}%`,
+        },
+    }));
+    const results = report.steps.map((step) => {
+        const passRatePct = (step.passRate * 100).toFixed(1);
+        const thresholdPct = (step.minPassRate * 100).toFixed(0);
+        if (step.belowThreshold) {
+            return {
+                ruleId: step.stepId,
+                level: 'error',
+                message: {
+                    text: `Step "${step.stepId}" pass rate ${passRatePct}% is below threshold ${thresholdPct}% (${step.passes}/${step.totalRuns} iterations passed)`,
+                },
+            };
+        }
+        return {
+            ruleId: step.stepId,
+            level: 'none',
+            message: {
+                text: `Step "${step.stepId}" passed — ${passRatePct}% pass rate (${step.passes}/${step.totalRuns} iterations passed)`,
+            },
+        };
+    });
+    const sarif = {
+        $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
+        version: '2.1.0',
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'stepproof',
+                        version: '0.1.0',
+                        rules,
+                    },
+                },
+                results,
+            },
+        ],
+    };
+    return JSON.stringify(sarif, null, 2);
+}
+//# sourceMappingURL=sarif-reporter.js.map

package/dist/reporters/sarif-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarif-reporter.js","sourceRoot":"","sources":["../../src/reporters/sarif-reporter.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,WAAW,CAAC,MAAsB;IAChD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,gBAAgB,EAAE;YAChB,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,oBAAoB,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SACrF;KACF,CAAC,CAAC,CAAC;IAEJ,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxC,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEzD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,eAAe,WAAW,wBAAwB,YAAY,MAAM,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,qBAAqB;iBACjJ;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,MAAM;YACb,OAAO,EAAE;gBACP,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,cAAc,WAAW,gBAAgB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,qBAAqB;aACtH;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG;QACZ,OAAO,EACL,gGAAgG;QAClG,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE;YACJ;gBACE,IAAI,EAAE;oBACJ,MAAM,EAAE;wBACN,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,OAAO;wBAChB,KAAK;qBACN;iBACF;gBACD,OAAO;aACR;SACF;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC"}

package/dist/reporters/terminal-reporter.d.ts

@@ -0,0 +1,4 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function printReport(report: ScenarioReport, reportPath?: string): void;
+export declare function printProgress(stepId: string, iteration: number, total: number): void;
+//# sourceMappingURL=terminal-reporter.d.ts.map