stepproof 0.2.22

package/dist/core/types.d.ts

@@ -0,0 +1,71 @@
+export type Provider = 'openai' | 'anthropic';
+export type AssertionType = 'contains' | 'not_contains' | 'regex' | 'json_schema' | 'llm_judge';
+export interface Assertion {
+    type: AssertionType;
+    /** For contains, not_contains, regex */
+    value?: string;
+    /** For json_schema: path to JSON schema file (relative to scenario file) */
+    schema?: string;
+    /** For llm_judge: the evaluation prompt */
+    prompt?: string;
+    /** For llm_judge: the expected response prefix (default: "yes") */
+    pass_on?: string;
+    /** For llm_judge: override provider (default: anthropic) */
+    provider?: Provider;
+    /** For llm_judge: override model (default: claude-haiku or gpt-4o-mini) */
+    model?: string;
+}
+export interface Step {
+    id: string;
+    provider: Provider;
+    model: string;
+    /** Prompt template. Use {{variable}} for variables, {{step_id.output}} for prior step outputs. */
+    prompt: string;
+    /** Optional system prompt */
+    system?: string;
+    /** Minimum pass rate threshold (0.0–1.0). Default: 0.8 */
+    min_pass_rate?: number;
+    assertions: Assertion[];
+}
+export interface Scenario {
+    name: string;
+    /** Number of iterations to run. Default: 10 */
+    iterations?: number;
+    /** Global variables for template substitution */
+    variables?: Record<string, string>;
+    steps: Step[];
+}
+export interface AssertionResult {
+    type: string;
+    passed: boolean;
+    message?: string;
+}
+export interface StepResult {
+    stepId: string;
+    iteration: number;
+    output: string;
+    passed: boolean;
+    assertionResults: AssertionResult[];
+    error?: string;
+    durationMs: number;
+}
+export interface StepSummary {
+    stepId: string;
+    totalRuns: number;
+    passes: number;
+    failures: number;
+    passRate: number;
+    minPassRate: number;
+    belowThreshold: boolean;
+}
+export interface ScenarioReport {
+    scenarioName: string;
+    iterations: number;
+    startedAt: string;
+    completedAt: string;
+    durationMs: number;
+    steps: StepSummary[];
+    allPassed: boolean;
+    results: StepResult[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE9C,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,GAAG,OAAO,GAAG,aAAa,GAAG,WAAW,CAAC;AAEhG,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,SAAS,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB"}

package/dist/core/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":""}

package/dist/reporters/json-reporter.d.ts

@@ -0,0 +1,4 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function writeJsonReport(report: ScenarioReport, outputPath: string): void;
+export declare function formatJsonReport(report: ScenarioReport): string;
+//# sourceMappingURL=json-reporter.d.ts.map

package/dist/reporters/json-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/json-reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAGhF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAE/D"}

package/dist/reporters/json-reporter.js

@@ -0,0 +1,9 @@
+import * as fs from 'node:fs';
+export function writeJsonReport(report, outputPath) {
+    const json = JSON.stringify(report, null, 2);
+    fs.writeFileSync(outputPath, json, 'utf-8');
+}
+export function formatJsonReport(report) {
+    return JSON.stringify(report, null, 2);
+}
+//# sourceMappingURL=json-reporter.js.map

package/dist/reporters/json-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-reporter.js","sourceRoot":"","sources":["../../src/reporters/json-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAG9B,MAAM,UAAU,eAAe,CAAC,MAAsB,EAAE,UAAkB;IACxE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAsB;IACrD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}

package/dist/reporters/junit-reporter.d.ts

@@ -0,0 +1,3 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function formatJunit(report: ScenarioReport): string;
+//# sourceMappingURL=junit-reporter.d.ts.map

package/dist/reporters/junit-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/junit-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAWvD,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CA8B1D"}

package/dist/reporters/junit-reporter.js

@@ -0,0 +1,34 @@
+function escapeXml(str) {
+    return str
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+export function formatJunit(report) {
+    const totalTests = report.steps.length;
+    const totalFailures = report.steps.filter((s) => s.belowThreshold).length;
+    const timeSeconds = (report.durationMs / 1000).toFixed(3);
+    const testCases = report.steps.map((step) => {
+        const passRatePct = (step.passRate * 100).toFixed(1);
+        const thresholdPct = (step.minPassRate * 100).toFixed(0);
+        const openTag = `    <testcase name="${escapeXml(step.stepId)}" classname="stepproof.steps" time="0">`;
+        if (step.belowThreshold) {
+            const failureMessage = `Pass rate ${passRatePct}% below threshold ${thresholdPct}%`;
+            const failureBody = `${escapeXml(step.stepId)}: ${step.passes}/${step.totalRuns} iterations passed`;
+            return `${openTag}\n      <failure message="${escapeXml(failureMessage)}">${escapeXml(failureBody)}</failure>\n    </testcase>`;
+        }
+        return `${openTag}\n    </testcase>`;
+    });
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites name="stepproof" tests="${totalTests}" failures="${totalFailures}" time="${timeSeconds}">`,
+        `  <testsuite name="${escapeXml(report.scenarioName)}" tests="${totalTests}" failures="${totalFailures}" time="${timeSeconds}">`,
+        ...testCases,
+        '  </testsuite>',
+        '</testsuites>',
+    ];
+    return lines.join('\n');
+}
+//# sourceMappingURL=junit-reporter.js.map

package/dist/reporters/junit-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit-reporter.js","sourceRoot":"","sources":["../../src/reporters/junit-reporter.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,GAAG;SACP,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAsB;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;IACvC,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;IAC1E,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEzD,MAAM,OAAO,GAAG,uBAAuB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,yCAAyC,CAAC;QAEvG,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,cAAc,GAAG,aAAa,WAAW,qBAAqB,YAAY,GAAG,CAAC;YACpF,MAAM,WAAW,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,oBAAoB,CAAC;YACpG,OAAO,GAAG,OAAO,6BAA6B,SAAS,CAAC,cAAc,CAAC,KAAK,SAAS,CAAC,WAAW,CAAC,6BAA6B,CAAC;QAClI,CAAC;QAED,OAAO,GAAG,OAAO,mBAAmB,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG;QACZ,wCAAwC;QACxC,uCAAuC,UAAU,eAAe,aAAa,WAAW,WAAW,IAAI;QACvG,sBAAsB,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,UAAU,eAAe,aAAa,WAAW,WAAW,IAAI;QAChI,GAAG,SAAS;QACZ,gBAAgB;QAChB,eAAe;KAChB,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/reporters/sarif-reporter.d.ts

@@ -0,0 +1,3 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function formatSarif(report: ScenarioReport): string;
+//# sourceMappingURL=sarif-reporter.d.ts.map

package/dist/reporters/sarif-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarif-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/sarif-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAmD1D"}

package/dist/reporters/sarif-reporter.js

@@ -0,0 +1,47 @@
+export function formatSarif(report) {
+    const rules = report.steps.map((step) => ({
+        id: step.stepId,
+        name: step.stepId,
+        shortDescription: {
+            text: `Step: ${step.stepId} — min pass rate ${(step.minPassRate * 100).toFixed(0)}%`,
+        },
+    }));
+    const results = report.steps.map((step) => {
+        const passRatePct = (step.passRate * 100).toFixed(1);
+        const thresholdPct = (step.minPassRate * 100).toFixed(0);
+        if (step.belowThreshold) {
+            return {
+                ruleId: step.stepId,
+                level: 'error',
+                message: {
+                    text: `Step "${step.stepId}" pass rate ${passRatePct}% is below threshold ${thresholdPct}% (${step.passes}/${step.totalRuns} iterations passed)`,
+                },
+            };
+        }
+        return {
+            ruleId: step.stepId,
+            level: 'none',
+            message: {
+                text: `Step "${step.stepId}" passed — ${passRatePct}% pass rate (${step.passes}/${step.totalRuns} iterations passed)`,
+            },
+        };
+    });
+    const sarif = {
+        $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
+        version: '2.1.0',
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'stepproof',
+                        version: '0.1.0',
+                        rules,
+                    },
+                },
+                results,
+            },
+        ],
+    };
+    return JSON.stringify(sarif, null, 2);
+}
+//# sourceMappingURL=sarif-reporter.js.map

package/dist/reporters/sarif-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarif-reporter.js","sourceRoot":"","sources":["../../src/reporters/sarif-reporter.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,WAAW,CAAC,MAAsB;IAChD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,gBAAgB,EAAE;YAChB,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,oBAAoB,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SACrF;KACF,CAAC,CAAC,CAAC;IAEJ,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxC,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEzD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,eAAe,WAAW,wBAAwB,YAAY,MAAM,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,qBAAqB;iBACjJ;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,MAAM;YACb,OAAO,EAAE;gBACP,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,cAAc,WAAW,gBAAgB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,qBAAqB;aACtH;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG;QACZ,OAAO,EACL,gGAAgG;QAClG,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE;YACJ;gBACE,IAAI,EAAE;oBACJ,MAAM,EAAE;wBACN,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,OAAO;wBAChB,KAAK;qBACN;iBACF;gBACD,OAAO;aACR;SACF;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC"}

package/dist/reporters/terminal-reporter.d.ts

@@ -0,0 +1,4 @@
+import type { ScenarioReport } from '../core/types.js';
+export declare function printReport(report: ScenarioReport, reportPath?: string): void;
+export declare function printProgress(stepId: string, iteration: number, total: number): void;
+//# sourceMappingURL=terminal-reporter.d.ts.map

package/dist/reporters/terminal-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAe,MAAM,kBAAkB,CAAC;AAKpE,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAwB7E;AAsDD,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAKpF"}

package/dist/reporters/terminal-reporter.js

@@ -0,0 +1,73 @@
+import chalk from 'chalk';
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const { version } = require('../../package.json');
+export function printReport(report, reportPath) {
+    const { scenarioName, iterations, steps, allPassed, durationMs } = report;
+    console.log('');
+    console.log(chalk.bold('stepproof') + chalk.dim(` v${version}`));
+    console.log(chalk.dim('─'.repeat(50)));
+    console.log(`${chalk.bold('Scenario:')} ${scenarioName}`);
+    console.log(`${chalk.bold('Iterations:')} ${iterations}`);
+    console.log(`${chalk.bold('Duration:')} ${formatDuration(durationMs)}`);
+    console.log('');
+    for (const step of steps) {
+        printStepSummary(step, iterations);
+    }
+    console.log(chalk.dim('─'.repeat(50)));
+    printSummaryLine(allPassed, steps);
+    if (reportPath) {
+        console.log('');
+        console.log(chalk.dim(`Report written to: ${reportPath}`));
+    }
+    console.log('');
+}
+function printStepSummary(step, iterations) {
+    const { stepId, passes, totalRuns, passRate, minPassRate, belowThreshold, failures } = step;
+    const statusIcon = belowThreshold ? chalk.red('✗') : chalk.green('✓');
+    const rateColor = belowThreshold ? chalk.red : chalk.green;
+    const pct = (passRate * 100).toFixed(1);
+    const threshold = (minPassRate * 100).toFixed(0);
+    console.log(`  ${statusIcon} ${chalk.bold(stepId)}`);
+    console.log(`    ${renderBar(passes, totalRuns)} ${passes}/${totalRuns} iterations`);
+    console.log(`    Pass rate: ${rateColor(`${pct}%`)}  ${chalk.dim(`(threshold: ${threshold}%)`)}`);
+    if (belowThreshold) {
+        console.log(`    ${chalk.red(`✗ BELOW THRESHOLD — ${failures} failure${failures === 1 ? '' : 's'}`)}`);
+    }
+    console.log('');
+}
+function renderBar(passes, total) {
+    const barWidth = 20;
+    const filled = Math.round((passes / total) * barWidth);
+    const empty = barWidth - filled;
+    const bar = chalk.green('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
+    return `[${bar}]`;
+}
+function printSummaryLine(allPassed, steps) {
+    const failing = steps.filter((s) => s.belowThreshold);
+    if (allPassed) {
+        console.log(chalk.bold.green('PASSED') + chalk.dim(' — all steps above threshold'));
+    }
+    else {
+        const stepList = failing.map((s) => chalk.red(s.stepId)).join(', ');
+        console.log(chalk.bold.red('FAILED') +
+            chalk.dim(` — ${failing.length} step${failing.length === 1 ? '' : 's'} below threshold: `) +
+            stepList);
+    }
+}
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    if (ms < 60_000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const mins = Math.floor(ms / 60_000);
+    const secs = ((ms % 60_000) / 1000).toFixed(0);
+    return `${mins}m ${secs}s`;
+}
+export function printProgress(stepId, iteration, total) {
+    process.stdout.write(`\r  ${chalk.dim('Running')} ${stepId} — iteration ${iteration}/${total}...`);
+    if (iteration === total) {
+        process.stdout.write('\r' + ' '.repeat(60) + '\r');
+    }
+}
+//# sourceMappingURL=terminal-reporter.js.map

package/dist/reporters/terminal-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal-reporter.js","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAwB,CAAC;AAEzE,MAAM,UAAU,WAAW,CAAC,MAAsB,EAAE,UAAmB;IACrE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAE1E,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAEnC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAiB,EAAE,UAAkB;IAC7D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAE5F,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEjD,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,IAAI,SAAS,aAAa,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CACT,kBAAkB,SAAS,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,IAAI,CAAC,EAAE,CACrF,CAAC;IAEF,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,uBAAuB,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAa;IAC9C,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,QAAQ,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,OAAO,IAAI,GAAG,GAAG,CAAC;AACpB,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAkB,EAAE,KAAoB;IAChE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAEtD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oBAAoB,CAAC;YAC1F,QAAQ,CACT,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,MAAM;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,SAAiB,EAAE,KAAa;IAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,MAAM,gBAAgB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;IACnG,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;AACH,CAAC"}

package/dist/telemetry.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Anonymous telemetry for stepproof CLI.
+ *
+ * What we collect: install_id (random UUID, never tied to a person), command name,
+ * node version, OS platform, run outcome (pass/fail/error), CLI version.
+ *
+ * What we do NOT collect: file paths, file contents, usernames, email, IP (hashed server-side).
+ *
+ * Opt-out: set PREFLIGHT_NO_TELEMETRY=1 in your environment.
+ *
+ * Data goes to: https://content-grade.onrender.com/api/telemetry
+ * Stored in: SQLite on Render (cli_telemetry table)
+ */
+export interface TelemetryPayload {
+    command: string;
+    success: boolean;
+    version: string;
+    /** Optional: outcome label — 'pass' | 'fail' | 'error' | 'license_activated' | 'cta_shown' */
+    outcome?: string;
+    /** Process exit code */
+    exit_code?: number;
+    /** Command wall-clock duration in milliseconds */
+    duration_ms?: number;
+    /** True if user has a valid pro license at time of event */
+    is_pro?: boolean;
+}
+/**
+ * Send a telemetry ping. Returns a Promise that resolves within 3 seconds.
+ * Always resolves — never throws or rejects.
+ */
+export declare function sendTelemetry(payload: TelemetryPayload): Promise<void>;
+//# sourceMappingURL=telemetry.d.ts.map

package/dist/telemetry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAmCH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,8FAA8F;IAC9F,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA2CtE"}

package/dist/telemetry.js

@@ -0,0 +1,93 @@
+/**
+ * Anonymous telemetry for stepproof CLI.
+ *
+ * What we collect: install_id (random UUID, never tied to a person), command name,
+ * node version, OS platform, run outcome (pass/fail/error), CLI version.
+ *
+ * What we do NOT collect: file paths, file contents, usernames, email, IP (hashed server-side).
+ *
+ * Opt-out: set PREFLIGHT_NO_TELEMETRY=1 in your environment.
+ *
+ * Data goes to: https://content-grade.onrender.com/api/telemetry
+ * Stored in: SQLite on Render (cli_telemetry table)
+ */
+import * as os from 'node:os';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import * as crypto from 'node:crypto';
+const TELEMETRY_URL = 'https://content-grade.onrender.com/api/telemetry/events';
+const CONFIG_DIR = path.join(os.homedir(), '.config', 'stepproof');
+const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
+/** Get or create a persistent anonymous install ID stored in ~/.config/stepproof/config.json */
+function getOrCreateInstallId() {
+    try {
+        let config = {};
+        if (fs.existsSync(CONFIG_FILE)) {
+            try {
+                config = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
+            }
+            catch {
+                // Corrupted config — start fresh
+            }
+        }
+        if (typeof config.install_id === 'string' && config.install_id.length > 0) {
+            return config.install_id;
+        }
+        const id = crypto.randomUUID();
+        config.install_id = id;
+        fs.mkdirSync(CONFIG_DIR, { recursive: true });
+        fs.writeFileSync(CONFIG_FILE, JSON.stringify(config), 'utf8');
+        return id;
+    }
+    catch {
+        return 'unknown';
+    }
+}
+/**
+ * Send a telemetry ping. Returns a Promise that resolves within 3 seconds.
+ * Always resolves — never throws or rejects.
+ */
+export function sendTelemetry(payload) {
+    if (process.env.PREFLIGHT_NO_TELEMETRY === '1') {
+        return Promise.resolve();
+    }
+    const installId = getOrCreateInstallId();
+    const body = JSON.stringify({
+        installId,
+        package: 'stepproof',
+        event: 'run',
+        command: payload.command,
+        success: payload.success,
+        version: payload.version,
+        platform: process.platform,
+        nodeVersion: process.version,
+        outcome: payload.outcome,
+        exit_code: payload.exit_code,
+        duration_ms: payload.duration_ms,
+        is_pro: payload.is_pro,
+    });
+    return new Promise((resolve) => {
+        const timeout = setTimeout(resolve, 3000);
+        try {
+            const controller = new AbortController();
+            const abortTimer = setTimeout(() => controller.abort(), 2500);
+            fetch(TELEMETRY_URL, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body,
+                signal: controller.signal,
+            })
+                .catch(() => { })
+                .finally(() => {
+                clearTimeout(abortTimer);
+                clearTimeout(timeout);
+                resolve();
+            });
+        }
+        catch {
+            clearTimeout(timeout);
+            resolve();
+        }
+    });
+}
+//# sourceMappingURL=telemetry.js.map

package/dist/telemetry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAEtC,MAAM,aAAa,GAAG,yDAAyD,CAAC;AAChF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAEzD,gGAAgG;AAChG,SAAS,oBAAoB;IAC3B,IAAI,CAAC;QACH,IAAI,MAAM,GAA4B,EAAE,CAAC;QACzC,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAA4B,CAAC;YACvF,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;QACH,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1E,OAAO,MAAM,CAAC,UAAU,CAAC;QAC3B,CAAC;QACD,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAC/B,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC;QACvB,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC9D,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAgBD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAyB;IACrD,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,GAAG,EAAE,CAAC;QAC/C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,MAAM,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1B,SAAS;QACT,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,KAAK;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,WAAW,EAAE,OAAO,CAAC,OAAO;QAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACnC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;YAC9D,KAAK,CAAC,aAAa,EAAE;gBACnB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI;gBACJ,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC;iBACC,KAAK,CAAC,GAAG,EAAE,GAA2C,CAAC,CAAC;iBACxD,OAAO,CAAC,GAAG,EAAE;gBACZ,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACP,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "stepproof",
+  "version": "0.2.22",
+  "description": "Regression testing for multi-step AI workflows. Not observability — a CI gate.",
+  "main": "dist/cli.js",
+  "bin": {
+    "stepproof": "dist/cli.js"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "prepublishOnly": "npm run build"
+  },
+  "files": [
+    "dist",
+    "schemas",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "ai",
+    "testing",
+    "regression",
+    "agents",
+    "llm",
+    "cli",
+    "openai",
+    "anthropic",
+    "ci"
+  ],
+  "author": "Bilko",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/StanislavBG/stepproof.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.37.0",
+    "@bilkobibitkov/preflight-license": "^1.0.0",
+    "ajv": "^8.17.1",
+    "chalk": "^5.3.0",
+    "commander": "^12.1.0",
+    "js-yaml": "^4.1.0",
+    "openai": "^4.77.0"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^20.17.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.7.0",
+    "vitest": "^3.2.4"
+  }
+}

package/schemas/scenario.schema.json

@@ -0,0 +1,119 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://stepproof.dev/schemas/scenario.schema.json",
+  "title": "StepProof Scenario",
+  "description": "Schema for a stepproof scenario YAML file",
+  "type": "object",
+  "required": ["name", "steps"],
+  "additionalProperties": false,
+  "properties": {
+    "name": {
+      "type": "string",
+      "description": "Human-readable name for this scenario"
+    },
+    "iterations": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 1000,
+      "default": 10,
+      "description": "Number of times to run the full scenario"
+    },
+    "variables": {
+      "type": "object",
+      "description": "Global variables available in prompt templates as {{variable_name}}",
+      "additionalProperties": { "type": "string" }
+    },
+    "steps": {
+      "type": "array",
+      "minItems": 1,
+      "description": "Ordered list of steps to execute per iteration",
+      "items": {
+        "$ref": "#/definitions/Step"
+      }
+    }
+  },
+  "definitions": {
+    "Step": {
+      "type": "object",
+      "required": ["id", "provider", "model", "prompt"],
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z][a-z0-9_]*$",
+          "description": "Unique step identifier (snake_case). Referenced as {{step_id.output}} in later steps."
+        },
+        "provider": {
+          "type": "string",
+          "enum": ["openai", "anthropic"],
+          "description": "LLM provider to use for this step"
+        },
+        "model": {
+          "type": "string",
+          "description": "Model ID (e.g., gpt-4o, claude-sonnet-4-6)"
+        },
+        "prompt": {
+          "type": "string",
+          "description": "Prompt template. Use {{variable}} for substitution, {{step_id.output}} for prior step outputs."
+        },
+        "system": {
+          "type": "string",
+          "description": "Optional system prompt"
+        },
+        "min_pass_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "default": 0.8,
+          "description": "Minimum fraction of iterations that must pass assertions (0.0–1.0). CI exits 1 if below this."
+        },
+        "assertions": {
+          "type": "array",
+          "default": [],
+          "description": "List of assertions to run on the step output",
+          "items": {
+            "$ref": "#/definitions/Assertion"
+          }
+        }
+      }
+    },
+    "Assertion": {
+      "type": "object",
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["contains", "not_contains", "regex", "json_schema", "llm_judge"],
+          "description": "Assertion type"
+        },
+        "value": {
+          "type": "string",
+          "description": "For contains/not_contains: the substring to check. For regex: the pattern."
+        },
+        "schema": {
+          "type": "string",
+          "description": "For json_schema: path to JSON schema file (relative to scenario file)"
+        },
+        "prompt": {
+          "type": "string",
+          "description": "For llm_judge: the evaluation prompt. The step output is appended automatically."
+        },
+        "pass_on": {
+          "type": "string",
+          "default": "yes",
+          "description": "For llm_judge: the expected response prefix (case-insensitive). Default: \"yes\""
+        },
+        "provider": {
+          "type": "string",
+          "enum": ["openai", "anthropic"],
+          "default": "anthropic",
+          "description": "For llm_judge: which provider to use. Default: anthropic"
+        },
+        "model": {
+          "type": "string",
+          "description": "For llm_judge: override model. Defaults to cheapest available (haiku/gpt-4o-mini)"
+        }
+      }
+    }
+  }
+}