stepproof 0.2.0

package/dist/reporters/terminal-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAe,MAAM,kBAAkB,CAAC;AAKpE,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAwB7E;AAsDD,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAKpF"}

package/dist/reporters/terminal-reporter.js

@@ -0,0 +1,73 @@
+import chalk from 'chalk';
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const { version } = require('../../package.json');
+export function printReport(report, reportPath) {
+    const { scenarioName, iterations, steps, allPassed, durationMs } = report;
+    console.log('');
+    console.log(chalk.bold('stepproof') + chalk.dim(` v${version}`));
+    console.log(chalk.dim('─'.repeat(50)));
+    console.log(`${chalk.bold('Scenario:')} ${scenarioName}`);
+    console.log(`${chalk.bold('Iterations:')} ${iterations}`);
+    console.log(`${chalk.bold('Duration:')} ${formatDuration(durationMs)}`);
+    console.log('');
+    for (const step of steps) {
+        printStepSummary(step, iterations);
+    }
+    console.log(chalk.dim('─'.repeat(50)));
+    printSummaryLine(allPassed, steps);
+    if (reportPath) {
+        console.log('');
+        console.log(chalk.dim(`Report written to: ${reportPath}`));
+    }
+    console.log('');
+}
+function printStepSummary(step, iterations) {
+    const { stepId, passes, totalRuns, passRate, minPassRate, belowThreshold, failures } = step;
+    const statusIcon = belowThreshold ? chalk.red('✗') : chalk.green('✓');
+    const rateColor = belowThreshold ? chalk.red : chalk.green;
+    const pct = (passRate * 100).toFixed(1);
+    const threshold = (minPassRate * 100).toFixed(0);
+    console.log(`  ${statusIcon} ${chalk.bold(stepId)}`);
+    console.log(`    ${renderBar(passes, totalRuns)} ${passes}/${totalRuns} iterations`);
+    console.log(`    Pass rate: ${rateColor(`${pct}%`)}  ${chalk.dim(`(threshold: ${threshold}%)`)}`);
+    if (belowThreshold) {
+        console.log(`    ${chalk.red(`✗ BELOW THRESHOLD — ${failures} failure${failures === 1 ? '' : 's'}`)}`);
+    }
+    console.log('');
+}
+function renderBar(passes, total) {
+    const barWidth = 20;
+    const filled = Math.round((passes / total) * barWidth);
+    const empty = barWidth - filled;
+    const bar = chalk.green('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
+    return `[${bar}]`;
+}
+function printSummaryLine(allPassed, steps) {
+    const failing = steps.filter((s) => s.belowThreshold);
+    if (allPassed) {
+        console.log(chalk.bold.green('PASSED') + chalk.dim(' — all steps above threshold'));
+    }
+    else {
+        const stepList = failing.map((s) => chalk.red(s.stepId)).join(', ');
+        console.log(chalk.bold.red('FAILED') +
+            chalk.dim(` — ${failing.length} step${failing.length === 1 ? '' : 's'} below threshold: `) +
+            stepList);
+    }
+}
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    if (ms < 60_000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const mins = Math.floor(ms / 60_000);
+    const secs = ((ms % 60_000) / 1000).toFixed(0);
+    return `${mins}m ${secs}s`;
+}
+export function printProgress(stepId, iteration, total) {
+    process.stdout.write(`\r  ${chalk.dim('Running')} ${stepId} — iteration ${iteration}/${total}...`);
+    if (iteration === total) {
+        process.stdout.write('\r' + ' '.repeat(60) + '\r');
+    }
+}
+//# sourceMappingURL=terminal-reporter.js.map

package/dist/reporters/terminal-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal-reporter.js","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAwB,CAAC;AAEzE,MAAM,UAAU,WAAW,CAAC,MAAsB,EAAE,UAAmB;IACrE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAE1E,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAEnC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAiB,EAAE,UAAkB;IAC7D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAE5F,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEjD,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,IAAI,SAAS,aAAa,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CACT,kBAAkB,SAAS,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,IAAI,CAAC,EAAE,CACrF,CAAC;IAEF,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,uBAAuB,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAa;IAC9C,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,QAAQ,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,OAAO,IAAI,GAAG,GAAG,CAAC;AACpB,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAkB,EAAE,KAAoB;IAChE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAEtD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oBAAoB,CAAC;YAC1F,QAAQ,CACT,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,MAAM;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,SAAiB,EAAE,KAAa;IAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,MAAM,gBAAgB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;IACnG,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "stepproof",
+  "version": "0.2.0",
+  "description": "Regression testing for multi-step AI workflows. Not observability — a CI gate.",
+  "main": "dist/cli.js",
+  "bin": {
+    "stepproof": "dist/cli.js"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "prepublishOnly": "npm run build"
+  },
+  "files": [
+    "dist",
+    "schemas",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "ai",
+    "testing",
+    "regression",
+    "agents",
+    "llm",
+    "cli",
+    "openai",
+    "anthropic",
+    "ci"
+  ],
+  "author": "Bilko",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/StanislavBG/stepproof.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.37.0",
+    "@preflight/license": "^1.0.0",
+    "ajv": "^8.17.1",
+    "chalk": "^5.3.0",
+    "commander": "^12.1.0",
+    "js-yaml": "^4.1.0",
+    "openai": "^4.77.0"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^20.17.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.7.0",
+    "vitest": "^2.1.0"
+  }
+}

package/schemas/scenario.schema.json

@@ -0,0 +1,119 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://stepproof.dev/schemas/scenario.schema.json",
+  "title": "StepProof Scenario",
+  "description": "Schema for a stepproof scenario YAML file",
+  "type": "object",
+  "required": ["name", "steps"],
+  "additionalProperties": false,
+  "properties": {
+    "name": {
+      "type": "string",
+      "description": "Human-readable name for this scenario"
+    },
+    "iterations": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 1000,
+      "default": 10,
+      "description": "Number of times to run the full scenario"
+    },
+    "variables": {
+      "type": "object",
+      "description": "Global variables available in prompt templates as {{variable_name}}",
+      "additionalProperties": { "type": "string" }
+    },
+    "steps": {
+      "type": "array",
+      "minItems": 1,
+      "description": "Ordered list of steps to execute per iteration",
+      "items": {
+        "$ref": "#/definitions/Step"
+      }
+    }
+  },
+  "definitions": {
+    "Step": {
+      "type": "object",
+      "required": ["id", "provider", "model", "prompt"],
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z][a-z0-9_]*$",
+          "description": "Unique step identifier (snake_case). Referenced as {{step_id.output}} in later steps."
+        },
+        "provider": {
+          "type": "string",
+          "enum": ["openai", "anthropic"],
+          "description": "LLM provider to use for this step"
+        },
+        "model": {
+          "type": "string",
+          "description": "Model ID (e.g., gpt-4o, claude-sonnet-4-6)"
+        },
+        "prompt": {
+          "type": "string",
+          "description": "Prompt template. Use {{variable}} for substitution, {{step_id.output}} for prior step outputs."
+        },
+        "system": {
+          "type": "string",
+          "description": "Optional system prompt"
+        },
+        "min_pass_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "default": 0.8,
+          "description": "Minimum fraction of iterations that must pass assertions (0.0–1.0). CI exits 1 if below this."
+        },
+        "assertions": {
+          "type": "array",
+          "default": [],
+          "description": "List of assertions to run on the step output",
+          "items": {
+            "$ref": "#/definitions/Assertion"
+          }
+        }
+      }
+    },
+    "Assertion": {
+      "type": "object",
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["contains", "not_contains", "regex", "json_schema", "llm_judge"],
+          "description": "Assertion type"
+        },
+        "value": {
+          "type": "string",
+          "description": "For contains/not_contains: the substring to check. For regex: the pattern."
+        },
+        "schema": {
+          "type": "string",
+          "description": "For json_schema: path to JSON schema file (relative to scenario file)"
+        },
+        "prompt": {
+          "type": "string",
+          "description": "For llm_judge: the evaluation prompt. The step output is appended automatically."
+        },
+        "pass_on": {
+          "type": "string",
+          "default": "yes",
+          "description": "For llm_judge: the expected response prefix (case-insensitive). Default: \"yes\""
+        },
+        "provider": {
+          "type": "string",
+          "enum": ["openai", "anthropic"],
+          "default": "anthropic",
+          "description": "For llm_judge: which provider to use. Default: anthropic"
+        },
+        "model": {
+          "type": "string",
+          "description": "For llm_judge: override model. Defaults to cheapest available (haiku/gpt-4o-mini)"
+        }
+      }
+    }
+  }
+}