stepproof 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +319 -0
- package/dist/adapters/anthropic.d.ts +8 -0
- package/dist/adapters/anthropic.d.ts.map +1 -0
- package/dist/adapters/anthropic.js +46 -0
- package/dist/adapters/anthropic.js.map +1 -0
- package/dist/adapters/base.d.ts +4 -0
- package/dist/adapters/base.d.ts.map +1 -0
- package/dist/adapters/base.js +2 -0
- package/dist/adapters/base.js.map +1 -0
- package/dist/adapters/index.d.ts +4 -0
- package/dist/adapters/index.d.ts.map +1 -0
- package/dist/adapters/index.js +13 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/openai.d.ts +8 -0
- package/dist/adapters/openai.d.ts.map +1 -0
- package/dist/adapters/openai.js +42 -0
- package/dist/adapters/openai.js.map +1 -0
- package/dist/assertions/engine.d.ts +6 -0
- package/dist/assertions/engine.d.ts.map +1 -0
- package/dist/assertions/engine.js +124 -0
- package/dist/assertions/engine.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +330 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/init.d.ts +2 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +42 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/core/scenario-parser.d.ts +4 -0
- package/dist/core/scenario-parser.d.ts.map +1 -0
- package/dist/core/scenario-parser.js +92 -0
- package/dist/core/scenario-parser.js.map +1 -0
- package/dist/core/scenario-runner.d.ts +11 -0
- package/dist/core/scenario-runner.d.ts.map +1 -0
- package/dist/core/scenario-runner.js +85 -0
- package/dist/core/scenario-runner.js.map +1 -0
- package/dist/core/types.d.ts +71 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +2 -0
- package/dist/core/types.js.map +1 -0
- package/dist/reporters/json-reporter.d.ts +4 -0
- package/dist/reporters/json-reporter.d.ts.map +1 -0
- package/dist/reporters/json-reporter.js +9 -0
- package/dist/reporters/json-reporter.js.map +1 -0
- package/dist/reporters/junit-reporter.d.ts +3 -0
- package/dist/reporters/junit-reporter.d.ts.map +1 -0
- package/dist/reporters/junit-reporter.js +34 -0
- package/dist/reporters/junit-reporter.js.map +1 -0
- package/dist/reporters/sarif-reporter.d.ts +3 -0
- package/dist/reporters/sarif-reporter.d.ts.map +1 -0
- package/dist/reporters/sarif-reporter.js +47 -0
- package/dist/reporters/sarif-reporter.js.map +1 -0
- package/dist/reporters/terminal-reporter.d.ts +4 -0
- package/dist/reporters/terminal-reporter.d.ts.map +1 -0
- package/dist/reporters/terminal-reporter.js +73 -0
- package/dist/reporters/terminal-reporter.js.map +1 -0
- package/dist/telemetry.d.ts +32 -0
- package/dist/telemetry.d.ts.map +1 -0
- package/dist/telemetry.js +93 -0
- package/dist/telemetry.js.map +1 -0
- package/package.json +62 -0
- package/schemas/scenario.schema.json +119 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import * as os from 'node:os';
|
|
5
|
+
import { parseScenario } from './core/scenario-parser.js';
|
|
6
|
+
import { runScenario } from './core/scenario-runner.js';
|
|
7
|
+
import { writeJsonReport } from './reporters/json-reporter.js';
|
|
8
|
+
import { printReport } from './reporters/terminal-reporter.js';
|
|
9
|
+
import { formatSarif } from './reporters/sarif-reporter.js';
|
|
10
|
+
import { formatJunit } from './reporters/junit-reporter.js';
|
|
11
|
+
import * as fs from 'node:fs';
|
|
12
|
+
import { guard, validate } from '@bilkobibitkov/preflight-license';
|
|
13
|
+
import { runInit } from './commands/init.js';
|
|
14
|
+
import { sendTelemetry } from './telemetry.js';
|
|
15
|
+
const CLI_VERSION = '0.2.21';
|
|
16
|
+
/* ── Usage-based monetization (Preflight Suite — shared) ────────────── */
|
|
17
|
+
const TOOL_NAME = 'stepproof';
|
|
18
|
+
const FREE_MONTHLY_LIMIT = 50;
|
|
19
|
+
const FREE_DAILY_LIMIT = 3;
|
|
20
|
+
const UPGRADE_URL = 'https://buy.stripe.com/3cIbJ3fA8am122VcwE8k804';
|
|
21
|
+
// Shared suite directory
|
|
22
|
+
const SUITE_DIR = path.join(os.homedir(), '.preflight-suite');
|
|
23
|
+
const SUITE_USAGE_FILE = path.join(SUITE_DIR, 'usage.json');
|
|
24
|
+
const SUITE_LICENSE_FILE = path.join(SUITE_DIR, 'license.json');
|
|
25
|
+
// Legacy per-tool config dir (kept for backwards-compat reads)
|
|
26
|
+
const CONFIG_DIR = path.join(os.homedir(), '.config', 'stepproof');
|
|
27
|
+
const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
|
|
28
|
+
/** Read license key: env var → shared suite → legacy tool config */
|
|
29
|
+
function getLicenseKey() {
|
|
30
|
+
const envKey = process.env.STEPPROOF_KEY;
|
|
31
|
+
if (envKey?.trim())
|
|
32
|
+
return envKey.trim();
|
|
33
|
+
try {
|
|
34
|
+
if (fs.existsSync(SUITE_LICENSE_FILE)) {
|
|
35
|
+
const parsed = JSON.parse(fs.readFileSync(SUITE_LICENSE_FILE, 'utf8'));
|
|
36
|
+
if (parsed.key?.trim())
|
|
37
|
+
return parsed.key.trim();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch { /* ignore */ }
|
|
41
|
+
try {
|
|
42
|
+
if (fs.existsSync(CONFIG_FILE)) {
|
|
43
|
+
const parsed = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
|
|
44
|
+
if (parsed.key?.trim())
|
|
45
|
+
return parsed.key.trim();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
catch { /* ignore */ }
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
/** Check if user has a valid pro license */
|
|
52
|
+
function isProUser() {
|
|
53
|
+
const key = getLicenseKey();
|
|
54
|
+
if (!key)
|
|
55
|
+
return false;
|
|
56
|
+
const result = validate(key);
|
|
57
|
+
return result.valid && result.tier !== 'free';
|
|
58
|
+
}
|
|
59
|
+
/** Read shared suite usage for the current month, resetting daily counter if needed */
|
|
60
|
+
function readSharedUsage() {
|
|
61
|
+
const currentMonth = new Date().toISOString().slice(0, 7);
|
|
62
|
+
const currentDay = new Date().toISOString().slice(0, 10);
|
|
63
|
+
try {
|
|
64
|
+
if (fs.existsSync(SUITE_USAGE_FILE)) {
|
|
65
|
+
const parsed = JSON.parse(fs.readFileSync(SUITE_USAGE_FILE, 'utf8'));
|
|
66
|
+
if (parsed.month === currentMonth) {
|
|
67
|
+
if (parsed.day !== currentDay) {
|
|
68
|
+
parsed.day = currentDay;
|
|
69
|
+
parsed.day_total = 0;
|
|
70
|
+
}
|
|
71
|
+
if (parsed.day_total === undefined)
|
|
72
|
+
parsed.day_total = 0;
|
|
73
|
+
return parsed;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch { /* corrupted — reset */ }
|
|
78
|
+
return { month: currentMonth, total: 0, day: currentDay, day_total: 0, tools: { stepproof: 0, 'agent-comply': 0, 'agent-gate': 0 } };
|
|
79
|
+
}
|
|
80
|
+
/** Write shared usage to ~/.preflight-suite/usage.json */
|
|
81
|
+
function writeSharedUsage(record) {
|
|
82
|
+
try {
|
|
83
|
+
fs.mkdirSync(SUITE_DIR, { recursive: true });
|
|
84
|
+
fs.writeFileSync(SUITE_USAGE_FILE, JSON.stringify(record, null, 2), 'utf8');
|
|
85
|
+
}
|
|
86
|
+
catch { /* degrade gracefully */ }
|
|
87
|
+
}
|
|
88
|
+
/** Check free limit before a run. Returns true if allowed, false if blocked. */
|
|
89
|
+
function checkUsageLimit() {
|
|
90
|
+
if (isProUser())
|
|
91
|
+
return true;
|
|
92
|
+
const usage = readSharedUsage();
|
|
93
|
+
if (usage.day_total >= FREE_DAILY_LIMIT) {
|
|
94
|
+
process.stderr.write(`\n You've used your ${FREE_DAILY_LIMIT} free checks today.\n` +
|
|
95
|
+
` Upgrade to Team tier ($49/mo) for unlimited daily checks: ${UPGRADE_URL}\n` +
|
|
96
|
+
` — or run \`stepproof activate <your-license-key>\` to activate.\n\n`);
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
/** Increment usage after a successful free run and show a state-based CTA */
|
|
102
|
+
async function trackUsageAfterRun() {
|
|
103
|
+
if (isProUser())
|
|
104
|
+
return;
|
|
105
|
+
const usage = readSharedUsage();
|
|
106
|
+
usage.total += 1;
|
|
107
|
+
usage.day_total += 1;
|
|
108
|
+
usage.tools[TOOL_NAME] = (usage.tools[TOOL_NAME] ?? 0) + 1;
|
|
109
|
+
writeSharedUsage(usage);
|
|
110
|
+
const used = usage.total;
|
|
111
|
+
const remaining = FREE_MONTHLY_LIMIT - used;
|
|
112
|
+
// Show CTA only when >80% of monthly quota used (remaining < 10 of 50)
|
|
113
|
+
if (remaining === 0) {
|
|
114
|
+
process.stderr.write(`\n ${used}/${FREE_MONTHLY_LIMIT} free runs used — next run will be blocked.\n` +
|
|
115
|
+
` Upgrade to Team tier ($49/mo) for unlimited runs: ${UPGRADE_URL}\n` +
|
|
116
|
+
` Already have a key? stepproof activate <key>\n\n`);
|
|
117
|
+
await sendTelemetry({ command: 'cta_shown', success: true, version: CLI_VERSION, outcome: 'cta_shown', exit_code: 0, is_pro: false });
|
|
118
|
+
}
|
|
119
|
+
else if (remaining <= 10) {
|
|
120
|
+
const runWord = remaining === 1 ? 'run' : 'runs';
|
|
121
|
+
process.stderr.write(`\n ${used}/${FREE_MONTHLY_LIMIT} free runs used this month — ${remaining} ${runWord} left.\n` +
|
|
122
|
+
` Upgrade to Team tier ($49/mo) for unlimited runs → ${UPGRADE_URL}\n` +
|
|
123
|
+
` Already have a key? stepproof activate <key>\n\n`);
|
|
124
|
+
await sendTelemetry({ command: 'cta_shown', success: true, version: CLI_VERSION, outcome: 'cta_shown', exit_code: 0, is_pro: false });
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
/* ── CLI ────────────────────────────────────────────────────────────── */
|
|
128
|
+
const program = new Command();
|
|
129
|
+
program
|
|
130
|
+
.name('stepproof')
|
|
131
|
+
.description('Regression testing for multi-step AI workflows. Not observability — a CI gate.')
|
|
132
|
+
.version('0.2.21')
|
|
133
|
+
.addHelpText('after', `
|
|
134
|
+
Examples:
|
|
135
|
+
stepproof init scaffold a starter scenario
|
|
136
|
+
stepproof run ./scenarios/first-test.yaml run one scenario
|
|
137
|
+
stepproof run ./scenarios/ run all scenarios in a directory
|
|
138
|
+
stepproof run test.yaml --format sarif --output results.sarif SARIF output for CI`);
|
|
139
|
+
program
|
|
140
|
+
.command('init [dir]')
|
|
141
|
+
.description('Scaffold a starter scenario in ./scenarios/first-test.yaml')
|
|
142
|
+
.action(async (dir) => {
|
|
143
|
+
runInit(dir);
|
|
144
|
+
await sendTelemetry({ command: 'init', success: true, version: CLI_VERSION, outcome: 'scaffold' });
|
|
145
|
+
});
|
|
146
|
+
program
|
|
147
|
+
.command('activate <key>')
|
|
148
|
+
.description('Store a license key for unlimited runs (applies to all Preflight Suite tools)')
|
|
149
|
+
.action(async (key) => {
|
|
150
|
+
const result = validate(key);
|
|
151
|
+
if (!result.valid) {
|
|
152
|
+
process.stderr.write(`\nInvalid license key: ${result.reason}\n\n`);
|
|
153
|
+
await sendTelemetry({ command: 'activate', success: false, version: CLI_VERSION, outcome: 'invalid_key', exit_code: 1 });
|
|
154
|
+
process.exit(1);
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
fs.mkdirSync(SUITE_DIR, { recursive: true });
|
|
158
|
+
fs.writeFileSync(SUITE_LICENSE_FILE, JSON.stringify({ key }), 'utf8');
|
|
159
|
+
console.log(`\nLicense activated (${result.tier} — ${result.org}). Unlimited runs enabled across all Preflight Suite tools.\n`);
|
|
160
|
+
await sendTelemetry({ command: 'activate', success: true, version: CLI_VERSION, outcome: 'license_activated', exit_code: 0, is_pro: true });
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
process.stderr.write(`\nFailed to save license: ${e.message}\n\n`);
|
|
164
|
+
await sendTelemetry({ command: 'activate', success: false, version: CLI_VERSION, outcome: 'save_failed', exit_code: 1 });
|
|
165
|
+
process.exit(1);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
program
|
|
169
|
+
.command('run [scenario]')
|
|
170
|
+
.description('Run a scenario YAML file and report pass rates per step')
|
|
171
|
+
.option('-n, --iterations <number>', 'Number of iterations to run (overrides scenario file)', parseInt)
|
|
172
|
+
.option('-o, --output <file>', 'Path for output file (JSON by default; SARIF or JUnit when --format is set)', 'stepproof-report.json')
|
|
173
|
+
.option('--no-json', 'Skip JSON report output')
|
|
174
|
+
.option('--quiet', 'Suppress terminal output (use with --output for CI)')
|
|
175
|
+
.option('--format <format>', 'Output format: sarif, junit')
|
|
176
|
+
.option('--report <format>', '(deprecated: use --format)')
|
|
177
|
+
.action(async (scenarioPath, opts) => {
|
|
178
|
+
// --report is deprecated; normalize to --format
|
|
179
|
+
if (opts.report && !opts.format) {
|
|
180
|
+
process.stderr.write('Warning: --report is deprecated, use --format instead\n');
|
|
181
|
+
opts.format = opts.report;
|
|
182
|
+
}
|
|
183
|
+
if (opts.format && opts.format !== 'sarif' && opts.format !== 'junit') {
|
|
184
|
+
console.error(`\nError: --format must be "sarif" or "junit", got "${opts.format}"`);
|
|
185
|
+
process.exit(2);
|
|
186
|
+
}
|
|
187
|
+
// License gate — check before running the scenario (avoid wasted API calls)
|
|
188
|
+
if (opts.format === 'sarif' || opts.format === 'junit') {
|
|
189
|
+
guard('team', { feature: `--format ${opts.format}` });
|
|
190
|
+
}
|
|
191
|
+
// Capture pro status once — used for telemetry throughout this command
|
|
192
|
+
const isPro = isProUser();
|
|
193
|
+
// Usage limit — check before running (avoid wasted API calls)
|
|
194
|
+
const startMs = Date.now();
|
|
195
|
+
if (!checkUsageLimit()) {
|
|
196
|
+
await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'rate_limited', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
197
|
+
process.exit(1);
|
|
198
|
+
}
|
|
199
|
+
// --format implies quiet (suppress terminal output) unless --quiet already set
|
|
200
|
+
const isQuiet = opts.quiet || !!opts.format;
|
|
201
|
+
if (opts.iterations !== undefined) {
|
|
202
|
+
if (!Number.isInteger(opts.iterations) || opts.iterations <= 0) {
|
|
203
|
+
console.error(`\nError: --iterations must be a positive integer, got "${process.argv[process.argv.indexOf('--iterations') + 1] ?? process.argv[process.argv.indexOf('-n') + 1]}"`);
|
|
204
|
+
process.exit(2);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (scenarioPath.includes('\0')) {
|
|
208
|
+
console.error('\nError: Invalid path — null bytes are not allowed');
|
|
209
|
+
process.exit(2);
|
|
210
|
+
}
|
|
211
|
+
if (opts.output && opts.output.includes('\0')) {
|
|
212
|
+
console.error('\nError: Invalid output path — null bytes are not allowed');
|
|
213
|
+
process.exit(2);
|
|
214
|
+
}
|
|
215
|
+
const resolvedPath = path.resolve(process.cwd(), scenarioPath);
|
|
216
|
+
try {
|
|
217
|
+
const stat = fs.statSync(resolvedPath);
|
|
218
|
+
if (stat.isDirectory()) {
|
|
219
|
+
console.error(`\nError: "${scenarioPath}" is a directory.`);
|
|
220
|
+
console.error('Run a specific file: stepproof run ./scenarios/first-test.yaml');
|
|
221
|
+
process.exit(2);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
catch (statErr) {
|
|
225
|
+
if (statErr.code === 'ENOENT') {
|
|
226
|
+
console.error(`\nError: Scenario not found: ${resolvedPath}`);
|
|
227
|
+
console.error("Run 'stepproof init' to scaffold a new scenario, or check the path.");
|
|
228
|
+
process.exit(2);
|
|
229
|
+
}
|
|
230
|
+
// Other stat errors — let parseScenario surface the message
|
|
231
|
+
}
|
|
232
|
+
let scenario;
|
|
233
|
+
try {
|
|
234
|
+
scenario = parseScenario(resolvedPath);
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
console.error(`\nError parsing scenario: ${e.message}`);
|
|
238
|
+
process.exit(2);
|
|
239
|
+
}
|
|
240
|
+
if (!isQuiet) {
|
|
241
|
+
console.log(`\nLoading: ${scenario.name}`);
|
|
242
|
+
if (opts.iterations) {
|
|
243
|
+
console.log(`Overriding iterations: ${scenario.iterations ?? 10} → ${opts.iterations}`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
let currentIteration = 0;
|
|
247
|
+
const totalIterations = opts.iterations ?? scenario.iterations ?? 10;
|
|
248
|
+
let report;
|
|
249
|
+
try {
|
|
250
|
+
report = await runScenario(scenario, resolvedPath, {
|
|
251
|
+
iterations: opts.iterations,
|
|
252
|
+
onIterationComplete: (iteration, total) => {
|
|
253
|
+
currentIteration = iteration;
|
|
254
|
+
if (!isQuiet) {
|
|
255
|
+
process.stdout.write(`\r Completed iteration ${iteration}/${total}...`);
|
|
256
|
+
if (iteration === total) {
|
|
257
|
+
process.stdout.write('\r' + ' '.repeat(50) + '\r');
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
},
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
catch (e) {
|
|
264
|
+
console.error(`\nError running scenario: ${e.message}`);
|
|
265
|
+
await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'error', exit_code: 2, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
266
|
+
process.exit(2);
|
|
267
|
+
}
|
|
268
|
+
// Handle --format sarif / --format junit
|
|
269
|
+
if (opts.format === 'sarif' || opts.format === 'junit') {
|
|
270
|
+
const formatted = opts.format === 'sarif' ? formatSarif(report) : formatJunit(report);
|
|
271
|
+
const hasExplicitOutput = process.argv.includes('--output') || process.argv.includes('-o');
|
|
272
|
+
if (hasExplicitOutput) {
|
|
273
|
+
try {
|
|
274
|
+
fs.writeFileSync(opts.output, formatted, 'utf-8');
|
|
275
|
+
}
|
|
276
|
+
catch (e) {
|
|
277
|
+
console.error(`Warning: Could not write ${opts.format} report: ${e.message}`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
else {
|
|
281
|
+
process.stdout.write(formatted + '\n');
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const reportPath = opts.json ? opts.output : undefined;
|
|
285
|
+
if (!isQuiet) {
|
|
286
|
+
printReport(report, reportPath);
|
|
287
|
+
}
|
|
288
|
+
if (opts.json) {
|
|
289
|
+
try {
|
|
290
|
+
writeJsonReport(report, opts.output);
|
|
291
|
+
}
|
|
292
|
+
catch (e) {
|
|
293
|
+
console.error(`Warning: Could not write JSON report: ${e.message}`);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
// Track usage after successful run completion
|
|
297
|
+
await trackUsageAfterRun();
|
|
298
|
+
// Exit 1 if any step below threshold — this is the CI gate
|
|
299
|
+
if (!report.allPassed) {
|
|
300
|
+
await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'fail', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
301
|
+
process.exit(1);
|
|
302
|
+
}
|
|
303
|
+
await sendTelemetry({ command: 'run', success: true, version: CLI_VERSION, outcome: 'pass', exit_code: 0, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
304
|
+
process.exit(0);
|
|
305
|
+
});
|
|
306
|
+
program.action(() => {
|
|
307
|
+
const extra = process.argv.slice(2).filter(a => !a.startsWith('-'));
|
|
308
|
+
if (extra.length > 0) {
|
|
309
|
+
process.stderr.write(`\nError: Unknown command '${extra[0]}'\nRun 'stepproof --help' for usage.\n\n`);
|
|
310
|
+
process.exit(2);
|
|
311
|
+
}
|
|
312
|
+
program.help(); // exits 0
|
|
313
|
+
});
|
|
314
|
+
// Override commander's default error handler for better UX
|
|
315
|
+
program.exitOverride((err) => {
|
|
316
|
+
if (err.code === 'commander.helpDisplayed' || err.code === 'commander.version') {
|
|
317
|
+
process.exit(err.exitCode ?? 0);
|
|
318
|
+
}
|
|
319
|
+
if (err.code === 'commander.missingArgument') {
|
|
320
|
+
const cmd = err.message.match(/'([^']+)'/)?.[1] ?? 'scenario';
|
|
321
|
+
process.stderr.write(`\nError: Missing required argument <${cmd}>\n`);
|
|
322
|
+
process.stderr.write(`Usage: stepproof run <scenario>\n`);
|
|
323
|
+
process.stderr.write(`\nQuick start:\n stepproof init scaffold a starter scenario\n stepproof run ./scenarios/first-test.yaml\n\n`);
|
|
324
|
+
process.exit(2);
|
|
325
|
+
}
|
|
326
|
+
// All other commander errors: exit with the provided code
|
|
327
|
+
process.exit(err.exitCode ?? 1);
|
|
328
|
+
});
|
|
329
|
+
program.parse(process.argv);
|
|
330
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,WAAW,GAAG,QAAQ,CAAC;AAE7B,2EAA2E;AAE3E,MAAM,SAAS,GAAG,WAAoB,CAAC;AACvC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,WAAW,GAAG,gDAAgD,CAAC;AAErE,yBAAyB;AACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;AAC5D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAEhE,+DAA+D;AAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAczD,oEAAoE;AACpE,SAAS,aAAa;IACpB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACzC,IAAI,MAAM,EAAE,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAqB,CAAC;YAC3F,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAqB,CAAC;YACpF,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;AAChD,CAAC;AAED,uFAAuF;AACvF,SAAS,eAAe;IACtB,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAgB,CAAC;YACpF,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;oBAC9B,MAAM,CAAC,GAAG,GAAG,UAAU,CAAC;oBACxB,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACvB,CAAC;gBACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;oBAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACzD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;IACnC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;AACvI,CAAC;AAED,0DAA0D;AAC1D,SAAS,gBAAgB,CAAC,MAAmB;IAC3C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;AACtC,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe;IACtB,IAAI,SAAS,EAAE;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,IAAI,KAAK,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wBAAwB,gBAAgB,uBAAuB;YAC/D,+DAA+D,WAAW,IAAI;YAC9E,uEAAuE,CACxE,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,kBAAkB;IAC/B,IAAI,SAAS,EAAE;QAAE,OAAO;IACxB,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACrB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;IACzB,MAAM,SAAS,GAAG,kBAAkB,GAAG,IAAI,CAAC;IAE5C,uEAAuE;IACvE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,+CAA+C;YAChF,uDAAuD,WAAW,IAAI;YACtE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;SAAM,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,gCAAgC,SAAS,IAAI,OAAO,UAAU;YAC/F,wDAAwD,WAAW,IAAI;YACvE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,OAAO,EAAE;;;;;qFAK6D,CAAC,CAAC;AAEvF,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;IAC7B,OAAO,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;AACrG,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;IAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;QACpE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,GAAG,+DAA+D,CAAC,CAAC;QAChI,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9I,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,MAAM,CAAC,CAAC;QAC9E,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,YAAgC,EAAE,IAOhD,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,uEAAuE;IACvE,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAE1B,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QACvB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACvK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,0DAA0D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnL,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,YAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAa,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,aAAa,YAAY,mBAAmB,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAK,OAAiC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gCAAgC,YAAY,EAAE,CAAC,CAAC;YAC9D,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,4DAA4D;IAC9D,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,mBAAmB,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACxC,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,kBAAkB,EAAE,CAAC;IAE3B,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU;AAC5B,CAAC,CAAC,CAAC;AAEH,2DAA2D;AAC3D,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,EAAE;IAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,yBAAyB,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;QAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC1D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0HAA0H,CAAC,CAAC;QACjJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,0DAA0D;IAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAuBA,wBAAgB,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAsBhD"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { writeFileSync, existsSync, mkdirSync } from 'fs';
|
|
2
|
+
import { resolve } from 'path';
|
|
3
|
+
const SCENARIO_SCAFFOLD = `# stepproof scenario — edit this, then run: stepproof run ./scenarios/first-test.yaml
|
|
4
|
+
name: First Test
|
|
5
|
+
description: "Test your AI agent's response quality"
|
|
6
|
+
iterations: 3
|
|
7
|
+
|
|
8
|
+
steps:
|
|
9
|
+
- id: step-1
|
|
10
|
+
provider: anthropic # "anthropic" or "openai"
|
|
11
|
+
model: claude-haiku-4-5-20251001 # model to test
|
|
12
|
+
prompt: "Explain what you do in one sentence."
|
|
13
|
+
min_pass_rate: 0.8 # 80% of iterations must pass
|
|
14
|
+
assertions:
|
|
15
|
+
- type: contains
|
|
16
|
+
value: "AI" # replace with text you expect in the response
|
|
17
|
+
|
|
18
|
+
# Assertion types: contains, not_contains, regex, json_schema, llm_judge
|
|
19
|
+
# Set ANTHROPIC_API_KEY or OPENAI_API_KEY before running
|
|
20
|
+
# Full docs: https://github.com/StanislavBG/stepproof
|
|
21
|
+
`;
|
|
22
|
+
export function runInit(outputDir) {
|
|
23
|
+
const dir = resolve(outputDir ?? './scenarios');
|
|
24
|
+
const dest = resolve(dir, 'first-test.yaml');
|
|
25
|
+
if (existsSync(dest)) {
|
|
26
|
+
console.log(`Scenario already exists: ${dest}`);
|
|
27
|
+
console.log('Edit it, then run: stepproof run ./scenarios/first-test.yaml');
|
|
28
|
+
process.exit(0);
|
|
29
|
+
}
|
|
30
|
+
mkdirSync(dir, { recursive: true });
|
|
31
|
+
writeFileSync(dest, SCENARIO_SCAFFOLD, 'utf-8');
|
|
32
|
+
console.log(`\n✔ Created ${dest}`);
|
|
33
|
+
console.log('');
|
|
34
|
+
console.log('Next:');
|
|
35
|
+
console.log(' 1. Edit the scenario — replace the prompt and checks with your actual test');
|
|
36
|
+
console.log(' 2. stepproof run ./scenarios/first-test.yaml');
|
|
37
|
+
console.log('');
|
|
38
|
+
console.log('Add to CI: stepproof run ./scenarios/ --format sarif --output results.sarif');
|
|
39
|
+
console.log('');
|
|
40
|
+
console.log('Ready for a deploy gate? Try: npx agent-gate init');
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=init.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;CAkBzB,CAAC;AAEF,MAAM,UAAU,OAAO,CAAC,SAAkB;IACxC,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAE7C,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,8EAA8E,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Scenario } from './types.js';
|
|
2
|
+
export declare function parseScenario(filePath: string): Scenario;
|
|
3
|
+
export declare function substituteVariables(template: string, variables: Record<string, string>, stepOutputs: Record<string, string>): string;
|
|
4
|
+
//# sourceMappingURL=scenario-parser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenario-parser.d.ts","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAQ,MAAM,YAAY,CAAC;AAEjD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAuCxD;AA0CD,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,MAAM,CAyBR"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as yaml from 'js-yaml';
|
|
3
|
+
export function parseScenario(filePath) {
|
|
4
|
+
let content;
|
|
5
|
+
try {
|
|
6
|
+
content = fs.readFileSync(filePath, 'utf-8');
|
|
7
|
+
}
|
|
8
|
+
catch (e) {
|
|
9
|
+
throw new Error(`Cannot read scenario file: ${filePath}\n Hint: Check the path, or run 'stepproof init' to scaffold a starter scenario.`);
|
|
10
|
+
}
|
|
11
|
+
let raw;
|
|
12
|
+
try {
|
|
13
|
+
raw = yaml.load(content);
|
|
14
|
+
}
|
|
15
|
+
catch (e) {
|
|
16
|
+
throw new Error(`Invalid YAML in scenario file: ${e.message}`);
|
|
17
|
+
}
|
|
18
|
+
if (!raw || typeof raw !== 'object') {
|
|
19
|
+
throw new Error('Scenario file must be a YAML object');
|
|
20
|
+
}
|
|
21
|
+
const scenario = raw;
|
|
22
|
+
if (!scenario.name || typeof scenario.name !== 'string') {
|
|
23
|
+
throw new Error('Scenario must have a "name" field (string)');
|
|
24
|
+
}
|
|
25
|
+
if (!Array.isArray(scenario.steps) || scenario.steps.length === 0) {
|
|
26
|
+
throw new Error('Scenario must have a "steps" array with at least one step');
|
|
27
|
+
}
|
|
28
|
+
const steps = scenario.steps.map((rawStep, i) => validateStep(rawStep, i));
|
|
29
|
+
return {
|
|
30
|
+
name: scenario.name,
|
|
31
|
+
iterations: typeof scenario.iterations === 'number' ? scenario.iterations : 10,
|
|
32
|
+
variables: typeof scenario.variables === 'object' && scenario.variables !== null
|
|
33
|
+
? scenario.variables
|
|
34
|
+
: {},
|
|
35
|
+
steps,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function validateStep(raw, index) {
|
|
39
|
+
if (!raw || typeof raw !== 'object') {
|
|
40
|
+
throw new Error(`Step ${index + 1} must be an object`);
|
|
41
|
+
}
|
|
42
|
+
const step = raw;
|
|
43
|
+
const pos = step.id ? `step "${step.id}"` : `step ${index + 1}`;
|
|
44
|
+
if (!step.id || typeof step.id !== 'string') {
|
|
45
|
+
throw new Error(`Step ${index + 1} must have an "id" field (string)`);
|
|
46
|
+
}
|
|
47
|
+
if (!step.provider || !['openai', 'anthropic'].includes(step.provider)) {
|
|
48
|
+
throw new Error(`${pos}: "provider" must be "openai" or "anthropic"`);
|
|
49
|
+
}
|
|
50
|
+
if (!step.model || typeof step.model !== 'string') {
|
|
51
|
+
throw new Error(`${pos}: "model" field is required (string)`);
|
|
52
|
+
}
|
|
53
|
+
if (!step.prompt || typeof step.prompt !== 'string') {
|
|
54
|
+
throw new Error(`${pos}: "prompt" field is required (string)`);
|
|
55
|
+
}
|
|
56
|
+
const minPassRate = typeof step.min_pass_rate === 'number' ? step.min_pass_rate : 0.8;
|
|
57
|
+
if (minPassRate < 0 || minPassRate > 1) {
|
|
58
|
+
throw new Error(`${pos}: "min_pass_rate" must be between 0.0 and 1.0`);
|
|
59
|
+
}
|
|
60
|
+
return {
|
|
61
|
+
id: step.id,
|
|
62
|
+
provider: step.provider,
|
|
63
|
+
model: step.model,
|
|
64
|
+
prompt: step.prompt,
|
|
65
|
+
system: typeof step.system === 'string' ? step.system : undefined,
|
|
66
|
+
min_pass_rate: minPassRate,
|
|
67
|
+
assertions: Array.isArray(step.assertions) ? step.assertions : [],
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
export function substituteVariables(template, variables, stepOutputs) {
|
|
71
|
+
return template.replace(/\{\{([^}]+)\}\}/g, (match, key) => {
|
|
72
|
+
const trimmed = key.trim();
|
|
73
|
+
// Check for step output reference: {{step_id.output}}
|
|
74
|
+
if (trimmed.includes('.')) {
|
|
75
|
+
const [stepId, field] = trimmed.split('.', 2);
|
|
76
|
+
if (field === 'output' && stepOutputs[stepId] !== undefined) {
|
|
77
|
+
return stepOutputs[stepId];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
// Check global variables
|
|
81
|
+
if (variables[trimmed] !== undefined) {
|
|
82
|
+
return variables[trimmed];
|
|
83
|
+
}
|
|
84
|
+
// Check environment variables
|
|
85
|
+
if (process.env[trimmed] !== undefined) {
|
|
86
|
+
return process.env[trimmed];
|
|
87
|
+
}
|
|
88
|
+
// Leave unresolved — caller decides if this is an error
|
|
89
|
+
return match;
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=scenario-parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenario-parser.js","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAGhC,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,mFAAmF,CAAC,CAAC;IAC7I,CAAC;IAED,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAG,GAA8B,CAAC;IAEhD,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAE5F,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,UAAU,EAAE,OAAO,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC9E,SAAS,EAAE,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI;YAC9E,CAAC,CAAC,QAAQ,CAAC,SAAmC;YAC9C,CAAC,CAAC,EAAE;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC;IAEhE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAkB,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,8CAA8C,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sCAAsC,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uCAAuC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;IACtF,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,+CAA+C,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,QAAQ,EAAE,IAAI,CAAC,QAAkC;QACjD,KAAK,EAAE,IAAI,CAAC,KAAe;QAC3B,MAAM,EAAE,IAAI,CAAC,MAAgB;QAC7B,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACjE,aAAa,EAAE,WAAW;QAC1B,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,SAAiC,EACjC,WAAmC;IAEnC,OAAO,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAE3B,sDAAsD;QACtD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC5D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAC/B,CAAC;QAED,wDAAwD;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Scenario, ScenarioReport } from './types.js';
|
|
2
|
+
export interface RunOptions {
|
|
3
|
+
/** Override iterations from scenario file */
|
|
4
|
+
iterations?: number;
|
|
5
|
+
/** Called after each iteration completes */
|
|
6
|
+
onIterationComplete?: (iteration: number, total: number) => void;
|
|
7
|
+
/** Called after each step within an iteration */
|
|
8
|
+
onStepComplete?: (stepId: string, passed: boolean) => void;
|
|
9
|
+
}
|
|
10
|
+
export declare function runScenario(scenario: Scenario, scenarioFilePath: string, options?: RunOptions): Promise<ScenarioReport>;
|
|
11
|
+
//# sourceMappingURL=scenario-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenario-runner.d.ts","sourceRoot":"","sources":["../../src/core/scenario-runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAA2B,MAAM,YAAY,CAAC;AAEpF,MAAM,WAAW,UAAU;IACzB,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACjE,iDAAiD;IACjD,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;CAC5D;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,MAAM,EACxB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CA8FzB"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import * as path from 'node:path';
|
|
2
|
+
import { getAdapter } from '../adapters/index.js';
|
|
3
|
+
import { runAssertions } from '../assertions/engine.js';
|
|
4
|
+
import { substituteVariables } from './scenario-parser.js';
|
|
5
|
+
export async function runScenario(scenario, scenarioFilePath, options = {}) {
|
|
6
|
+
const iterations = options.iterations ?? scenario.iterations ?? 10;
|
|
7
|
+
const scenarioDir = path.dirname(path.resolve(scenarioFilePath));
|
|
8
|
+
const variables = scenario.variables ?? {};
|
|
9
|
+
const startedAt = new Date().toISOString();
|
|
10
|
+
const startMs = Date.now();
|
|
11
|
+
const allResults = [];
|
|
12
|
+
for (let i = 1; i <= iterations; i++) {
|
|
13
|
+
const stepOutputs = {};
|
|
14
|
+
for (const step of scenario.steps) {
|
|
15
|
+
const resolvedPrompt = substituteVariables(step.prompt, variables, stepOutputs);
|
|
16
|
+
const resolvedSystem = step.system
|
|
17
|
+
? substituteVariables(step.system, variables, stepOutputs)
|
|
18
|
+
: undefined;
|
|
19
|
+
const stepStartMs = Date.now();
|
|
20
|
+
let output = '';
|
|
21
|
+
let error;
|
|
22
|
+
try {
|
|
23
|
+
const adapter = getAdapter(step.provider, step.model);
|
|
24
|
+
output = await adapter.call(resolvedPrompt, resolvedSystem);
|
|
25
|
+
stepOutputs[step.id] = output;
|
|
26
|
+
}
|
|
27
|
+
catch (e) {
|
|
28
|
+
error = e.message;
|
|
29
|
+
stepOutputs[step.id] = '';
|
|
30
|
+
}
|
|
31
|
+
const durationMs = Date.now() - stepStartMs;
|
|
32
|
+
let assertionResults = [];
|
|
33
|
+
let assertionsPassed = false;
|
|
34
|
+
if (!error) {
|
|
35
|
+
const { results, allPassed } = await runAssertions(output, step.assertions, scenarioDir);
|
|
36
|
+
assertionResults = results;
|
|
37
|
+
assertionsPassed = allPassed;
|
|
38
|
+
}
|
|
39
|
+
const stepPassed = !error && assertionsPassed;
|
|
40
|
+
const result = {
|
|
41
|
+
stepId: step.id,
|
|
42
|
+
iteration: i,
|
|
43
|
+
output,
|
|
44
|
+
passed: stepPassed,
|
|
45
|
+
assertionResults,
|
|
46
|
+
error,
|
|
47
|
+
durationMs,
|
|
48
|
+
};
|
|
49
|
+
allResults.push(result);
|
|
50
|
+
options.onStepComplete?.(step.id, stepPassed);
|
|
51
|
+
}
|
|
52
|
+
options.onIterationComplete?.(i, iterations);
|
|
53
|
+
}
|
|
54
|
+
// Aggregate per-step summaries
|
|
55
|
+
const steps = scenario.steps.map((step) => {
|
|
56
|
+
const stepResults = allResults.filter((r) => r.stepId === step.id);
|
|
57
|
+
const passes = stepResults.filter((r) => r.passed).length;
|
|
58
|
+
const failures = stepResults.length - passes;
|
|
59
|
+
const passRate = stepResults.length > 0 ? passes / stepResults.length : 0;
|
|
60
|
+
const minPassRate = step.min_pass_rate ?? 0.8;
|
|
61
|
+
return {
|
|
62
|
+
stepId: step.id,
|
|
63
|
+
totalRuns: stepResults.length,
|
|
64
|
+
passes,
|
|
65
|
+
failures,
|
|
66
|
+
passRate,
|
|
67
|
+
minPassRate,
|
|
68
|
+
belowThreshold: passRate < minPassRate,
|
|
69
|
+
};
|
|
70
|
+
});
|
|
71
|
+
const allPassed = steps.every((s) => !s.belowThreshold);
|
|
72
|
+
const completedAt = new Date().toISOString();
|
|
73
|
+
const durationMs = Date.now() - startMs;
|
|
74
|
+
return {
|
|
75
|
+
scenarioName: scenario.name,
|
|
76
|
+
iterations,
|
|
77
|
+
startedAt,
|
|
78
|
+
completedAt,
|
|
79
|
+
durationMs,
|
|
80
|
+
steps,
|
|
81
|
+
allPassed,
|
|
82
|
+
results: allResults,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=scenario-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenario-runner.js","sourceRoot":"","sources":["../../src/core/scenario-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAkB,EAClB,gBAAwB,EACxB,UAAsB,EAAE;IAExB,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IACnE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;IAE3C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAiB,EAAE,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,WAAW,GAA2B,EAAE,CAAC;QAE/C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;YAChF,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM;gBAChC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC;gBAC1D,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,KAAyB,CAAC;YAE9B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBACtD,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;gBAC5D,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,KAAK,GAAI,CAAW,CAAC,OAAO,CAAC;gBAC7B,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;YAE5C,IAAI,gBAAgB,GAA0D,EAAE,CAAC;YACjF,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACzF,gBAAgB,GAAG,OAAO,CAAC;gBAC3B,gBAAgB,GAAG,SAAS,CAAC;YAC/B,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,gBAAgB,CAAC;YAE9C,MAAM,MAAM,GAAe;gBACzB,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,SAAS,EAAE,CAAC;gBACZ,MAAM;gBACN,MAAM,EAAE,UAAU;gBAClB,gBAAgB;gBAChB,KAAK;gBACL,UAAU;aACX,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,GAAkB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvD,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC1D,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC;QAE9C,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,cAAc,EAAE,QAAQ,GAAG,WAAW;SACvC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAExC,OAAO;QACL,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,UAAU;QACV,SAAS;QACT,WAAW;QACX,UAAU;QACV,KAAK;QACL,SAAS;QACT,OAAO,EAAE,UAAU;KACpB,CAAC;AACJ,CAAC"}
|