stepproof 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +319 -0
  3. package/dist/adapters/anthropic.d.ts +8 -0
  4. package/dist/adapters/anthropic.d.ts.map +1 -0
  5. package/dist/adapters/anthropic.js +46 -0
  6. package/dist/adapters/anthropic.js.map +1 -0
  7. package/dist/adapters/base.d.ts +4 -0
  8. package/dist/adapters/base.d.ts.map +1 -0
  9. package/dist/adapters/base.js +2 -0
  10. package/dist/adapters/base.js.map +1 -0
  11. package/dist/adapters/index.d.ts +4 -0
  12. package/dist/adapters/index.d.ts.map +1 -0
  13. package/dist/adapters/index.js +13 -0
  14. package/dist/adapters/index.js.map +1 -0
  15. package/dist/adapters/openai.d.ts +8 -0
  16. package/dist/adapters/openai.d.ts.map +1 -0
  17. package/dist/adapters/openai.js +42 -0
  18. package/dist/adapters/openai.js.map +1 -0
  19. package/dist/assertions/engine.d.ts +6 -0
  20. package/dist/assertions/engine.d.ts.map +1 -0
  21. package/dist/assertions/engine.js +124 -0
  22. package/dist/assertions/engine.js.map +1 -0
  23. package/dist/cli.d.ts +3 -0
  24. package/dist/cli.d.ts.map +1 -0
  25. package/dist/cli.js +330 -0
  26. package/dist/cli.js.map +1 -0
  27. package/dist/commands/init.d.ts +2 -0
  28. package/dist/commands/init.d.ts.map +1 -0
  29. package/dist/commands/init.js +42 -0
  30. package/dist/commands/init.js.map +1 -0
  31. package/dist/core/scenario-parser.d.ts +4 -0
  32. package/dist/core/scenario-parser.d.ts.map +1 -0
  33. package/dist/core/scenario-parser.js +92 -0
  34. package/dist/core/scenario-parser.js.map +1 -0
  35. package/dist/core/scenario-runner.d.ts +11 -0
  36. package/dist/core/scenario-runner.d.ts.map +1 -0
  37. package/dist/core/scenario-runner.js +85 -0
  38. package/dist/core/scenario-runner.js.map +1 -0
  39. package/dist/core/types.d.ts +71 -0
  40. package/dist/core/types.d.ts.map +1 -0
  41. package/dist/core/types.js +2 -0
  42. package/dist/core/types.js.map +1 -0
  43. package/dist/reporters/json-reporter.d.ts +4 -0
  44. package/dist/reporters/json-reporter.d.ts.map +1 -0
  45. package/dist/reporters/json-reporter.js +9 -0
  46. package/dist/reporters/json-reporter.js.map +1 -0
  47. package/dist/reporters/junit-reporter.d.ts +3 -0
  48. package/dist/reporters/junit-reporter.d.ts.map +1 -0
  49. package/dist/reporters/junit-reporter.js +34 -0
  50. package/dist/reporters/junit-reporter.js.map +1 -0
  51. package/dist/reporters/sarif-reporter.d.ts +3 -0
  52. package/dist/reporters/sarif-reporter.d.ts.map +1 -0
  53. package/dist/reporters/sarif-reporter.js +47 -0
  54. package/dist/reporters/sarif-reporter.js.map +1 -0
  55. package/dist/reporters/terminal-reporter.d.ts +4 -0
  56. package/dist/reporters/terminal-reporter.d.ts.map +1 -0
  57. package/dist/reporters/terminal-reporter.js +73 -0
  58. package/dist/reporters/terminal-reporter.js.map +1 -0
  59. package/dist/telemetry.d.ts +32 -0
  60. package/dist/telemetry.d.ts.map +1 -0
  61. package/dist/telemetry.js +93 -0
  62. package/dist/telemetry.js.map +1 -0
  63. package/package.json +62 -0
  64. package/schemas/scenario.schema.json +119 -0
package/dist/cli.js ADDED
@@ -0,0 +1,330 @@
1
+ #!/usr/bin/env node
2
+ import { Command } from 'commander';
3
+ import * as path from 'node:path';
4
+ import * as os from 'node:os';
5
+ import { parseScenario } from './core/scenario-parser.js';
6
+ import { runScenario } from './core/scenario-runner.js';
7
+ import { writeJsonReport } from './reporters/json-reporter.js';
8
+ import { printReport } from './reporters/terminal-reporter.js';
9
+ import { formatSarif } from './reporters/sarif-reporter.js';
10
+ import { formatJunit } from './reporters/junit-reporter.js';
11
+ import * as fs from 'node:fs';
12
+ import { guard, validate } from '@bilkobibitkov/preflight-license';
13
+ import { runInit } from './commands/init.js';
14
+ import { sendTelemetry } from './telemetry.js';
15
+ const CLI_VERSION = '0.2.21';
16
+ /* ── Usage-based monetization (Preflight Suite — shared) ────────────── */
17
+ const TOOL_NAME = 'stepproof';
18
+ const FREE_MONTHLY_LIMIT = 50;
19
+ const FREE_DAILY_LIMIT = 3;
20
+ const UPGRADE_URL = 'https://buy.stripe.com/3cIbJ3fA8am122VcwE8k804';
21
+ // Shared suite directory
22
+ const SUITE_DIR = path.join(os.homedir(), '.preflight-suite');
23
+ const SUITE_USAGE_FILE = path.join(SUITE_DIR, 'usage.json');
24
+ const SUITE_LICENSE_FILE = path.join(SUITE_DIR, 'license.json');
25
+ // Legacy per-tool config dir (kept for backwards-compat reads)
26
+ const CONFIG_DIR = path.join(os.homedir(), '.config', 'stepproof');
27
+ const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
28
+ /** Read license key: env var → shared suite → legacy tool config */
29
+ function getLicenseKey() {
30
+ const envKey = process.env.STEPPROOF_KEY;
31
+ if (envKey?.trim())
32
+ return envKey.trim();
33
+ try {
34
+ if (fs.existsSync(SUITE_LICENSE_FILE)) {
35
+ const parsed = JSON.parse(fs.readFileSync(SUITE_LICENSE_FILE, 'utf8'));
36
+ if (parsed.key?.trim())
37
+ return parsed.key.trim();
38
+ }
39
+ }
40
+ catch { /* ignore */ }
41
+ try {
42
+ if (fs.existsSync(CONFIG_FILE)) {
43
+ const parsed = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
44
+ if (parsed.key?.trim())
45
+ return parsed.key.trim();
46
+ }
47
+ }
48
+ catch { /* ignore */ }
49
+ return undefined;
50
+ }
51
+ /** Check if user has a valid pro license */
52
+ function isProUser() {
53
+ const key = getLicenseKey();
54
+ if (!key)
55
+ return false;
56
+ const result = validate(key);
57
+ return result.valid && result.tier !== 'free';
58
+ }
59
+ /** Read shared suite usage for the current month, resetting daily counter if needed */
60
+ function readSharedUsage() {
61
+ const currentMonth = new Date().toISOString().slice(0, 7);
62
+ const currentDay = new Date().toISOString().slice(0, 10);
63
+ try {
64
+ if (fs.existsSync(SUITE_USAGE_FILE)) {
65
+ const parsed = JSON.parse(fs.readFileSync(SUITE_USAGE_FILE, 'utf8'));
66
+ if (parsed.month === currentMonth) {
67
+ if (parsed.day !== currentDay) {
68
+ parsed.day = currentDay;
69
+ parsed.day_total = 0;
70
+ }
71
+ if (parsed.day_total === undefined)
72
+ parsed.day_total = 0;
73
+ return parsed;
74
+ }
75
+ }
76
+ }
77
+ catch { /* corrupted — reset */ }
78
+ return { month: currentMonth, total: 0, day: currentDay, day_total: 0, tools: { stepproof: 0, 'agent-comply': 0, 'agent-gate': 0 } };
79
+ }
80
+ /** Write shared usage to ~/.preflight-suite/usage.json */
81
+ function writeSharedUsage(record) {
82
+ try {
83
+ fs.mkdirSync(SUITE_DIR, { recursive: true });
84
+ fs.writeFileSync(SUITE_USAGE_FILE, JSON.stringify(record, null, 2), 'utf8');
85
+ }
86
+ catch { /* degrade gracefully */ }
87
+ }
88
+ /** Check free limit before a run. Returns true if allowed, false if blocked. */
89
+ function checkUsageLimit() {
90
+ if (isProUser())
91
+ return true;
92
+ const usage = readSharedUsage();
93
+ if (usage.day_total >= FREE_DAILY_LIMIT) {
94
+ process.stderr.write(`\n You've used your ${FREE_DAILY_LIMIT} free checks today.\n` +
95
+ ` Upgrade to Team tier ($49/mo) for unlimited daily checks: ${UPGRADE_URL}\n` +
96
+ ` — or run \`stepproof activate <your-license-key>\` to activate.\n\n`);
97
+ return false;
98
+ }
99
+ return true;
100
+ }
101
+ /** Increment usage after a successful free run and show a state-based CTA */
102
+ async function trackUsageAfterRun() {
103
+ if (isProUser())
104
+ return;
105
+ const usage = readSharedUsage();
106
+ usage.total += 1;
107
+ usage.day_total += 1;
108
+ usage.tools[TOOL_NAME] = (usage.tools[TOOL_NAME] ?? 0) + 1;
109
+ writeSharedUsage(usage);
110
+ const used = usage.total;
111
+ const remaining = FREE_MONTHLY_LIMIT - used;
112
+ // Show CTA only when >80% of monthly quota used (remaining < 10 of 50)
113
+ if (remaining === 0) {
114
+ process.stderr.write(`\n ${used}/${FREE_MONTHLY_LIMIT} free runs used — next run will be blocked.\n` +
115
+ ` Upgrade to Team tier ($49/mo) for unlimited runs: ${UPGRADE_URL}\n` +
116
+ ` Already have a key? stepproof activate <key>\n\n`);
117
+ await sendTelemetry({ command: 'cta_shown', success: true, version: CLI_VERSION, outcome: 'cta_shown', exit_code: 0, is_pro: false });
118
+ }
119
+ else if (remaining <= 10) {
120
+ const runWord = remaining === 1 ? 'run' : 'runs';
121
+ process.stderr.write(`\n ${used}/${FREE_MONTHLY_LIMIT} free runs used this month — ${remaining} ${runWord} left.\n` +
122
+ ` Upgrade to Team tier ($49/mo) for unlimited runs → ${UPGRADE_URL}\n` +
123
+ ` Already have a key? stepproof activate <key>\n\n`);
124
+ await sendTelemetry({ command: 'cta_shown', success: true, version: CLI_VERSION, outcome: 'cta_shown', exit_code: 0, is_pro: false });
125
+ }
126
+ }
127
+ /* ── CLI ────────────────────────────────────────────────────────────── */
128
+ const program = new Command();
129
+ program
130
+ .name('stepproof')
131
+ .description('Regression testing for multi-step AI workflows. Not observability — a CI gate.')
132
+ .version('0.2.21')
133
+ .addHelpText('after', `
134
+ Examples:
135
+ stepproof init scaffold a starter scenario
136
+ stepproof run ./scenarios/first-test.yaml run one scenario
137
+ stepproof run ./scenarios/ run all scenarios in a directory
138
+ stepproof run test.yaml --format sarif --output results.sarif SARIF output for CI`);
139
+ program
140
+ .command('init [dir]')
141
+ .description('Scaffold a starter scenario in ./scenarios/first-test.yaml')
142
+ .action(async (dir) => {
143
+ runInit(dir);
144
+ await sendTelemetry({ command: 'init', success: true, version: CLI_VERSION, outcome: 'scaffold' });
145
+ });
146
+ program
147
+ .command('activate <key>')
148
+ .description('Store a license key for unlimited runs (applies to all Preflight Suite tools)')
149
+ .action(async (key) => {
150
+ const result = validate(key);
151
+ if (!result.valid) {
152
+ process.stderr.write(`\nInvalid license key: ${result.reason}\n\n`);
153
+ await sendTelemetry({ command: 'activate', success: false, version: CLI_VERSION, outcome: 'invalid_key', exit_code: 1 });
154
+ process.exit(1);
155
+ }
156
+ try {
157
+ fs.mkdirSync(SUITE_DIR, { recursive: true });
158
+ fs.writeFileSync(SUITE_LICENSE_FILE, JSON.stringify({ key }), 'utf8');
159
+ console.log(`\nLicense activated (${result.tier} — ${result.org}). Unlimited runs enabled across all Preflight Suite tools.\n`);
160
+ await sendTelemetry({ command: 'activate', success: true, version: CLI_VERSION, outcome: 'license_activated', exit_code: 0, is_pro: true });
161
+ }
162
+ catch (e) {
163
+ process.stderr.write(`\nFailed to save license: ${e.message}\n\n`);
164
+ await sendTelemetry({ command: 'activate', success: false, version: CLI_VERSION, outcome: 'save_failed', exit_code: 1 });
165
+ process.exit(1);
166
+ }
167
+ });
168
+ program
169
+ .command('run [scenario]')
170
+ .description('Run a scenario YAML file and report pass rates per step')
171
+ .option('-n, --iterations <number>', 'Number of iterations to run (overrides scenario file)', parseInt)
172
+ .option('-o, --output <file>', 'Path for output file (JSON by default; SARIF or JUnit when --format is set)', 'stepproof-report.json')
173
+ .option('--no-json', 'Skip JSON report output')
174
+ .option('--quiet', 'Suppress terminal output (use with --output for CI)')
175
+ .option('--format <format>', 'Output format: sarif, junit')
176
+ .option('--report <format>', '(deprecated: use --format)')
177
+ .action(async (scenarioPath, opts) => {
178
+ // --report is deprecated; normalize to --format
179
+ if (opts.report && !opts.format) {
180
+ process.stderr.write('Warning: --report is deprecated, use --format instead\n');
181
+ opts.format = opts.report;
182
+ }
183
+ if (opts.format && opts.format !== 'sarif' && opts.format !== 'junit') {
184
+ console.error(`\nError: --format must be "sarif" or "junit", got "${opts.format}"`);
185
+ process.exit(2);
186
+ }
187
+ // License gate — check before running the scenario (avoid wasted API calls)
188
+ if (opts.format === 'sarif' || opts.format === 'junit') {
189
+ guard('team', { feature: `--format ${opts.format}` });
190
+ }
191
+ // Capture pro status once — used for telemetry throughout this command
192
+ const isPro = isProUser();
193
+ // Usage limit — check before running (avoid wasted API calls)
194
+ const startMs = Date.now();
195
+ if (!checkUsageLimit()) {
196
+ await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'rate_limited', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
197
+ process.exit(1);
198
+ }
199
+ // --format implies quiet (suppress terminal output) unless --quiet already set
200
+ const isQuiet = opts.quiet || !!opts.format;
201
+ if (opts.iterations !== undefined) {
202
+ if (!Number.isInteger(opts.iterations) || opts.iterations <= 0) {
203
+ console.error(`\nError: --iterations must be a positive integer, got "${process.argv[process.argv.indexOf('--iterations') + 1] ?? process.argv[process.argv.indexOf('-n') + 1]}"`);
204
+ process.exit(2);
205
+ }
206
+ }
207
+ if (scenarioPath.includes('\0')) {
208
+ console.error('\nError: Invalid path — null bytes are not allowed');
209
+ process.exit(2);
210
+ }
211
+ if (opts.output && opts.output.includes('\0')) {
212
+ console.error('\nError: Invalid output path — null bytes are not allowed');
213
+ process.exit(2);
214
+ }
215
+ const resolvedPath = path.resolve(process.cwd(), scenarioPath);
216
+ try {
217
+ const stat = fs.statSync(resolvedPath);
218
+ if (stat.isDirectory()) {
219
+ console.error(`\nError: "${scenarioPath}" is a directory.`);
220
+ console.error('Run a specific file: stepproof run ./scenarios/first-test.yaml');
221
+ process.exit(2);
222
+ }
223
+ }
224
+ catch (statErr) {
225
+ if (statErr.code === 'ENOENT') {
226
+ console.error(`\nError: Scenario not found: ${resolvedPath}`);
227
+ console.error("Run 'stepproof init' to scaffold a new scenario, or check the path.");
228
+ process.exit(2);
229
+ }
230
+ // Other stat errors — let parseScenario surface the message
231
+ }
232
+ let scenario;
233
+ try {
234
+ scenario = parseScenario(resolvedPath);
235
+ }
236
+ catch (e) {
237
+ console.error(`\nError parsing scenario: ${e.message}`);
238
+ process.exit(2);
239
+ }
240
+ if (!isQuiet) {
241
+ console.log(`\nLoading: ${scenario.name}`);
242
+ if (opts.iterations) {
243
+ console.log(`Overriding iterations: ${scenario.iterations ?? 10} → ${opts.iterations}`);
244
+ }
245
+ }
246
+ let currentIteration = 0;
247
+ const totalIterations = opts.iterations ?? scenario.iterations ?? 10;
248
+ let report;
249
+ try {
250
+ report = await runScenario(scenario, resolvedPath, {
251
+ iterations: opts.iterations,
252
+ onIterationComplete: (iteration, total) => {
253
+ currentIteration = iteration;
254
+ if (!isQuiet) {
255
+ process.stdout.write(`\r Completed iteration ${iteration}/${total}...`);
256
+ if (iteration === total) {
257
+ process.stdout.write('\r' + ' '.repeat(50) + '\r');
258
+ }
259
+ }
260
+ },
261
+ });
262
+ }
263
+ catch (e) {
264
+ console.error(`\nError running scenario: ${e.message}`);
265
+ await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'error', exit_code: 2, duration_ms: Date.now() - startMs, is_pro: isPro });
266
+ process.exit(2);
267
+ }
268
+ // Handle --format sarif / --format junit
269
+ if (opts.format === 'sarif' || opts.format === 'junit') {
270
+ const formatted = opts.format === 'sarif' ? formatSarif(report) : formatJunit(report);
271
+ const hasExplicitOutput = process.argv.includes('--output') || process.argv.includes('-o');
272
+ if (hasExplicitOutput) {
273
+ try {
274
+ fs.writeFileSync(opts.output, formatted, 'utf-8');
275
+ }
276
+ catch (e) {
277
+ console.error(`Warning: Could not write ${opts.format} report: ${e.message}`);
278
+ }
279
+ }
280
+ else {
281
+ process.stdout.write(formatted + '\n');
282
+ }
283
+ }
284
+ const reportPath = opts.json ? opts.output : undefined;
285
+ if (!isQuiet) {
286
+ printReport(report, reportPath);
287
+ }
288
+ if (opts.json) {
289
+ try {
290
+ writeJsonReport(report, opts.output);
291
+ }
292
+ catch (e) {
293
+ console.error(`Warning: Could not write JSON report: ${e.message}`);
294
+ }
295
+ }
296
+ // Track usage after successful run completion
297
+ await trackUsageAfterRun();
298
+ // Exit 1 if any step below threshold — this is the CI gate
299
+ if (!report.allPassed) {
300
+ await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'fail', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
301
+ process.exit(1);
302
+ }
303
+ await sendTelemetry({ command: 'run', success: true, version: CLI_VERSION, outcome: 'pass', exit_code: 0, duration_ms: Date.now() - startMs, is_pro: isPro });
304
+ process.exit(0);
305
+ });
306
+ program.action(() => {
307
+ const extra = process.argv.slice(2).filter(a => !a.startsWith('-'));
308
+ if (extra.length > 0) {
309
+ process.stderr.write(`\nError: Unknown command '${extra[0]}'\nRun 'stepproof --help' for usage.\n\n`);
310
+ process.exit(2);
311
+ }
312
+ program.help(); // exits 0
313
+ });
314
+ // Override commander's default error handler for better UX
315
+ program.exitOverride((err) => {
316
+ if (err.code === 'commander.helpDisplayed' || err.code === 'commander.version') {
317
+ process.exit(err.exitCode ?? 0);
318
+ }
319
+ if (err.code === 'commander.missingArgument') {
320
+ const cmd = err.message.match(/'([^']+)'/)?.[1] ?? 'scenario';
321
+ process.stderr.write(`\nError: Missing required argument <${cmd}>\n`);
322
+ process.stderr.write(`Usage: stepproof run <scenario>\n`);
323
+ process.stderr.write(`\nQuick start:\n stepproof init scaffold a starter scenario\n stepproof run ./scenarios/first-test.yaml\n\n`);
324
+ process.exit(2);
325
+ }
326
+ // All other commander errors: exit with the provided code
327
+ process.exit(err.exitCode ?? 1);
328
+ });
329
+ program.parse(process.argv);
330
+ //# sourceMappingURL=cli.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,WAAW,GAAG,QAAQ,CAAC;AAE7B,2EAA2E;AAE3E,MAAM,SAAS,GAAG,WAAoB,CAAC;AACvC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,WAAW,GAAG,gDAAgD,CAAC;AAErE,yBAAyB;AACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;AAC5D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAEhE,+DAA+D;AAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAczD,oEAAoE;AACpE,SAAS,aAAa;IACpB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACzC,IAAI,MAAM,EAAE,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAqB,CAAC;YAC3F,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAqB,CAAC;YACpF,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;AAChD,CAAC;AAED,uFAAuF;AACvF,SAAS,eAAe;IACtB,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAgB,CAAC;YACpF,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;oBAC9B,MAAM,CAAC,GAAG,GAAG,UAAU,CAAC;oBACxB,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACvB,CAAC;gBACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;oBAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACzD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;IACnC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;AACvI,CAAC;AAED,0DAA0D;AAC1D,SAAS,gBAAgB,CAAC,MAAmB;IAC3C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;AACtC,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe;IACtB,IAAI,SAAS,EAAE;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,IAAI,KAAK,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wBAAwB,gBAAgB,uBAAuB;YAC/D,+DAA+D,WAAW,IAAI;YAC9E,uEAAuE,CACxE,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,kBAAkB;IAC/B,IAAI,SAAS,EAAE;QAAE,OAAO;IACxB,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACrB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;IACzB,MAAM,SAAS,GAAG,kBAAkB,GAAG,IAAI,CAAC;IAE5C,uEAAuE;IACvE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,+CAA+C;YAChF,uDAAuD,WAAW,IAAI;YACtE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;SAAM,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,gCAAgC,SAAS,IAAI,OAAO,UAAU;YAC/F,wDAAwD,WAAW,IAAI;YACvE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,OAAO,EAAE;;;;;qFAK6D,CAAC,CAAC;AAEvF,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;IAC7B,OAAO,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;AACrG,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;IAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;QACpE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,GAAG,+DAA+D,CAAC,CAAC;QAChI,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9I,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,MAAM,CAAC,CAAC;QAC9E,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,YAAgC,EAAE,IAOhD,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,uEAAuE;IACvE,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAE1B,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QACvB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACvK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,0DAA0D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnL,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,YAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAa,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,aAAa,YAAY,mBAAmB,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAK,OAAiC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gCAAgC,YAAY,EAAE,CAAC,CAAC;YAC9D,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,4DAA4D;IAC9D,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,mBAAmB,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACxC,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,kBAAkB,EAAE,CAAC;IAE3B,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU;AAC5B,CAAC,CAAC,CAAC;AAEH,2DAA2D;AAC3D,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,EAAE;IAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,yBAAyB,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;QAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC1D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0HAA0H,CAAC,CAAC;QACjJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,0DAA0D;IAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function runInit(outputDir?: string): void;
2
+ //# sourceMappingURL=init.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAuBA,wBAAgB,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAsBhD"}
@@ -0,0 +1,42 @@
1
+ import { writeFileSync, existsSync, mkdirSync } from 'fs';
2
+ import { resolve } from 'path';
3
+ const SCENARIO_SCAFFOLD = `# stepproof scenario — edit this, then run: stepproof run ./scenarios/first-test.yaml
4
+ name: First Test
5
+ description: "Test your AI agent's response quality"
6
+ iterations: 3
7
+
8
+ steps:
9
+ - id: step-1
10
+ provider: anthropic # "anthropic" or "openai"
11
+ model: claude-haiku-4-5-20251001 # model to test
12
+ prompt: "Explain what you do in one sentence."
13
+ min_pass_rate: 0.8 # 80% of iterations must pass
14
+ assertions:
15
+ - type: contains
16
+ value: "AI" # replace with text you expect in the response
17
+
18
+ # Assertion types: contains, not_contains, regex, json_schema, llm_judge
19
+ # Set ANTHROPIC_API_KEY or OPENAI_API_KEY before running
20
+ # Full docs: https://github.com/StanislavBG/stepproof
21
+ `;
22
+ export function runInit(outputDir) {
23
+ const dir = resolve(outputDir ?? './scenarios');
24
+ const dest = resolve(dir, 'first-test.yaml');
25
+ if (existsSync(dest)) {
26
+ console.log(`Scenario already exists: ${dest}`);
27
+ console.log('Edit it, then run: stepproof run ./scenarios/first-test.yaml');
28
+ process.exit(0);
29
+ }
30
+ mkdirSync(dir, { recursive: true });
31
+ writeFileSync(dest, SCENARIO_SCAFFOLD, 'utf-8');
32
+ console.log(`\n✔ Created ${dest}`);
33
+ console.log('');
34
+ console.log('Next:');
35
+ console.log(' 1. Edit the scenario — replace the prompt and checks with your actual test');
36
+ console.log(' 2. stepproof run ./scenarios/first-test.yaml');
37
+ console.log('');
38
+ console.log('Add to CI: stepproof run ./scenarios/ --format sarif --output results.sarif');
39
+ console.log('');
40
+ console.log('Ready for a deploy gate? Try: npx agent-gate init');
41
+ }
42
+ //# sourceMappingURL=init.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;CAkBzB,CAAC;AAEF,MAAM,UAAU,OAAO,CAAC,SAAkB;IACxC,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAE7C,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,8EAA8E,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;AACnE,CAAC"}
@@ -0,0 +1,4 @@
1
+ import type { Scenario } from './types.js';
2
+ export declare function parseScenario(filePath: string): Scenario;
3
+ export declare function substituteVariables(template: string, variables: Record<string, string>, stepOutputs: Record<string, string>): string;
4
+ //# sourceMappingURL=scenario-parser.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenario-parser.d.ts","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAQ,MAAM,YAAY,CAAC;AAEjD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAuCxD;AA0CD,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,MAAM,CAyBR"}
@@ -0,0 +1,92 @@
1
+ import * as fs from 'node:fs';
2
+ import * as yaml from 'js-yaml';
3
+ export function parseScenario(filePath) {
4
+ let content;
5
+ try {
6
+ content = fs.readFileSync(filePath, 'utf-8');
7
+ }
8
+ catch (e) {
9
+ throw new Error(`Cannot read scenario file: ${filePath}\n Hint: Check the path, or run 'stepproof init' to scaffold a starter scenario.`);
10
+ }
11
+ let raw;
12
+ try {
13
+ raw = yaml.load(content);
14
+ }
15
+ catch (e) {
16
+ throw new Error(`Invalid YAML in scenario file: ${e.message}`);
17
+ }
18
+ if (!raw || typeof raw !== 'object') {
19
+ throw new Error('Scenario file must be a YAML object');
20
+ }
21
+ const scenario = raw;
22
+ if (!scenario.name || typeof scenario.name !== 'string') {
23
+ throw new Error('Scenario must have a "name" field (string)');
24
+ }
25
+ if (!Array.isArray(scenario.steps) || scenario.steps.length === 0) {
26
+ throw new Error('Scenario must have a "steps" array with at least one step');
27
+ }
28
+ const steps = scenario.steps.map((rawStep, i) => validateStep(rawStep, i));
29
+ return {
30
+ name: scenario.name,
31
+ iterations: typeof scenario.iterations === 'number' ? scenario.iterations : 10,
32
+ variables: typeof scenario.variables === 'object' && scenario.variables !== null
33
+ ? scenario.variables
34
+ : {},
35
+ steps,
36
+ };
37
+ }
38
+ function validateStep(raw, index) {
39
+ if (!raw || typeof raw !== 'object') {
40
+ throw new Error(`Step ${index + 1} must be an object`);
41
+ }
42
+ const step = raw;
43
+ const pos = step.id ? `step "${step.id}"` : `step ${index + 1}`;
44
+ if (!step.id || typeof step.id !== 'string') {
45
+ throw new Error(`Step ${index + 1} must have an "id" field (string)`);
46
+ }
47
+ if (!step.provider || !['openai', 'anthropic'].includes(step.provider)) {
48
+ throw new Error(`${pos}: "provider" must be "openai" or "anthropic"`);
49
+ }
50
+ if (!step.model || typeof step.model !== 'string') {
51
+ throw new Error(`${pos}: "model" field is required (string)`);
52
+ }
53
+ if (!step.prompt || typeof step.prompt !== 'string') {
54
+ throw new Error(`${pos}: "prompt" field is required (string)`);
55
+ }
56
+ const minPassRate = typeof step.min_pass_rate === 'number' ? step.min_pass_rate : 0.8;
57
+ if (minPassRate < 0 || minPassRate > 1) {
58
+ throw new Error(`${pos}: "min_pass_rate" must be between 0.0 and 1.0`);
59
+ }
60
+ return {
61
+ id: step.id,
62
+ provider: step.provider,
63
+ model: step.model,
64
+ prompt: step.prompt,
65
+ system: typeof step.system === 'string' ? step.system : undefined,
66
+ min_pass_rate: minPassRate,
67
+ assertions: Array.isArray(step.assertions) ? step.assertions : [],
68
+ };
69
+ }
70
+ export function substituteVariables(template, variables, stepOutputs) {
71
+ return template.replace(/\{\{([^}]+)\}\}/g, (match, key) => {
72
+ const trimmed = key.trim();
73
+ // Check for step output reference: {{step_id.output}}
74
+ if (trimmed.includes('.')) {
75
+ const [stepId, field] = trimmed.split('.', 2);
76
+ if (field === 'output' && stepOutputs[stepId] !== undefined) {
77
+ return stepOutputs[stepId];
78
+ }
79
+ }
80
+ // Check global variables
81
+ if (variables[trimmed] !== undefined) {
82
+ return variables[trimmed];
83
+ }
84
+ // Check environment variables
85
+ if (process.env[trimmed] !== undefined) {
86
+ return process.env[trimmed];
87
+ }
88
+ // Leave unresolved — caller decides if this is an error
89
+ return match;
90
+ });
91
+ }
92
+ //# sourceMappingURL=scenario-parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenario-parser.js","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAGhC,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,mFAAmF,CAAC,CAAC;IAC7I,CAAC;IAED,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAG,GAA8B,CAAC;IAEhD,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAE5F,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,UAAU,EAAE,OAAO,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC9E,SAAS,EAAE,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI;YAC9E,CAAC,CAAC,QAAQ,CAAC,SAAmC;YAC9C,CAAC,CAAC,EAAE;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC;IAEhE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAkB,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,8CAA8C,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sCAAsC,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uCAAuC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;IACtF,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,+CAA+C,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,QAAQ,EAAE,IAAI,CAAC,QAAkC;QACjD,KAAK,EAAE,IAAI,CAAC,KAAe;QAC3B,MAAM,EAAE,IAAI,CAAC,MAAgB;QAC7B,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACjE,aAAa,EAAE,WAAW;QAC1B,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,SAAiC,EACjC,WAAmC;IAEnC,OAAO,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAE3B,sDAAsD;QACtD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC5D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAC/B,CAAC;QAED,wDAAwD;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,11 @@
1
+ import type { Scenario, ScenarioReport } from './types.js';
2
+ export interface RunOptions {
3
+ /** Override iterations from scenario file */
4
+ iterations?: number;
5
+ /** Called after each iteration completes */
6
+ onIterationComplete?: (iteration: number, total: number) => void;
7
+ /** Called after each step within an iteration */
8
+ onStepComplete?: (stepId: string, passed: boolean) => void;
9
+ }
10
+ export declare function runScenario(scenario: Scenario, scenarioFilePath: string, options?: RunOptions): Promise<ScenarioReport>;
11
+ //# sourceMappingURL=scenario-runner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenario-runner.d.ts","sourceRoot":"","sources":["../../src/core/scenario-runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAA2B,MAAM,YAAY,CAAC;AAEpF,MAAM,WAAW,UAAU;IACzB,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACjE,iDAAiD;IACjD,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;CAC5D;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,MAAM,EACxB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CA8FzB"}
@@ -0,0 +1,85 @@
1
+ import * as path from 'node:path';
2
+ import { getAdapter } from '../adapters/index.js';
3
+ import { runAssertions } from '../assertions/engine.js';
4
+ import { substituteVariables } from './scenario-parser.js';
5
+ export async function runScenario(scenario, scenarioFilePath, options = {}) {
6
+ const iterations = options.iterations ?? scenario.iterations ?? 10;
7
+ const scenarioDir = path.dirname(path.resolve(scenarioFilePath));
8
+ const variables = scenario.variables ?? {};
9
+ const startedAt = new Date().toISOString();
10
+ const startMs = Date.now();
11
+ const allResults = [];
12
+ for (let i = 1; i <= iterations; i++) {
13
+ const stepOutputs = {};
14
+ for (const step of scenario.steps) {
15
+ const resolvedPrompt = substituteVariables(step.prompt, variables, stepOutputs);
16
+ const resolvedSystem = step.system
17
+ ? substituteVariables(step.system, variables, stepOutputs)
18
+ : undefined;
19
+ const stepStartMs = Date.now();
20
+ let output = '';
21
+ let error;
22
+ try {
23
+ const adapter = getAdapter(step.provider, step.model);
24
+ output = await adapter.call(resolvedPrompt, resolvedSystem);
25
+ stepOutputs[step.id] = output;
26
+ }
27
+ catch (e) {
28
+ error = e.message;
29
+ stepOutputs[step.id] = '';
30
+ }
31
+ const durationMs = Date.now() - stepStartMs;
32
+ let assertionResults = [];
33
+ let assertionsPassed = false;
34
+ if (!error) {
35
+ const { results, allPassed } = await runAssertions(output, step.assertions, scenarioDir);
36
+ assertionResults = results;
37
+ assertionsPassed = allPassed;
38
+ }
39
+ const stepPassed = !error && assertionsPassed;
40
+ const result = {
41
+ stepId: step.id,
42
+ iteration: i,
43
+ output,
44
+ passed: stepPassed,
45
+ assertionResults,
46
+ error,
47
+ durationMs,
48
+ };
49
+ allResults.push(result);
50
+ options.onStepComplete?.(step.id, stepPassed);
51
+ }
52
+ options.onIterationComplete?.(i, iterations);
53
+ }
54
+ // Aggregate per-step summaries
55
+ const steps = scenario.steps.map((step) => {
56
+ const stepResults = allResults.filter((r) => r.stepId === step.id);
57
+ const passes = stepResults.filter((r) => r.passed).length;
58
+ const failures = stepResults.length - passes;
59
+ const passRate = stepResults.length > 0 ? passes / stepResults.length : 0;
60
+ const minPassRate = step.min_pass_rate ?? 0.8;
61
+ return {
62
+ stepId: step.id,
63
+ totalRuns: stepResults.length,
64
+ passes,
65
+ failures,
66
+ passRate,
67
+ minPassRate,
68
+ belowThreshold: passRate < minPassRate,
69
+ };
70
+ });
71
+ const allPassed = steps.every((s) => !s.belowThreshold);
72
+ const completedAt = new Date().toISOString();
73
+ const durationMs = Date.now() - startMs;
74
+ return {
75
+ scenarioName: scenario.name,
76
+ iterations,
77
+ startedAt,
78
+ completedAt,
79
+ durationMs,
80
+ steps,
81
+ allPassed,
82
+ results: allResults,
83
+ };
84
+ }
85
+ //# sourceMappingURL=scenario-runner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenario-runner.js","sourceRoot":"","sources":["../../src/core/scenario-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAkB,EAClB,gBAAwB,EACxB,UAAsB,EAAE;IAExB,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IACnE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;IAE3C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAiB,EAAE,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,WAAW,GAA2B,EAAE,CAAC;QAE/C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;YAChF,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM;gBAChC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC;gBAC1D,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,KAAyB,CAAC;YAE9B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBACtD,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;gBAC5D,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,KAAK,GAAI,CAAW,CAAC,OAAO,CAAC;gBAC7B,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;YAE5C,IAAI,gBAAgB,GAA0D,EAAE,CAAC;YACjF,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACzF,gBAAgB,GAAG,OAAO,CAAC;gBAC3B,gBAAgB,GAAG,SAAS,CAAC;YAC/B,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,gBAAgB,CAAC;YAE9C,MAAM,MAAM,GAAe;gBACzB,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,SAAS,EAAE,CAAC;gBACZ,MAAM;gBACN,MAAM,EAAE,UAAU;gBAClB,gBAAgB;gBAChB,KAAK;gBACL,UAAU;aACX,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,GAAkB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvD,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC1D,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC;QAE9C,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,cAAc,EAAE,QAAQ,GAAG,WAAW;SACvC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAExC,OAAO;QACL,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,UAAU;QACV,SAAS;QACT,WAAW;QACX,UAAU;QACV,KAAK;QACL,SAAS;QACT,OAAO,EAAE,UAAU;KACpB,CAAC;AACJ,CAAC"}