stepproof 0.2.22 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/anthropic.d.ts +3 -2
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +27 -5
- package/dist/adapters/anthropic.js.map +1 -1
- package/dist/adapters/base.d.ts +14 -1
- package/dist/adapters/base.d.ts.map +1 -1
- package/dist/adapters/gemini.d.ts +9 -0
- package/dist/adapters/gemini.d.ts.map +1 -0
- package/dist/adapters/gemini.js +80 -0
- package/dist/adapters/gemini.js.map +1 -0
- package/dist/adapters/index.d.ts.map +1 -1
- package/dist/adapters/index.js +7 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/adapters/ollama.d.ts +9 -0
- package/dist/adapters/ollama.d.ts.map +1 -0
- package/dist/adapters/ollama.js +61 -0
- package/dist/adapters/ollama.js.map +1 -0
- package/dist/adapters/openai.d.ts +3 -2
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +16 -5
- package/dist/adapters/openai.js.map +1 -1
- package/dist/assertions/engine.d.ts +6 -1
- package/dist/assertions/engine.d.ts.map +1 -1
- package/dist/assertions/engine.js +176 -11
- package/dist/assertions/engine.js.map +1 -1
- package/dist/baseline.d.ts +23 -0
- package/dist/baseline.d.ts.map +1 -0
- package/dist/baseline.js +98 -0
- package/dist/baseline.js.map +1 -0
- package/dist/cache.d.ts +5 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +71 -0
- package/dist/cache.js.map +1 -0
- package/dist/cli.js +283 -18
- package/dist/cli.js.map +1 -1
- package/dist/commands/compare.d.ts +43 -0
- package/dist/commands/compare.d.ts.map +1 -0
- package/dist/commands/compare.js +75 -0
- package/dist/commands/compare.js.map +1 -0
- package/dist/commands/diff.d.ts +2 -0
- package/dist/commands/diff.d.ts.map +1 -0
- package/dist/commands/diff.js +252 -0
- package/dist/commands/diff.js.map +1 -0
- package/dist/commands/history.d.ts +2 -0
- package/dist/commands/history.d.ts.map +1 -0
- package/dist/commands/history.js +46 -0
- package/dist/commands/history.js.map +1 -0
- package/dist/commands/results-store.d.ts +15 -0
- package/dist/commands/results-store.d.ts.map +1 -0
- package/dist/commands/results-store.js +77 -0
- package/dist/commands/results-store.js.map +1 -0
- package/dist/commands/view.d.ts +2 -0
- package/dist/commands/view.d.ts.map +1 -0
- package/dist/commands/view.js +51 -0
- package/dist/commands/view.js.map +1 -0
- package/dist/commands/watch.d.ts +7 -0
- package/dist/commands/watch.d.ts.map +1 -0
- package/dist/commands/watch.js +84 -0
- package/dist/commands/watch.js.map +1 -0
- package/dist/core/scenario-parser.d.ts.map +1 -1
- package/dist/core/scenario-parser.js +47 -5
- package/dist/core/scenario-parser.js.map +1 -1
- package/dist/core/scenario-runner.d.ts +12 -0
- package/dist/core/scenario-runner.d.ts.map +1 -1
- package/dist/core/scenario-runner.js +372 -41
- package/dist/core/scenario-runner.js.map +1 -1
- package/dist/core/types.d.ts +63 -9
- package/dist/core/types.d.ts.map +1 -1
- package/dist/dataset.d.ts +6 -0
- package/dist/dataset.d.ts.map +1 -0
- package/dist/dataset.js +108 -0
- package/dist/dataset.js.map +1 -0
- package/dist/reporters/github-comment.d.ts +8 -0
- package/dist/reporters/github-comment.d.ts.map +1 -0
- package/dist/reporters/github-comment.js +114 -0
- package/dist/reporters/github-comment.js.map +1 -0
- package/dist/reporters/html-reporter.d.ts +3 -0
- package/dist/reporters/html-reporter.d.ts.map +1 -0
- package/dist/reporters/html-reporter.js +152 -0
- package/dist/reporters/html-reporter.js.map +1 -0
- package/dist/reporters/terminal-reporter.d.ts +10 -1
- package/dist/reporters/terminal-reporter.d.ts.map +1 -1
- package/dist/reporters/terminal-reporter.js +139 -8
- package/dist/reporters/terminal-reporter.js.map +1 -1
- package/package.json +2 -1
package/dist/cli.js
CHANGED
|
@@ -1,17 +1,28 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { Command } from 'commander';
|
|
3
|
+
import chalk from 'chalk';
|
|
3
4
|
import * as path from 'node:path';
|
|
4
5
|
import * as os from 'node:os';
|
|
5
6
|
import { parseScenario } from './core/scenario-parser.js';
|
|
6
7
|
import { runScenario } from './core/scenario-runner.js';
|
|
7
8
|
import { writeJsonReport } from './reporters/json-reporter.js';
|
|
8
|
-
import { printReport } from './reporters/terminal-reporter.js';
|
|
9
|
+
import { printReport, formatComparison } from './reporters/terminal-reporter.js';
|
|
9
10
|
import { formatSarif } from './reporters/sarif-reporter.js';
|
|
10
11
|
import { formatJunit } from './reporters/junit-reporter.js';
|
|
11
12
|
import * as fs from 'node:fs';
|
|
12
13
|
import { guard, validate } from '@bilkobibitkov/preflight-license';
|
|
13
14
|
import { runInit } from './commands/init.js';
|
|
15
|
+
import { runView } from './commands/view.js';
|
|
16
|
+
import { runHistory } from './commands/history.js';
|
|
17
|
+
import { saveReport } from './commands/results-store.js';
|
|
14
18
|
import { sendTelemetry } from './telemetry.js';
|
|
19
|
+
import { saveBaseline, loadBaseline, compareWithBaseline, resetBaseline, listBaselines } from './baseline.js';
|
|
20
|
+
import { formatPRComment } from './reporters/github-comment.js';
|
|
21
|
+
import { runDiff } from './commands/diff.js';
|
|
22
|
+
import { clearCache } from './cache.js';
|
|
23
|
+
import { runComparison, parseProviderSpecs } from './commands/compare.js';
|
|
24
|
+
import { loadDataset } from './dataset.js';
|
|
25
|
+
import { startWatch } from './commands/watch.js';
|
|
15
26
|
const CLI_VERSION = '0.2.21';
|
|
16
27
|
/* ── Usage-based monetization (Preflight Suite — shared) ────────────── */
|
|
17
28
|
const TOOL_NAME = 'stepproof';
|
|
@@ -135,7 +146,9 @@ Examples:
|
|
|
135
146
|
stepproof init scaffold a starter scenario
|
|
136
147
|
stepproof run ./scenarios/first-test.yaml run one scenario
|
|
137
148
|
stepproof run ./scenarios/ run all scenarios in a directory
|
|
138
|
-
stepproof run test.yaml --format sarif --output results.sarif SARIF output for CI
|
|
149
|
+
stepproof run test.yaml --format sarif --output results.sarif SARIF output for CI
|
|
150
|
+
stepproof view open latest report in browser
|
|
151
|
+
stepproof history list recent runs`);
|
|
139
152
|
program
|
|
140
153
|
.command('init [dir]')
|
|
141
154
|
.description('Scaffold a starter scenario in ./scenarios/first-test.yaml')
|
|
@@ -165,6 +178,20 @@ program
|
|
|
165
178
|
process.exit(1);
|
|
166
179
|
}
|
|
167
180
|
});
|
|
181
|
+
program
|
|
182
|
+
.command('view [scenario]')
|
|
183
|
+
.description('Open the latest HTML report in your browser')
|
|
184
|
+
.action(async (scenario) => {
|
|
185
|
+
runView(scenario);
|
|
186
|
+
await sendTelemetry({ command: 'view', success: true, version: CLI_VERSION, outcome: 'opened' });
|
|
187
|
+
});
|
|
188
|
+
program
|
|
189
|
+
.command('history [scenario]')
|
|
190
|
+
.description('List recent runs with pass/fail, date, duration')
|
|
191
|
+
.action(async (scenario) => {
|
|
192
|
+
runHistory(scenario);
|
|
193
|
+
await sendTelemetry({ command: 'history', success: true, version: CLI_VERSION, outcome: 'listed' });
|
|
194
|
+
});
|
|
168
195
|
program
|
|
169
196
|
.command('run [scenario]')
|
|
170
197
|
.description('Run a scenario YAML file and report pass rates per step')
|
|
@@ -174,20 +201,29 @@ program
|
|
|
174
201
|
.option('--quiet', 'Suppress terminal output (use with --output for CI)')
|
|
175
202
|
.option('--format <format>', 'Output format: sarif, junit')
|
|
176
203
|
.option('--report <format>', '(deprecated: use --format)')
|
|
204
|
+
.option('--no-baseline', 'Skip baseline comparison and saving')
|
|
205
|
+
.option('--no-cache', 'Disable LLM response caching')
|
|
206
|
+
.option('--dataset <path>', 'CSV file with input rows — run scenario once per row')
|
|
177
207
|
.action(async (scenarioPath, opts) => {
|
|
178
208
|
// --report is deprecated; normalize to --format
|
|
179
209
|
if (opts.report && !opts.format) {
|
|
180
210
|
process.stderr.write('Warning: --report is deprecated, use --format instead\n');
|
|
181
211
|
opts.format = opts.report;
|
|
182
212
|
}
|
|
183
|
-
if (opts.format && opts.format !== 'sarif' && opts.format !== 'junit') {
|
|
184
|
-
console.error(`\nError: --format must be "sarif" or "
|
|
213
|
+
if (opts.format && opts.format !== 'sarif' && opts.format !== 'junit' && opts.format !== 'github') {
|
|
214
|
+
console.error(`\nError: --format must be "sarif", "junit", or "github", got "${opts.format}"`);
|
|
185
215
|
process.exit(2);
|
|
186
216
|
}
|
|
187
217
|
// License gate — check before running the scenario (avoid wasted API calls)
|
|
188
218
|
if (opts.format === 'sarif' || opts.format === 'junit') {
|
|
189
219
|
guard('team', { feature: `--format ${opts.format}` });
|
|
190
220
|
}
|
|
221
|
+
// Read scenario YAML for baseline storage (before any mutation)
|
|
222
|
+
let scenarioYaml;
|
|
223
|
+
try {
|
|
224
|
+
scenarioYaml = fs.readFileSync(path.resolve(process.cwd(), scenarioPath), 'utf-8');
|
|
225
|
+
}
|
|
226
|
+
catch { /* will fail later in parseScenario */ }
|
|
191
227
|
// Capture pro status once — used for telemetry throughout this command
|
|
192
228
|
const isPro = isProUser();
|
|
193
229
|
// Usage limit — check before running (avoid wasted API calls)
|
|
@@ -243,29 +279,61 @@ program
|
|
|
243
279
|
console.log(`Overriding iterations: ${scenario.iterations ?? 10} → ${opts.iterations}`);
|
|
244
280
|
}
|
|
245
281
|
}
|
|
282
|
+
// Load dataset if provided
|
|
283
|
+
let dataset;
|
|
284
|
+
if (opts.dataset) {
|
|
285
|
+
const datasetPath = path.resolve(process.cwd(), opts.dataset);
|
|
286
|
+
try {
|
|
287
|
+
dataset = loadDataset(datasetPath);
|
|
288
|
+
if (!isQuiet) {
|
|
289
|
+
console.log(`Dataset: ${opts.dataset} (${dataset.length} rows)`);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
catch (e) {
|
|
293
|
+
console.error(`\nError loading dataset: ${e.message}`);
|
|
294
|
+
process.exit(2);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
246
297
|
let currentIteration = 0;
|
|
247
298
|
const totalIterations = opts.iterations ?? scenario.iterations ?? 10;
|
|
299
|
+
const runOptions = {
|
|
300
|
+
iterations: opts.iterations,
|
|
301
|
+
noCache: !opts.cache,
|
|
302
|
+
cacheStats: undefined,
|
|
303
|
+
dataset,
|
|
304
|
+
datasetPath: opts.dataset,
|
|
305
|
+
onIterationComplete: (iteration, total) => {
|
|
306
|
+
currentIteration = iteration;
|
|
307
|
+
if (!isQuiet) {
|
|
308
|
+
process.stdout.write(`\r Completed iteration ${iteration}/${total}...`);
|
|
309
|
+
if (iteration === total) {
|
|
310
|
+
process.stdout.write('\r' + ' '.repeat(50) + '\r');
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
},
|
|
314
|
+
};
|
|
248
315
|
let report;
|
|
249
316
|
try {
|
|
250
|
-
report = await runScenario(scenario, resolvedPath,
|
|
251
|
-
iterations: opts.iterations,
|
|
252
|
-
onIterationComplete: (iteration, total) => {
|
|
253
|
-
currentIteration = iteration;
|
|
254
|
-
if (!isQuiet) {
|
|
255
|
-
process.stdout.write(`\r Completed iteration ${iteration}/${total}...`);
|
|
256
|
-
if (iteration === total) {
|
|
257
|
-
process.stdout.write('\r' + ' '.repeat(50) + '\r');
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
},
|
|
261
|
-
});
|
|
317
|
+
report = await runScenario(scenario, resolvedPath, runOptions);
|
|
262
318
|
}
|
|
263
319
|
catch (e) {
|
|
264
320
|
console.error(`\nError running scenario: ${e.message}`);
|
|
265
321
|
await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'error', exit_code: 2, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
266
322
|
process.exit(2);
|
|
267
323
|
}
|
|
268
|
-
//
|
|
324
|
+
// Baseline comparison
|
|
325
|
+
let baselineComparison;
|
|
326
|
+
let hasRegression = false;
|
|
327
|
+
if (opts.baseline) {
|
|
328
|
+
const baseline = loadBaseline(scenario.name);
|
|
329
|
+
if (baseline) {
|
|
330
|
+
baselineComparison = compareWithBaseline(report, baseline);
|
|
331
|
+
hasRegression = baselineComparison.hasRegression;
|
|
332
|
+
}
|
|
333
|
+
// Save current run as the new baseline (including scenario YAML for diff)
|
|
334
|
+
saveBaseline(scenario.name, report, scenarioYaml);
|
|
335
|
+
}
|
|
336
|
+
// Handle --format sarif / --format junit / --format github
|
|
269
337
|
if (opts.format === 'sarif' || opts.format === 'junit') {
|
|
270
338
|
const formatted = opts.format === 'sarif' ? formatSarif(report) : formatJunit(report);
|
|
271
339
|
const hasExplicitOutput = process.argv.includes('--output') || process.argv.includes('-o');
|
|
@@ -281,9 +349,28 @@ program
|
|
|
281
349
|
process.stdout.write(formatted + '\n');
|
|
282
350
|
}
|
|
283
351
|
}
|
|
352
|
+
if (opts.format === 'github') {
|
|
353
|
+
const markdown = formatPRComment(report, baselineComparison);
|
|
354
|
+
const hasExplicitOutput = process.argv.includes('--output') || process.argv.includes('-o');
|
|
355
|
+
if (hasExplicitOutput) {
|
|
356
|
+
try {
|
|
357
|
+
fs.writeFileSync(opts.output, markdown, 'utf-8');
|
|
358
|
+
}
|
|
359
|
+
catch (e) {
|
|
360
|
+
console.error(`Warning: Could not write GitHub comment: ${e.message}`);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
process.stdout.write(markdown + '\n');
|
|
365
|
+
}
|
|
366
|
+
}
|
|
284
367
|
const reportPath = opts.json ? opts.output : undefined;
|
|
285
368
|
if (!isQuiet) {
|
|
286
|
-
printReport(report,
|
|
369
|
+
printReport(report, {
|
|
370
|
+
reportPath,
|
|
371
|
+
baselineComparison,
|
|
372
|
+
cacheStats: runOptions.cacheStats,
|
|
373
|
+
});
|
|
287
374
|
}
|
|
288
375
|
if (opts.json) {
|
|
289
376
|
try {
|
|
@@ -293,8 +380,22 @@ program
|
|
|
293
380
|
console.error(`Warning: Could not write JSON report: ${e.message}`);
|
|
294
381
|
}
|
|
295
382
|
}
|
|
383
|
+
// Save report to .stepproof/results/ for view/history commands
|
|
384
|
+
try {
|
|
385
|
+
const savedPath = saveReport(report);
|
|
386
|
+
if (!isQuiet) {
|
|
387
|
+
console.log(chalk.dim(`Results saved to: ${savedPath}`));
|
|
388
|
+
console.log(chalk.dim('Run `stepproof view` to open in browser'));
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
catch { /* degrade gracefully */ }
|
|
296
392
|
// Track usage after successful run completion
|
|
297
393
|
await trackUsageAfterRun();
|
|
394
|
+
// Exit 1 if regression detected
|
|
395
|
+
if (hasRegression) {
|
|
396
|
+
await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'regression', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
397
|
+
process.exit(1);
|
|
398
|
+
}
|
|
298
399
|
// Exit 1 if any step below threshold — this is the CI gate
|
|
299
400
|
if (!report.allPassed) {
|
|
300
401
|
await sendTelemetry({ command: 'run', success: false, version: CLI_VERSION, outcome: 'fail', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
@@ -303,6 +404,170 @@ program
|
|
|
303
404
|
await sendTelemetry({ command: 'run', success: true, version: CLI_VERSION, outcome: 'pass', exit_code: 0, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
304
405
|
process.exit(0);
|
|
305
406
|
});
|
|
407
|
+
/* ── Watch command ────────────────────────────────────────────────── */
|
|
408
|
+
program
|
|
409
|
+
.command('watch <scenario>')
|
|
410
|
+
.description('Watch a scenario file and re-run on changes')
|
|
411
|
+
.option('-n, --iterations <number>', 'Number of iterations to run', parseInt)
|
|
412
|
+
.option('--no-cache', 'Disable LLM response caching')
|
|
413
|
+
.option('--dataset <path>', 'CSV file with input rows — run scenario once per row')
|
|
414
|
+
.action(async (scenarioPath, opts) => {
|
|
415
|
+
await sendTelemetry({ command: 'watch', success: true, version: CLI_VERSION, outcome: 'started' });
|
|
416
|
+
startWatch(scenarioPath, {
|
|
417
|
+
iterations: opts.iterations,
|
|
418
|
+
noCache: !opts.cache,
|
|
419
|
+
dataset: opts.dataset,
|
|
420
|
+
});
|
|
421
|
+
});
|
|
422
|
+
/* ── Baseline subcommands ──────────────────────────────────────────── */
|
|
423
|
+
const baselineCmd = program
|
|
424
|
+
.command('baseline')
|
|
425
|
+
.description('Manage baseline snapshots for regression detection');
|
|
426
|
+
baselineCmd
|
|
427
|
+
.command('show [scenario]')
|
|
428
|
+
.description('Show current baselines (all or for a specific scenario)')
|
|
429
|
+
.action(async (scenarioName) => {
|
|
430
|
+
const baselines = listBaselines();
|
|
431
|
+
if (baselines.length === 0) {
|
|
432
|
+
console.log('\nNo baselines saved yet. Run `stepproof run` to create one.\n');
|
|
433
|
+
process.exit(0);
|
|
434
|
+
}
|
|
435
|
+
const filtered = scenarioName
|
|
436
|
+
? baselines.filter(b => b.scenarioName.toLowerCase().includes(scenarioName.toLowerCase()))
|
|
437
|
+
: baselines;
|
|
438
|
+
if (filtered.length === 0) {
|
|
439
|
+
console.log(`\nNo baseline found for "${scenarioName}"\n`);
|
|
440
|
+
process.exit(0);
|
|
441
|
+
}
|
|
442
|
+
console.log('');
|
|
443
|
+
console.log(chalk.bold('Baselines'));
|
|
444
|
+
console.log(chalk.dim('─'.repeat(50)));
|
|
445
|
+
for (const b of filtered) {
|
|
446
|
+
console.log(` ${chalk.bold(b.scenarioName)}`);
|
|
447
|
+
for (const step of b.report.steps) {
|
|
448
|
+
const pct = (step.passRate * 100).toFixed(1);
|
|
449
|
+
console.log(` ${step.stepId}: ${pct}% (${step.passes}/${step.totalRuns})`);
|
|
450
|
+
}
|
|
451
|
+
console.log(chalk.dim(` Recorded: ${b.report.completedAt}`));
|
|
452
|
+
console.log('');
|
|
453
|
+
}
|
|
454
|
+
await sendTelemetry({ command: 'baseline_show', success: true, version: CLI_VERSION, outcome: 'listed' });
|
|
455
|
+
});
|
|
456
|
+
baselineCmd
|
|
457
|
+
.command('reset [scenario]')
|
|
458
|
+
.description('Clear baselines (all or for a specific scenario)')
|
|
459
|
+
.action(async (scenarioName) => {
|
|
460
|
+
resetBaseline(scenarioName);
|
|
461
|
+
if (scenarioName) {
|
|
462
|
+
console.log(`\nBaseline cleared for "${scenarioName}"\n`);
|
|
463
|
+
}
|
|
464
|
+
else {
|
|
465
|
+
console.log('\nAll baselines cleared\n');
|
|
466
|
+
}
|
|
467
|
+
await sendTelemetry({ command: 'baseline_reset', success: true, version: CLI_VERSION, outcome: 'cleared' });
|
|
468
|
+
});
|
|
469
|
+
/* ── Cache subcommands ────────────────────────────────────────────── */
|
|
470
|
+
const cacheCmd = program
|
|
471
|
+
.command('cache')
|
|
472
|
+
.description('Manage LLM response cache');
|
|
473
|
+
cacheCmd
|
|
474
|
+
.command('clear')
|
|
475
|
+
.description('Delete all cached LLM responses')
|
|
476
|
+
.action(async () => {
|
|
477
|
+
const count = clearCache();
|
|
478
|
+
console.log(`\nCleared ${count} cached response${count === 1 ? '' : 's'}\n`);
|
|
479
|
+
await sendTelemetry({ command: 'cache_clear', success: true, version: CLI_VERSION, outcome: 'cleared' });
|
|
480
|
+
});
|
|
481
|
+
/* ── Compare command ──────────────────────────────────────────────── */
|
|
482
|
+
program
|
|
483
|
+
.command('compare <scenario>')
|
|
484
|
+
.description('Run a scenario against multiple providers and compare results')
|
|
485
|
+
.requiredOption('--providers <list>', 'Comma-separated provider:model pairs (e.g. anthropic:claude-sonnet-4-6,openai:gpt-4o)')
|
|
486
|
+
.option('-n, --iterations <number>', 'Number of iterations per provider', parseInt)
|
|
487
|
+
.option('--quiet', 'Suppress progress output')
|
|
488
|
+
.action(async (scenarioPath, opts) => {
|
|
489
|
+
const isPro = isProUser();
|
|
490
|
+
const startMs = Date.now();
|
|
491
|
+
if (!checkUsageLimit()) {
|
|
492
|
+
await sendTelemetry({ command: 'compare', success: false, version: CLI_VERSION, outcome: 'rate_limited', exit_code: 1, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
493
|
+
process.exit(1);
|
|
494
|
+
}
|
|
495
|
+
let providerSpecs;
|
|
496
|
+
try {
|
|
497
|
+
providerSpecs = parseProviderSpecs(opts.providers);
|
|
498
|
+
}
|
|
499
|
+
catch (e) {
|
|
500
|
+
console.error(`\nError: ${e.message}\n`);
|
|
501
|
+
process.exit(2);
|
|
502
|
+
}
|
|
503
|
+
if (providerSpecs.length < 2) {
|
|
504
|
+
console.error('\nError: --providers must specify at least 2 provider:model pairs\n');
|
|
505
|
+
process.exit(2);
|
|
506
|
+
}
|
|
507
|
+
const resolvedPath = path.resolve(process.cwd(), scenarioPath);
|
|
508
|
+
try {
|
|
509
|
+
const stat = fs.statSync(resolvedPath);
|
|
510
|
+
if (stat.isDirectory()) {
|
|
511
|
+
console.error(`\nError: "${scenarioPath}" is a directory. Specify a single scenario file.\n`);
|
|
512
|
+
process.exit(2);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
catch (statErr) {
|
|
516
|
+
if (statErr.code === 'ENOENT') {
|
|
517
|
+
console.error(`\nError: Scenario not found: ${resolvedPath}\n`);
|
|
518
|
+
process.exit(2);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
let scenario;
|
|
522
|
+
try {
|
|
523
|
+
scenario = parseScenario(resolvedPath);
|
|
524
|
+
}
|
|
525
|
+
catch (e) {
|
|
526
|
+
console.error(`\nError parsing scenario: ${e.message}`);
|
|
527
|
+
process.exit(2);
|
|
528
|
+
}
|
|
529
|
+
const iterations = opts.iterations ?? scenario.iterations ?? 10;
|
|
530
|
+
if (!opts.quiet) {
|
|
531
|
+
console.log(`\nComparing ${providerSpecs.length} providers on: ${scenario.name}`);
|
|
532
|
+
console.log(`Iterations per provider: ${iterations}\n`);
|
|
533
|
+
}
|
|
534
|
+
let comparisonReport;
|
|
535
|
+
try {
|
|
536
|
+
comparisonReport = await runComparison(scenario, resolvedPath, providerSpecs, iterations, {
|
|
537
|
+
quiet: opts.quiet,
|
|
538
|
+
onProviderStart: (provider, model) => {
|
|
539
|
+
if (!opts.quiet) {
|
|
540
|
+
console.log(` Running ${provider}:${model}...`);
|
|
541
|
+
}
|
|
542
|
+
},
|
|
543
|
+
onIterationComplete: (provider, iteration, total) => {
|
|
544
|
+
if (!opts.quiet) {
|
|
545
|
+
process.stdout.write(`\r ${provider}: iteration ${iteration}/${total}...`);
|
|
546
|
+
if (iteration === total) {
|
|
547
|
+
process.stdout.write('\r' + ' '.repeat(60) + '\r');
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
},
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
catch (e) {
|
|
554
|
+
console.error(`\nError running comparison: ${e.message}`);
|
|
555
|
+
await sendTelemetry({ command: 'compare', success: false, version: CLI_VERSION, outcome: 'error', exit_code: 2, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
556
|
+
process.exit(2);
|
|
557
|
+
}
|
|
558
|
+
console.log(formatComparison(comparisonReport));
|
|
559
|
+
await trackUsageAfterRun();
|
|
560
|
+
await sendTelemetry({ command: 'compare', success: true, version: CLI_VERSION, outcome: 'compared', exit_code: 0, duration_ms: Date.now() - startMs, is_pro: isPro });
|
|
561
|
+
process.exit(0);
|
|
562
|
+
});
|
|
563
|
+
/* ── Diff command ─────────────────────────────────────────────────── */
|
|
564
|
+
program
|
|
565
|
+
.command('diff <scenario>')
|
|
566
|
+
.description('Show what changed in a scenario since the last baseline')
|
|
567
|
+
.action(async (scenarioPath) => {
|
|
568
|
+
runDiff(scenarioPath);
|
|
569
|
+
await sendTelemetry({ command: 'diff', success: true, version: CLI_VERSION, outcome: 'shown' });
|
|
570
|
+
});
|
|
306
571
|
program.action(() => {
|
|
307
572
|
const extra = process.argv.slice(2).filter(a => !a.startsWith('-'));
|
|
308
573
|
if (extra.length > 0) {
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,WAAW,GAAG,QAAQ,CAAC;AAE7B,2EAA2E;AAE3E,MAAM,SAAS,GAAG,WAAoB,CAAC;AACvC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,WAAW,GAAG,gDAAgD,CAAC;AAErE,yBAAyB;AACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;AAC5D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAEhE,+DAA+D;AAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAczD,oEAAoE;AACpE,SAAS,aAAa;IACpB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACzC,IAAI,MAAM,EAAE,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAqB,CAAC;YAC3F,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAqB,CAAC;YACpF,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;AAChD,CAAC;AAED,uFAAuF;AACvF,SAAS,eAAe;IACtB,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAgB,CAAC;YACpF,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;oBAC9B,MAAM,CAAC,GAAG,GAAG,UAAU,CAAC;oBACxB,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACvB,CAAC;gBACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;oBAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACzD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;IACnC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;AACvI,CAAC;AAED,0DAA0D;AAC1D,SAAS,gBAAgB,CAAC,MAAmB;IAC3C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;AACtC,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe;IACtB,IAAI,SAAS,EAAE;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,IAAI,KAAK,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wBAAwB,gBAAgB,uBAAuB;YAC/D,+DAA+D,WAAW,IAAI;YAC9E,uEAAuE,CACxE,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,kBAAkB;IAC/B,IAAI,SAAS,EAAE;QAAE,OAAO;IACxB,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACrB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;IACzB,MAAM,SAAS,GAAG,kBAAkB,GAAG,IAAI,CAAC;IAE5C,uEAAuE;IACvE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,+CAA+C;YAChF,uDAAuD,WAAW,IAAI;YACtE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;SAAM,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,gCAAgC,SAAS,IAAI,OAAO,UAAU;YAC/F,wDAAwD,WAAW,IAAI;YACvE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,OAAO,EAAE;;;;;qFAK6D,CAAC,CAAC;AAEvF,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;IAC7B,OAAO,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;AACrG,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;IAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;QACpE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,GAAG,+DAA+D,CAAC,CAAC;QAChI,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9I,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,MAAM,CAAC,CAAC;QAC9E,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,YAAgC,EAAE,IAOhD,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,uEAAuE;IACvE,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAE1B,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QACvB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACvK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,0DAA0D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnL,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,YAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAa,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,aAAa,YAAY,mBAAmB,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAK,OAAiC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gCAAgC,YAAY,EAAE,CAAC,CAAC;YAC9D,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,4DAA4D;IAC9D,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,mBAAmB,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACxC,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,kBAAkB,EAAE,CAAC;IAE3B,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU;AAC5B,CAAC,CAAC,CAAC;AAEH,2DAA2D;AAC3D,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,EAAE;IAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,yBAAyB,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;QAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC1D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0HAA0H,CAAC,CAAC;QACjJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,0DAA0D;IAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AAChG,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,aAAa,EAAE,aAAa,EAAoB,MAAM,eAAe,CAAC;AAChI,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,WAAW,GAAG,QAAQ,CAAC;AAE7B,2EAA2E;AAE3E,MAAM,SAAS,GAAG,WAAoB,CAAC;AACvC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,WAAW,GAAG,gDAAgD,CAAC;AAErE,yBAAyB;AACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;AAC5D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAEhE,+DAA+D;AAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAczD,oEAAoE;AACpE,SAAS,aAAa;IACpB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACzC,IAAI,MAAM,EAAE,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAqB,CAAC;YAC3F,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAqB,CAAC;YACpF,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;AAChD,CAAC;AAED,uFAAuF;AACvF,SAAS,eAAe;IACtB,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAgB,CAAC;YACpF,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;oBAC9B,MAAM,CAAC,GAAG,GAAG,UAAU,CAAC;oBACxB,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACvB,CAAC;gBACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;oBAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACzD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;IACnC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;AACvI,CAAC;AAED,0DAA0D;AAC1D,SAAS,gBAAgB,CAAC,MAAmB;IAC3C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;AACtC,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe;IACtB,IAAI,SAAS,EAAE;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,IAAI,KAAK,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wBAAwB,gBAAgB,uBAAuB;YAC/D,+DAA+D,WAAW,IAAI;YAC9E,uEAAuE,CACxE,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,kBAAkB;IAC/B,IAAI,SAAS,EAAE;QAAE,OAAO;IACxB,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACrB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;IACzB,MAAM,SAAS,GAAG,kBAAkB,GAAG,IAAI,CAAC;IAE5C,uEAAuE;IACvE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,+CAA+C;YAChF,uDAAuD,WAAW,IAAI;YACtE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;SAAM,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,OAAO,IAAI,IAAI,kBAAkB,gCAAgC,SAAS,IAAI,OAAO,UAAU;YAC/F,wDAAwD,WAAW,IAAI;YACvE,oDAAoD,CACrD,CAAC;QACF,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,OAAO,EAAE;;;;;;;yEAOiD,CAAC,CAAC;AAE3E,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;IAC7B,OAAO,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;AACrG,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;IAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;QACpE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,GAAG,+DAA+D,CAAC,CAAC;QAChI,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9I,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,MAAM,CAAC,CAAC;QAC9E,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,iBAAiB,CAAC;KAC1B,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,KAAK,EAAE,QAAiB,EAAE,EAAE;IAClC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AACnG,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,oBAAoB,CAAC;KAC7B,WAAW,CAAC,iDAAiD,CAAC;KAC9D,MAAM,CAAC,KAAK,EAAE,QAAiB,EAAE,EAAE;IAClC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AACtG,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,eAAe,EAAE,qCAAqC,CAAC;KAC9D,MAAM,CAAC,YAAY,EAAE,8BAA8B,CAAC;KACpD,MAAM,CAAC,kBAAkB,EAAE,sDAAsD,CAAC;KAClF,MAAM,CAAC,KAAK,EAAE,YAAgC,EAAE,IAUhD,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAClG,OAAO,CAAC,KAAK,CAAC,iEAAiE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAC/F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,gEAAgE;IAChE,IAAI,YAAgC,CAAC;IACrC,IAAI,CAAC;QACH,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAa,CAAC,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC,CAAC,sCAAsC,CAAC,CAAC;IAElD,uEAAuE;IACvE,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAE1B,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QACvB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACvK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,0DAA0D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnL,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,YAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAa,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,aAAa,YAAY,mBAAmB,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAK,OAAiC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gCAAgC,YAAY,EAAE,CAAC,CAAC;YAC9D,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,4DAA4D;IAC9D,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,IAAI,OAAkD,CAAC;IACvD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,4BAA6B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,MAAM,UAAU,GAAmD;QACjE,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK;QACpB,UAAU,EAAE,SAAS;QACrB,OAAO;QACP,WAAW,EAAE,IAAI,CAAC,OAAO;QACzB,mBAAmB,EAAE,CAAC,SAAiB,EAAE,KAAa,EAAE,EAAE;YACxD,gBAAgB,GAAG,SAAS,CAAC;YAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;gBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;IAEF,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;IACjE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,sBAAsB;IACtB,IAAI,kBAAkB,CAAC;IACvB,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,kBAAkB,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC3D,aAAa,GAAG,kBAAkB,CAAC,aAAa,CAAC;QACnD,CAAC;QACD,0EAA0E;QAC1E,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IACpD,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QAC7D,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4CAA6C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE;YAClB,UAAU;YACV,kBAAkB;YAClB,UAAU,EAAE,UAAU,CAAC,UAAU;SAClC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAC,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEpC,8CAA8C;IAC9C,MAAM,kBAAkB,EAAE,CAAC;IAE3B,gCAAgC;IAChC,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACrK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9J,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,yEAAyE;AAEzE,OAAO;KACJ,OAAO,CAAC,kBAAkB,CAAC;KAC3B,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,2BAA2B,EAAE,6BAA6B,EAAE,QAAQ,CAAC;KAC5E,MAAM,CAAC,YAAY,EAAE,8BAA8B,CAAC;KACpD,MAAM,CAAC,kBAAkB,EAAE,sDAAsD,CAAC;KAClF,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,IAIpC,EAAE,EAAE;IACH,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IACnG,UAAU,CAAC,YAAY,EAAE;QACvB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK;QACpB,OAAO,EAAE,IAAI,CAAC,OAAO;KACtB,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,0EAA0E;AAE1E,MAAM,WAAW,GAAG,OAAO;KACxB,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,oDAAoD,CAAC,CAAC;AAErE,WAAW;KACR,OAAO,CAAC,iBAAiB,CAAC;KAC1B,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,KAAK,EAAE,YAAqB,EAAE,EAAE;IACtC,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;QAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY;QAC3B,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1F,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B,YAAY,KAAK,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC/C,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC5G,CAAC,CAAC,CAAC;AAEL,WAAW;KACR,OAAO,CAAC,kBAAkB,CAAC;KAC3B,WAAW,CAAC,kDAAkD,CAAC;KAC/D,MAAM,CAAC,KAAK,EAAE,YAAqB,EAAE,EAAE;IACtC,aAAa,CAAC,YAAY,CAAC,CAAC;IAC5B,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,2BAA2B,YAAY,KAAK,CAAC,CAAC;IAC5D,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9G,CAAC,CAAC,CAAC;AAEL,yEAAyE;AAEzE,MAAM,QAAQ,GAAG,OAAO;KACrB,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,2BAA2B,CAAC,CAAC;AAE5C,QAAQ;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,iCAAiC,CAAC;KAC9C,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,mBAAmB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7E,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;AAC3G,CAAC,CAAC,CAAC;AAEL,yEAAyE;AAEzE,OAAO;KACJ,OAAO,CAAC,oBAAoB,CAAC;KAC7B,WAAW,CAAC,+DAA+D,CAAC;KAC5E,cAAc,CAAC,oBAAoB,EAAE,uFAAuF,CAAC;KAC7H,MAAM,CAAC,2BAA2B,EAAE,mCAAmC,EAAE,QAAQ,CAAC;KAClF,MAAM,CAAC,SAAS,EAAE,0BAA0B,CAAC;KAC7C,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,IAIpC,EAAE,EAAE;IACH,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QACvB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3K,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,aAAa,CAAC;IAClB,IAAI,CAAC;QACH,aAAa,GAAG,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,YAAa,CAAW,CAAC,OAAO,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;IAE/D,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,aAAa,YAAY,qDAAqD,CAAC,CAAC;YAC9F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAK,OAAiC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gCAAgC,YAAY,IAAI,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAEhE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,eAAe,aAAa,CAAC,MAAM,kBAAkB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAClF,OAAO,CAAC,GAAG,CAAC,4BAA4B,UAAU,IAAI,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,gBAAgB,CAAC;IACrB,IAAI,CAAC;QACH,gBAAgB,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE;YACxF,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,eAAe,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACnC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBAChB,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,IAAI,KAAK,KAAK,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YACD,mBAAmB,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;gBAClD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,QAAQ,eAAe,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBAC9E,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,+BAAgC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACrE,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACpK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAEhD,MAAM,kBAAkB,EAAE,CAAC;IAC3B,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACtK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,yEAAyE;AAEzE,OAAO;KACJ,OAAO,CAAC,iBAAiB,CAAC;KAC1B,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,EAAE;IACrC,OAAO,CAAC,YAAY,CAAC,CAAC;IACtB,MAAM,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AAClG,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU;AAC5B,CAAC,CAAC,CAAC;AAEH,2DAA2D;AAC3D,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,EAAE;IAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,yBAAyB,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;QAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC1D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0HAA0H,CAAC,CAAC;QACjJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,0DAA0D;IAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Scenario, ScenarioReport } from '../core/types.js';
|
|
2
|
+
export interface ProviderSpec {
|
|
3
|
+
provider: string;
|
|
4
|
+
model: string;
|
|
5
|
+
}
|
|
6
|
+
export interface ProviderResult {
|
|
7
|
+
provider: string;
|
|
8
|
+
model: string;
|
|
9
|
+
report: ScenarioReport;
|
|
10
|
+
}
|
|
11
|
+
export interface StepBreakdown {
|
|
12
|
+
stepId: string;
|
|
13
|
+
rates: Array<{
|
|
14
|
+
provider: string;
|
|
15
|
+
model: string;
|
|
16
|
+
passRate: number;
|
|
17
|
+
passes: number;
|
|
18
|
+
totalRuns: number;
|
|
19
|
+
}>;
|
|
20
|
+
bestProvider: string;
|
|
21
|
+
bestModel: string;
|
|
22
|
+
}
|
|
23
|
+
export interface ComparisonReport {
|
|
24
|
+
scenarioName: string;
|
|
25
|
+
iterations: number;
|
|
26
|
+
providers: ProviderResult[];
|
|
27
|
+
winner: string;
|
|
28
|
+
winnerModel: string;
|
|
29
|
+
stepBreakdown: StepBreakdown[];
|
|
30
|
+
durationMs: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Run the same scenario against multiple providers and produce a comparison.
|
|
34
|
+
* Substitutes provider/model on ALL steps for each provider run.
|
|
35
|
+
*/
|
|
36
|
+
export declare function runComparison(scenario: Scenario, scenarioFilePath: string, providers: ProviderSpec[], iterations: number, options?: {
|
|
37
|
+
quiet?: boolean;
|
|
38
|
+
onProviderStart?: (provider: string, model: string) => void;
|
|
39
|
+
onIterationComplete?: (provider: string, iteration: number, total: number) => void;
|
|
40
|
+
}): Promise<ComparisonReport>;
|
|
41
|
+
/** Parse "provider:model" strings like "anthropic:claude-sonnet-4-6,openai:gpt-4o" */
|
|
42
|
+
export declare function parseProviderSpecs(input: string): ProviderSpec[];
|
|
43
|
+
//# sourceMappingURL=compare.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/commands/compare.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAe,MAAM,kBAAkB,CAAC;AAG9E,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvG,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,aAAa,EAAE,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,YAAY,EAAE,EACzB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5D,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/E,GACL,OAAO,CAAC,gBAAgB,CAAC,CA6D3B;AAED,sFAAsF;AACtF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAYhE"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { runScenario } from '../core/scenario-runner.js';
|
|
2
|
+
/**
|
|
3
|
+
* Run the same scenario against multiple providers and produce a comparison.
|
|
4
|
+
* Substitutes provider/model on ALL steps for each provider run.
|
|
5
|
+
*/
|
|
6
|
+
export async function runComparison(scenario, scenarioFilePath, providers, iterations, options = {}) {
|
|
7
|
+
const startMs = Date.now();
|
|
8
|
+
const results = [];
|
|
9
|
+
for (const spec of providers) {
|
|
10
|
+
options.onProviderStart?.(spec.provider, spec.model);
|
|
11
|
+
// Clone scenario with overridden provider/model on all steps
|
|
12
|
+
const overriddenScenario = {
|
|
13
|
+
...scenario,
|
|
14
|
+
iterations,
|
|
15
|
+
steps: scenario.steps.map(step => ({
|
|
16
|
+
...step,
|
|
17
|
+
provider: spec.provider,
|
|
18
|
+
model: spec.model,
|
|
19
|
+
})),
|
|
20
|
+
};
|
|
21
|
+
const report = await runScenario(overriddenScenario, scenarioFilePath, {
|
|
22
|
+
iterations,
|
|
23
|
+
onIterationComplete: (iter, total) => {
|
|
24
|
+
options.onIterationComplete?.(spec.provider, iter, total);
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
results.push({ provider: spec.provider, model: spec.model, report });
|
|
28
|
+
}
|
|
29
|
+
// Build step breakdown — compare pass rates per step across providers
|
|
30
|
+
const stepIds = scenario.steps.map(s => s.id);
|
|
31
|
+
const stepBreakdown = stepIds.map(stepId => {
|
|
32
|
+
const rates = results.map(r => {
|
|
33
|
+
const stepSummary = r.report.steps.find(s => s.stepId === stepId);
|
|
34
|
+
return {
|
|
35
|
+
provider: r.provider,
|
|
36
|
+
model: r.model,
|
|
37
|
+
passRate: stepSummary?.passRate ?? 0,
|
|
38
|
+
passes: stepSummary?.passes ?? 0,
|
|
39
|
+
totalRuns: stepSummary?.totalRuns ?? 0,
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
const best = rates.reduce((a, b) => a.passRate >= b.passRate ? a : b);
|
|
43
|
+
return { stepId, rates, bestProvider: best.provider, bestModel: best.model };
|
|
44
|
+
});
|
|
45
|
+
// Overall winner: highest average pass rate across all steps
|
|
46
|
+
const avgRates = results.map(r => {
|
|
47
|
+
const avg = r.report.steps.reduce((sum, s) => sum + s.passRate, 0) / r.report.steps.length;
|
|
48
|
+
return { provider: r.provider, model: r.model, avg };
|
|
49
|
+
});
|
|
50
|
+
const winner = avgRates.reduce((a, b) => a.avg >= b.avg ? a : b);
|
|
51
|
+
return {
|
|
52
|
+
scenarioName: scenario.name,
|
|
53
|
+
iterations,
|
|
54
|
+
providers: results,
|
|
55
|
+
winner: winner.provider,
|
|
56
|
+
winnerModel: winner.model,
|
|
57
|
+
stepBreakdown,
|
|
58
|
+
durationMs: Date.now() - startMs,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** Parse "provider:model" strings like "anthropic:claude-sonnet-4-6,openai:gpt-4o" */
|
|
62
|
+
export function parseProviderSpecs(input) {
|
|
63
|
+
return input.split(',').map(s => {
|
|
64
|
+
const trimmed = s.trim();
|
|
65
|
+
const colonIdx = trimmed.indexOf(':');
|
|
66
|
+
if (colonIdx === -1) {
|
|
67
|
+
throw new Error(`Invalid provider spec "${trimmed}" — expected format: provider:model (e.g. anthropic:claude-sonnet-4-6)`);
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
provider: trimmed.slice(0, colonIdx),
|
|
71
|
+
model: trimmed.slice(colonIdx + 1),
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=compare.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/commands/compare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AA8BzD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAkB,EAClB,gBAAwB,EACxB,SAAyB,EACzB,UAAkB,EAClB,UAII,EAAE;IAEN,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAErD,6DAA6D;QAC7D,MAAM,kBAAkB,GAAa;YACnC,GAAG,QAAQ;YACX,UAAU;YACV,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjC,GAAG,IAAI;gBACP,QAAQ,EAAE,IAAI,CAAC,QAA4C;gBAC3D,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;SACJ,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,kBAAkB,EAAE,gBAAgB,EAAE;YACrE,UAAU;YACV,mBAAmB,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBACnC,OAAO,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC5D,CAAC;SACF,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,sEAAsE;IACtE,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAoB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC5B,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;YAClE,OAAO;gBACL,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,QAAQ,EAAE,WAAW,EAAE,QAAQ,IAAI,CAAC;gBACpC,MAAM,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;gBAChC,SAAS,EAAE,WAAW,EAAE,SAAS,IAAI,CAAC;aACvC,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QAC/B,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QAC3F,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;IACvD,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjE,OAAO;QACL,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,UAAU;QACV,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,MAAM,CAAC,QAAQ;QACvB,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,aAAa;QACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KACjC,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QAC9B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,wEAAwE,CAAC,CAAC;QAC7H,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC;YACpC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;SACnC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../src/commands/diff.ts"],"names":[],"mappings":"AAoHA,wBAAgB,OAAO,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CA8KlD"}
|