stepproof 0.2.22 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/adapters/anthropic.d.ts +2 -2
  2. package/dist/adapters/anthropic.d.ts.map +1 -1
  3. package/dist/adapters/anthropic.js +11 -4
  4. package/dist/adapters/anthropic.js.map +1 -1
  5. package/dist/adapters/base.d.ts +9 -1
  6. package/dist/adapters/base.d.ts.map +1 -1
  7. package/dist/adapters/gemini.d.ts +8 -0
  8. package/dist/adapters/gemini.d.ts.map +1 -0
  9. package/dist/adapters/gemini.js +49 -0
  10. package/dist/adapters/gemini.js.map +1 -0
  11. package/dist/adapters/index.d.ts.map +1 -1
  12. package/dist/adapters/index.js +7 -1
  13. package/dist/adapters/index.js.map +1 -1
  14. package/dist/adapters/ollama.d.ts +8 -0
  15. package/dist/adapters/ollama.d.ts.map +1 -0
  16. package/dist/adapters/ollama.js +51 -0
  17. package/dist/adapters/ollama.js.map +1 -0
  18. package/dist/adapters/openai.d.ts +2 -2
  19. package/dist/adapters/openai.d.ts.map +1 -1
  20. package/dist/adapters/openai.js +7 -1
  21. package/dist/adapters/openai.js.map +1 -1
  22. package/dist/assertions/engine.d.ts +6 -1
  23. package/dist/assertions/engine.d.ts.map +1 -1
  24. package/dist/assertions/engine.js +176 -11
  25. package/dist/assertions/engine.js.map +1 -1
  26. package/dist/baseline.d.ts +22 -0
  27. package/dist/baseline.d.ts.map +1 -0
  28. package/dist/baseline.js +81 -0
  29. package/dist/baseline.js.map +1 -0
  30. package/dist/cache.d.ts +5 -0
  31. package/dist/cache.d.ts.map +1 -0
  32. package/dist/cache.js +71 -0
  33. package/dist/cache.js.map +1 -0
  34. package/dist/cli.js +214 -15
  35. package/dist/cli.js.map +1 -1
  36. package/dist/commands/compare.d.ts +43 -0
  37. package/dist/commands/compare.d.ts.map +1 -0
  38. package/dist/commands/compare.js +75 -0
  39. package/dist/commands/compare.js.map +1 -0
  40. package/dist/commands/history.d.ts +2 -0
  41. package/dist/commands/history.d.ts.map +1 -0
  42. package/dist/commands/history.js +46 -0
  43. package/dist/commands/history.js.map +1 -0
  44. package/dist/commands/results-store.d.ts +15 -0
  45. package/dist/commands/results-store.d.ts.map +1 -0
  46. package/dist/commands/results-store.js +77 -0
  47. package/dist/commands/results-store.js.map +1 -0
  48. package/dist/commands/view.d.ts +2 -0
  49. package/dist/commands/view.d.ts.map +1 -0
  50. package/dist/commands/view.js +51 -0
  51. package/dist/commands/view.js.map +1 -0
  52. package/dist/core/scenario-runner.d.ts +8 -0
  53. package/dist/core/scenario-runner.d.ts.map +1 -1
  54. package/dist/core/scenario-runner.js +56 -5
  55. package/dist/core/scenario-runner.js.map +1 -1
  56. package/dist/core/types.d.ts +21 -7
  57. package/dist/core/types.d.ts.map +1 -1
  58. package/dist/reporters/html-reporter.d.ts +3 -0
  59. package/dist/reporters/html-reporter.d.ts.map +1 -0
  60. package/dist/reporters/html-reporter.js +152 -0
  61. package/dist/reporters/html-reporter.js.map +1 -0
  62. package/dist/reporters/terminal-reporter.d.ts +10 -1
  63. package/dist/reporters/terminal-reporter.d.ts.map +1 -1
  64. package/dist/reporters/terminal-reporter.js +111 -7
  65. package/dist/reporters/terminal-reporter.js.map +1 -1
  66. package/package.json +2 -1
package/dist/reporters/html-reporter.js ADDED
@@ -0,0 +1,152 @@
1
+ export function generateHtmlReport(report) {
2
+ const reportJson = JSON.stringify(report);
3
+ const passColor = '#22c55e';
4
+ const failColor = '#ef4444';
5
+ const verdictColor = report.allPassed ? passColor : failColor;
6
+ const verdictText = report.allPassed ? 'PASSED' : 'FAILED';
7
+ return `<!DOCTYPE html>
8
+ <html lang="en">
9
+ <head>
10
+ <meta charset="UTF-8">
11
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
12
+ <title>Stepproof — ${escHtml(report.scenarioName)}</title>
13
+ <style>
14
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
15
+ body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #0f172a; color: #e2e8f0; line-height: 1.6; padding: 2rem; max-width: 900px; margin: 0 auto; }
16
+ h1 { font-size: 1.5rem; font-weight: 600; margin-bottom: 0.25rem; }
17
+ .meta { color: #94a3b8; font-size: 0.875rem; margin-bottom: 1.5rem; }
18
+ .meta span { margin-right: 1.5rem; }
19
+ .verdict { display: inline-block; padding: 0.25rem 0.75rem; border-radius: 4px; font-weight: 700; font-size: 0.875rem; background: ${verdictColor}20; color: ${verdictColor}; border: 1px solid ${verdictColor}40; margin-bottom: 1.5rem; }
20
+ .card { background: #1e293b; border-radius: 8px; padding: 1.25rem; margin-bottom: 1rem; border: 1px solid #334155; }
21
+ .step-header { display: flex; align-items: center; justify-content: space-between; cursor: pointer; user-select: none; }
22
+ .step-header:hover { opacity: 0.85; }
23
+ .step-name { font-weight: 600; font-size: 1.05rem; }
24
+ .step-icon { font-size: 1.1rem; margin-right: 0.5rem; }
25
+ .step-rate { font-size: 0.875rem; color: #94a3b8; }
26
+ .bar-container { background: #334155; border-radius: 4px; height: 8px; margin: 0.75rem 0; overflow: hidden; }
27
+ .bar-fill { height: 100%; border-radius: 4px; transition: width 0.3s ease; }
28
+ .bar-pass { background: ${passColor}; }
29
+ .bar-fail { background: ${failColor}; }
30
+ .step-stats { display: flex; gap: 1.5rem; font-size: 0.8rem; color: #94a3b8; }
31
+ .iterations { display: none; margin-top: 1rem; border-top: 1px solid #334155; padding-top: 0.75rem; }
32
+ .iterations.open { display: block; }
33
+ .iter-row { display: flex; align-items: center; gap: 0.75rem; padding: 0.4rem 0; font-size: 0.8rem; border-bottom: 1px solid #1e293b; }
34
+ .iter-row:last-child { border-bottom: none; }
35
+ .iter-badge { display: inline-block; width: 18px; height: 18px; border-radius: 50%; text-align: center; line-height: 18px; font-size: 0.65rem; font-weight: 700; color: #fff; flex-shrink: 0; }
36
+ .iter-pass { background: ${passColor}; }
37
+ .iter-fail { background: ${failColor}; }
38
+ .iter-assertions { color: #94a3b8; flex: 1; }
39
+ .iter-duration { color: #64748b; white-space: nowrap; }
40
+ .iter-output { margin-top: 0.25rem; font-family: 'SF Mono', 'Fira Code', monospace; font-size: 0.75rem; color: #cbd5e1; background: #0f172a; padding: 0.5rem; border-radius: 4px; max-height: 120px; overflow-y: auto; white-space: pre-wrap; word-break: break-word; display: none; }
41
+ .iter-output.open { display: block; }
42
+ .iter-toggle { cursor: pointer; color: #60a5fa; font-size: 0.75rem; }
43
+ .iter-toggle:hover { text-decoration: underline; }
44
+ .assertion-line { font-size: 0.75rem; padding-left: 1.5rem; }
45
+ .assertion-pass { color: ${passColor}; }
46
+ .assertion-fail { color: ${failColor}; }
47
+ .summary-bar { display: flex; gap: 1.5rem; margin-top: 1rem; font-size: 0.8rem; color: #94a3b8; }
48
+ .chevron { transition: transform 0.2s; display: inline-block; }
49
+ .chevron.open { transform: rotate(90deg); }
50
+ @media (max-width: 640px) { body { padding: 1rem; } .step-stats { flex-wrap: wrap; gap: 0.75rem; } }
51
+ </style>
52
+ </head>
53
+ <body>
54
+
55
+ <h1>${escHtml(report.scenarioName)}</h1>
56
+ <div class="meta">
57
+ <span>${new Date(report.startedAt).toLocaleString()}</span>
58
+ <span>${formatDuration(report.durationMs)}</span>
59
+ <span>${report.iterations} iteration${report.iterations === 1 ? '' : 's'}</span>
60
+ </div>
61
+ <div class="verdict">${verdictText}</div>
62
+
63
+ ${report.steps.map((step, i) => renderStep(step, report.results.filter(r => r.stepId === step.stepId), i)).join('\n')}
64
+
65
+ <div class="summary-bar">
66
+ <span>${report.steps.filter(s => !s.belowThreshold).length}/${report.steps.length} steps passed</span>
67
+ <span>Total: ${formatDuration(report.durationMs)}</span>
68
+ </div>
69
+
70
+ <script>
71
+ const REPORT = ${reportJson};
72
+
73
+ function toggleIterations(idx) {
74
+ const el = document.getElementById('iters-' + idx);
75
+ const chev = document.getElementById('chev-' + idx);
76
+ if (el) { el.classList.toggle('open'); }
77
+ if (chev) { chev.classList.toggle('open'); }
78
+ }
79
+
80
+ function toggleOutput(id) {
81
+ const el = document.getElementById(id);
82
+ if (el) { el.classList.toggle('open'); }
83
+ }
84
+ </script>
85
+ </body>
86
+ </html>`;
87
+ }
88
+ function renderStep(step, results, index) {
89
+ const pct = (step.passRate * 100).toFixed(1);
90
+ const threshPct = (step.minPassRate * 100).toFixed(0);
91
+ const icon = step.belowThreshold ? '&#10005;' : '&#10003;';
92
+ const iconColor = step.belowThreshold ? '#ef4444' : '#22c55e';
93
+ const barColor = step.belowThreshold ? 'bar-fail' : 'bar-pass';
94
+ const avgDur = step.avgDurationMs ? formatDuration(step.avgDurationMs) : '';
95
+ const costStr = step.totalCostUsd > 0 ? `$${step.totalCostUsd.toFixed(4)} total / $${step.avgCostUsd.toFixed(4)} avg` : '';
96
+ const iterRows = results.map((r, i) => {
97
+ const badge = r.passed ? 'iter-pass' : 'iter-fail';
98
+ const sym = r.passed ? '&#10003;' : '&#10005;';
99
+ const iterCost = r.costUsd != null && r.costUsd > 0 ? ` &middot; $${r.costUsd.toFixed(4)}` : '';
100
+ const assertions = r.assertionResults.map(a => {
101
+ const cls = a.passed ? 'assertion-pass' : 'assertion-fail';
102
+ const asym = a.passed ? '&#10003;' : '&#10005;';
103
+ return `<div class="assertion-line ${cls}">${asym} ${escHtml(a.type)}${a.message ? ': ' + escHtml(a.message) : ''}</div>`;
104
+ }).join('');
105
+ const outputId = `output-${index}-${i}`;
106
+ return `<div>
107
+ <div class="iter-row">
108
+ <span class="iter-badge ${badge}">${sym}</span>
109
+ <span>Iteration ${r.iteration}</span>
110
+ <span class="iter-duration">${formatDuration(r.durationMs)}${iterCost}</span>
111
+ <span class="iter-toggle" onclick="toggleOutput('${outputId}')">output</span>
112
+ </div>
113
+ ${assertions}
114
+ <div class="iter-output" id="${outputId}">${escHtml(r.output || '(no output)')}</div>
115
+ </div>`;
116
+ }).join('');
117
+ return `<div class="card">
118
+ <div class="step-header" onclick="toggleIterations(${index})">
119
+ <div>
120
+ <span class="step-icon" style="color:${iconColor}">${icon}</span>
121
+ <span class="step-name">${escHtml(step.stepId)}</span>
122
+ </div>
123
+ <div style="display:flex;align-items:center;gap:0.75rem;">
124
+ <span class="step-rate">${pct}% (threshold: ${threshPct}%)</span>
125
+ <span class="chevron" id="chev-${index}">&#9654;</span>
126
+ </div>
127
+ </div>
128
+ <div class="bar-container"><div class="bar-fill ${barColor}" style="width:${pct}%"></div></div>
129
+ <div class="step-stats">
130
+ <span>${step.passes}/${step.totalRuns} passed</span>
131
+ <span>${step.failures} failed</span>
132
+ ${avgDur ? `<span>avg ${avgDur}</span>` : ''}
133
+ ${costStr ? `<span>${costStr}</span>` : ''}
134
+ </div>
135
+ <div class="iterations" id="iters-${index}">
136
+ ${iterRows}
137
+ </div>
138
+ </div>`;
139
+ }
140
+ function formatDuration(ms) {
141
+ if (ms < 1000)
142
+ return `${ms}ms`;
143
+ if (ms < 60_000)
144
+ return `${(ms / 1000).toFixed(1)}s`;
145
+ const mins = Math.floor(ms / 60_000);
146
+ const secs = ((ms % 60_000) / 1000).toFixed(0);
147
+ return `${mins}m ${secs}s`;
148
+ }
149
+ function escHtml(s) {
150
+ return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
151
+ }
152
+ //# sourceMappingURL=html-reporter.js.map
package/dist/reporters/html-reporter.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"html-reporter.js","sourceRoot":"","sources":["../../src/reporters/html-reporter.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,kBAAkB,CAAC,MAAsB;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC;IAC5B,MAAM,SAAS,GAAG,SAAS,CAAC;IAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE3D,OAAO;;;;;qBAKY,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC;;;;;;;uIAOsF,YAAY,cAAc,YAAY,uBAAuB,YAAY;;;;;;;;;4BASpL,SAAS;4BACT,SAAS;;;;;;;6BAOR,SAAS;6BACT,SAAS;;;;;;;;6BAQT,SAAS;6BACT,SAAS;;;;;;;;;MAShC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC;;UAExB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE;UAC3C,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC;UACjC,MAAM,CAAC,UAAU,aAAa,MAAM,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG;;uBAEnD,WAAW;;EAEhC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;UAG3G,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM;iBAClE,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC;;;;iBAIjC,UAAU;;;;;;;;;;;;;;;QAenB,CAAC;AACT,CAAC;AAED,SAAS,UAAU,CAAC,IAAiB,EAAE,OAAqB,EAAE,KAAa;IACzE,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;IAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3H,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC;QACnD,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/C,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,UAAU,GAAG,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;YAC3D,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;YAChD,OAAO,8BAA8B,GAAG,KAAK,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC;QAC5H,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,MAAM,QAAQ,GAAG,UAAU,KAAK,IAAI,CAAC,EAAE,CAAC;QACxC,OAAO;;kCAEuB,KAAK,KAAK,GAAG;0BACrB,CAAC,CAAC,SAAS;sCACC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,QAAQ;2DAClB,QAAQ;;QAE3D,UAAU;qCACmB,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,aAAa,CAAC;WACzE,CAAC;IACV,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEZ,OAAO;uDAC8C,KAAK;;6CAEf,SAAS,KAAK,IAAI;gCAC/B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;;;gCAGpB,GAAG,iBAAiB,SAAS;uCACtB,KAAK;;;oDAGQ,QAAQ,kBAAkB,GAAG;;YAErE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAC7B,IAAI,CAAC,QAAQ;MACnB,MAAM,CAAC,CAAC,CAAC,aAAa,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;MAC1C,OAAO,CAAC,CAAC,CAAC,SAAS,OAAO,SAAS,CAAC,CAAC,CAAC,EAAE;;sCAER,KAAK;MACrC,QAAQ;;OAEP,CAAC;AACR,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,MAAM;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7B,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AACtG,CAAC"}
package/dist/reporters/terminal-reporter.d.ts CHANGED
@@ -1,4 +1,13 @@
1
1
  import type { ScenarioReport } from '../core/types.js';
2
- export declare function printReport(report: ScenarioReport, reportPath?: string): void;
2
+ import type { BaselineComparison } from '../baseline.js';
3
+ import type { CacheStats } from '../core/scenario-runner.js';
4
+ import type { ComparisonReport } from '../commands/compare.js';
5
+ export interface PrintReportOptions {
6
+ reportPath?: string;
7
+ baselineComparison?: BaselineComparison;
8
+ cacheStats?: CacheStats;
9
+ }
10
+ export declare function printReport(report: ScenarioReport, reportPathOrOpts?: string | PrintReportOptions): void;
3
11
  export declare function printProgress(stepId: string, iteration: number, total: number): void;
12
+ export declare function formatComparison(report: ComparisonReport): string;
4
13
  //# sourceMappingURL=terminal-reporter.d.ts.map
package/dist/reporters/terminal-reporter.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"terminal-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAe,MAAM,kBAAkB,CAAC;AAKpE,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAwB7E;AAsDD,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAKpF"}
1
+ {"version":3,"file":"terminal-reporter.d.ts","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAe,MAAM,kBAAkB,CAAC;AACpE,OAAO,KAAK,EAAE,kBAAkB,EAAkB,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAK/D,MAAM,WAAW,kBAAkB;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,gBAAgB,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAkDxG;AAmFD,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAKpF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA6DjE"}
package/dist/reporters/terminal-reporter.js CHANGED
@@ -2,35 +2,77 @@ import chalk from 'chalk';
2
2
  import { createRequire } from 'node:module';
3
3
  const require = createRequire(import.meta.url);
4
4
  const { version } = require('../../package.json');
5
- export function printReport(report, reportPath) {
5
+ export function printReport(report, reportPathOrOpts) {
6
6
  const { scenarioName, iterations, steps, allPassed, durationMs } = report;
7
+ // Normalize overloaded param
8
+ const opts = typeof reportPathOrOpts === 'string'
9
+ ? { reportPath: reportPathOrOpts }
10
+ : reportPathOrOpts ?? {};
11
+ const baselineMap = new Map();
12
+ if (opts.baselineComparison) {
13
+ for (const s of opts.baselineComparison.steps) {
14
+ baselineMap.set(s.stepId, s);
15
+ }
16
+ }
7
17
  console.log('');
8
18
  console.log(chalk.bold('stepproof') + chalk.dim(` v${version}`));
9
19
  console.log(chalk.dim('─'.repeat(50)));
10
20
  console.log(`${chalk.bold('Scenario:')} ${scenarioName}`);
11
21
  console.log(`${chalk.bold('Iterations:')} ${iterations}`);
12
22
  console.log(`${chalk.bold('Duration:')} ${formatDuration(durationMs)}`);
23
+ if (opts.cacheStats && (opts.cacheStats.hits > 0 || opts.cacheStats.misses > 0)) {
24
+ console.log(`${chalk.bold('Cache:')} ${opts.cacheStats.hits} hits, ${opts.cacheStats.misses} misses`);
25
+ }
13
26
  console.log('');
14
27
  for (const step of steps) {
15
- printStepSummary(step, iterations);
28
+ printStepSummary(step, iterations, baselineMap.get(step.stepId));
16
29
  }
17
30
  console.log(chalk.dim('─'.repeat(50)));
18
31
  printSummaryLine(allPassed, steps);
19
- if (reportPath) {
32
+ if (opts.baselineComparison?.hasRegression) {
33
+ const regSteps = opts.baselineComparison.steps.filter(s => s.regression);
34
+ console.log(chalk.bold.yellow('REGRESSION DETECTED') +
35
+ chalk.dim(` — ${regSteps.length} step${regSteps.length === 1 ? '' : 's'} dropped: `) +
36
+ regSteps.map(s => chalk.yellow(s.stepId)).join(', '));
37
+ }
38
+ if (opts.reportPath) {
20
39
  console.log('');
21
- console.log(chalk.dim(`Report written to: ${reportPath}`));
40
+ console.log(chalk.dim(`Report written to: ${opts.reportPath}`));
22
41
  }
23
42
  console.log('');
24
43
  }
25
- function printStepSummary(step, iterations) {
44
+ function printStepSummary(step, iterations, baseline) {
26
45
  const { stepId, passes, totalRuns, passRate, minPassRate, belowThreshold, failures } = step;
27
46
  const statusIcon = belowThreshold ? chalk.red('✗') : chalk.green('✓');
28
47
  const rateColor = belowThreshold ? chalk.red : chalk.green;
29
48
  const pct = (passRate * 100).toFixed(1);
30
49
  const threshold = (minPassRate * 100).toFixed(0);
31
- console.log(` ${statusIcon} ${chalk.bold(stepId)}`);
50
+ // Baseline delta indicator
51
+ let baselineStr = '';
52
+ if (baseline) {
53
+ const bPct = (baseline.baselineRate * 100).toFixed(1);
54
+ const deltaPct = (baseline.delta * 100).toFixed(1);
55
+ if (baseline.delta > 0) {
56
+ baselineStr = chalk.green(` ↑ +${deltaPct}%`) + chalk.dim(` (baseline: ${bPct}%)`);
57
+ }
58
+ else if (baseline.delta < 0) {
59
+ baselineStr = chalk.red(` ↓ ${deltaPct}%`) + chalk.dim(` (baseline: ${bPct}%)`);
60
+ }
61
+ else {
62
+ baselineStr = chalk.dim(` = (baseline: ${bPct}%)`);
63
+ }
64
+ }
65
+ // Cost/latency metrics
66
+ const avgSec = (step.avgDurationMs / 1000).toFixed(1);
67
+ const avgCost = formatCost(step.avgCostUsd);
68
+ const totalCost = formatCost(step.totalCostUsd);
69
+ const hasCost = step.totalCostUsd > 0;
70
+ const metricsSuffix = hasCost
71
+ ? chalk.dim(` — avg ${avgSec}s, ${avgCost}/call, total ${totalCost}`)
72
+ : chalk.dim(` — avg ${avgSec}s`);
73
+ console.log(` ${statusIcon} ${chalk.bold(stepId)}${metricsSuffix}`);
32
74
  console.log(` ${renderBar(passes, totalRuns)} ${passes}/${totalRuns} iterations`);
33
- console.log(` Pass rate: ${rateColor(`${pct}%`)} ${chalk.dim(`(threshold: ${threshold}%)`)}`);
75
+ console.log(` Pass rate: ${rateColor(`${pct}%`)} ${chalk.dim(`(threshold: ${threshold}%)`)}${baselineStr}`);
34
76
  if (belowThreshold) {
35
77
  console.log(` ${chalk.red(`✗ BELOW THRESHOLD — ${failures} failure${failures === 1 ? '' : 's'}`)}`);
36
78
  }
@@ -64,10 +106,72 @@ function formatDuration(ms) {
64
106
  const secs = ((ms % 60_000) / 1000).toFixed(0);
65
107
  return `${mins}m ${secs}s`;
66
108
  }
109
+ function formatCost(usd) {
110
+ if (usd === 0)
111
+ return '$0.00';
112
+ if (usd < 0.01)
113
+ return `$${usd.toFixed(4)}`;
114
+ return `$${usd.toFixed(2)}`;
115
+ }
67
116
  export function printProgress(stepId, iteration, total) {
68
117
  process.stdout.write(`\r ${chalk.dim('Running')} ${stepId} — iteration ${iteration}/${total}...`);
69
118
  if (iteration === total) {
70
119
  process.stdout.write('\r' + ' '.repeat(60) + '\r');
71
120
  }
72
121
  }
122
+ export function formatComparison(report) {
123
+ const lines = [];
124
+ lines.push('');
125
+ lines.push(chalk.bold('stepproof compare') + chalk.dim(` v${version}`));
126
+ lines.push(chalk.dim('─'.repeat(60)));
127
+ lines.push(`${chalk.bold('Scenario:')} ${report.scenarioName}`);
128
+ lines.push(`${chalk.bold('Iterations:')} ${report.iterations}`);
129
+ lines.push(`${chalk.bold('Duration:')} ${formatDuration(report.durationMs)}`);
130
+ lines.push(`${chalk.bold('Providers:')} ${report.providers.map(p => `${p.provider}:${p.model}`).join(', ')}`);
131
+ lines.push('');
132
+ // Header row
133
+ const providerLabels = report.providers.map(p => `${p.provider}:${p.model}`);
134
+ const colWidth = Math.max(20, ...providerLabels.map(l => l.length + 2));
135
+ const stepColWidth = Math.max(12, ...report.stepBreakdown.map(s => s.stepId.length + 2));
136
+ const headerPad = ' '.repeat(stepColWidth);
137
+ lines.push(` ${headerPad}${providerLabels.map(l => l.padEnd(colWidth)).join('')}`);
138
+ lines.push(` ${chalk.dim('─'.repeat(stepColWidth + colWidth * providerLabels.length))}`);
139
+ // Step rows
140
+ for (const step of report.stepBreakdown) {
141
+ const stepLabel = step.stepId.padEnd(stepColWidth);
142
+ const cells = step.rates.map(r => {
143
+ const pct = (r.passRate * 100).toFixed(1) + '%';
144
+ const isBest = r.provider === step.bestProvider && r.model === step.bestModel;
145
+ const display = `${r.passes}/${r.totalRuns} (${pct})`;
146
+ const colored = isBest ? chalk.green(display) : display;
147
+ return colored.padEnd(colWidth + (isBest ? colored.length - display.length : 0));
148
+ });
149
+ // padEnd doesn't work well with chalk, so we use a simpler approach
150
+ const rawCells = step.rates.map(r => {
151
+ const pct = (r.passRate * 100).toFixed(1) + '%';
152
+ const isBest = r.provider === step.bestProvider && r.model === step.bestModel;
153
+ const display = `${r.passes}/${r.totalRuns} (${pct})`;
154
+ const padding = ' '.repeat(Math.max(0, colWidth - display.length));
155
+ return (isBest ? chalk.green(display) : display) + padding;
156
+ });
157
+ lines.push(` ${stepLabel}${rawCells.join('')}`);
158
+ }
159
+ lines.push('');
160
+ lines.push(chalk.dim('─'.repeat(60)));
161
+ // Overall averages
162
+ const avgLines = report.providers.map(p => {
163
+ const avg = p.report.steps.reduce((sum, s) => sum + s.passRate, 0) / p.report.steps.length;
164
+ return { label: `${p.provider}:${p.model}`, avg };
165
+ });
166
+ for (const a of avgLines) {
167
+ const isWinner = a.label === `${report.winner}:${report.winnerModel}`;
168
+ const pct = (a.avg * 100).toFixed(1) + '%';
169
+ const line = ` ${a.label}: ${pct}${isWinner ? ' ★' : ''}`;
170
+ lines.push(isWinner ? chalk.bold.green(line) : line);
171
+ }
172
+ lines.push('');
173
+ lines.push(chalk.bold('Winner: ') + chalk.green(`${report.winner}:${report.winnerModel}`));
174
+ lines.push('');
175
+ return lines.join('\n');
176
+ }
73
177
  //# sourceMappingURL=terminal-reporter.js.map
package/dist/reporters/terminal-reporter.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"terminal-reporter.js","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAwB,CAAC;AAEzE,MAAM,UAAU,WAAW,CAAC,MAAsB,EAAE,UAAmB;IACrE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAE1E,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAEnC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAiB,EAAE,UAAkB;IAC7D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAE5F,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEjD,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,IAAI,SAAS,aAAa,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CACT,kBAAkB,SAAS,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,IAAI,CAAC,EAAE,CACrF,CAAC;IAEF,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,uBAAuB,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAa;IAC9C,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,QAAQ,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,OAAO,IAAI,GAAG,GAAG,CAAC;AACpB,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAkB,EAAE,KAAoB;IAChE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAEtD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oBAAoB,CAAC;YAC1F,QAAQ,CACT,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,MAAM;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,SAAiB,EAAE,KAAa;IAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,MAAM,gBAAgB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;IACnG,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"terminal-reporter.js","sourceRoot":"","sources":["../../src/reporters/terminal-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAM5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAwB,CAAC;AAQzE,MAAM,UAAU,WAAW,CAAC,MAAsB,EAAE,gBAA8C;IAChG,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAE1E,6BAA6B;IAC7B,MAAM,IAAI,GAAuB,OAAO,gBAAgB,KAAK,QAAQ;QACnE,CAAC,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE;QAClC,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IACtD,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;YAC9C,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAExE,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAEnC,IAAI,IAAI,CAAC,kBAAkB,EAAE,aAAa,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC;YACxC,KAAK,CAAC,GAAG,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC;YACpF,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CACrD,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAiB,EAAE,UAAkB,EAAE,QAAyB;IACxF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAE5F,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEjD,2BAA2B;IAC3B,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACvB,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC;QACrF,CAAC;aAAM,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9B,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC;QAClF,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IACtC,MAAM,aAAa,GAAG,OAAO;QAC3B,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,MAAM,MAAM,OAAO,gBAAgB,SAAS,EAAE,CAAC;QACrE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,MAAM,GAAG,CAAC,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,aAAa,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,IAAI,SAAS,aAAa,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CACT,kBAAkB,SAAS,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,IAAI,CAAC,GAAG,WAAW,EAAE,CACnG,CAAC;IAEF,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,uBAAuB,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAa;IAC9C,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,QAAQ,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,OAAO,IAAI,GAAG,GAAG,CAAC;AACpB,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAkB,EAAE,KAAoB;IAChE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAEtD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oBAAoB,CAAC;YAC1F,QAAQ,CACT,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,MAAM;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7B,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9B,IAAI,GAAG,GAAG,IAAI;QAAE,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,SAAiB,EAAE,KAAa;IAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,MAAM,gBAAgB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;IACnG,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAwB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,aAAa;IACb,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAEzF,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,GAAG,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;IAE1F,YAAY;IACZ,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC/B,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YAChD,MAAM,MAAM,GAAG,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC;YAC9E,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,KAAK,GAAG,GAAG,CAAC;YACtD,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YACxD,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QACH,oEAAoE;QACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAClC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YAChD,MAAM,MAAM,GAAG,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC;YAC9E,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,KAAK,GAAG,GAAG,CAAC;YACtD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC7D,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtC,mBAAmB;IACnB,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QACxC,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QAC3F,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC;IACpD,CAAC,CAAC,CAAC;IACH,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,KAAK,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACtE,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC3F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "stepproof",
3
- "version": "0.2.22",
3
+ "version": "0.3.0",
4
4
  "description": "Regression testing for multi-step AI workflows. Not observability — a CI gate.",
5
5
  "main": "dist/cli.js",
6
6
  "bin": {
@@ -46,6 +46,7 @@
46
46
  "dependencies": {
47
47
  "@anthropic-ai/sdk": "^0.37.0",
48
48
  "@bilkobibitkov/preflight-license": "^1.0.0",
49
+ "@google/generative-ai": "^0.24.1",
49
50
  "ajv": "^8.17.1",
50
51
  "chalk": "^5.3.0",
51
52
  "commander": "^12.1.0",