stepproof 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -1
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +22 -2
- package/dist/adapters/anthropic.js.map +1 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +21 -4
- package/dist/adapters/openai.js.map +1 -1
- package/dist/cli.js +154 -3
- package/dist/cli.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +8 -5
- package/dist/commands/init.js.map +1 -1
- package/dist/core/scenario-parser.js +1 -1
- package/dist/core/scenario-parser.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# stepproof
|
|
2
2
|
|
|
3
|
-
[](https://
|
|
3
|
+
[](https://www.npmjs.com/package/@bilkobibitkov/agent-gate)
|
|
4
4
|
[]()
|
|
5
5
|
[]()
|
|
6
6
|
|
|
@@ -231,6 +231,41 @@ These are different jobs. Use both.
|
|
|
231
231
|
|
|
232
232
|
---
|
|
233
233
|
|
|
234
|
+
## Troubleshooting
|
|
235
|
+
|
|
236
|
+
### `Error: "scenarios/" is a directory`
|
|
237
|
+
```
|
|
238
|
+
stepproof run ./scenarios/first-test.yaml # ← run a specific file
|
|
239
|
+
stepproof run ./scenarios/ # ← or run the whole dir (note trailing slash)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
### `Error parsing scenario: ...`
|
|
243
|
+
Your YAML has a syntax error. Common culprits: inconsistent indentation, unquoted `{{vars}}`, or a missing `steps:` key. Run `node -e "require('fs').readFileSync('./your.yaml')"` to catch basic issues.
|
|
244
|
+
|
|
245
|
+
### API errors (`401 Unauthorized`, `403 Forbidden`)
|
|
246
|
+
Set the API key for whichever provider your scenario uses:
|
|
247
|
+
```bash
|
|
248
|
+
export ANTHROPIC_API_KEY=sk-ant-...
|
|
249
|
+
export OPENAI_API_KEY=sk-...
|
|
250
|
+
```
|
|
251
|
+
Only the keys for providers you use in your scenarios are required.
|
|
252
|
+
|
|
253
|
+
### Steps failing when they should pass
|
|
254
|
+
Check `min_pass_rate` in your scenario. The default is not 100% — if you set `min_pass_rate: 0.90` you expect 1-in-10 to fail. Lower it, or improve your prompt.
|
|
255
|
+
|
|
256
|
+
### `--format must be "sarif" or "junit"`
|
|
257
|
+
Only `sarif` and `junit` are valid format values. For terminal output, omit the `--format` flag entirely.
|
|
258
|
+
|
|
259
|
+
### Pro features blocked (SARIF / JUnit output)
|
|
260
|
+
SARIF and JUnit formats require a Team license. Set your key:
|
|
261
|
+
```bash
|
|
262
|
+
export PREFLIGHT_LICENSE_KEY=preflight_...
|
|
263
|
+
stepproof run scenarios/ --format sarif --output results.sarif
|
|
264
|
+
```
|
|
265
|
+
Get a license at the [Preflight pricing page](https://stanislavbg.github.io/preflight/).
|
|
266
|
+
|
|
267
|
+
---
|
|
268
|
+
|
|
234
269
|
## Scenarios
|
|
235
270
|
|
|
236
271
|
See [`/examples`](./examples) for copy-paste ready scenarios:
|
|
@@ -274,3 +309,11 @@ npm install -g agent-gate stepproof agent-comply agent-shift agent-trace
|
|
|
274
309
|
---
|
|
275
310
|
|
|
276
311
|
*stepproof — because "I checked manually before the deploy" is not a test.*
|
|
312
|
+
|
|
313
|
+
---
|
|
314
|
+
|
|
315
|
+
## Legal
|
|
316
|
+
|
|
317
|
+
- [Privacy Policy](https://stanislavbg.github.io/preflight/privacy.html)
|
|
318
|
+
- [Terms of Service](https://stanislavbg.github.io/preflight/terms.html)
|
|
319
|
+
- Contact: [bilko@bglabs.app](mailto:bilko@bglabs.app)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../src/adapters/anthropic.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../src/adapters/anthropic.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAsBjD,qBAAa,gBAAiB,YAAW,eAAe;IACtD,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,KAAK,CAAS;gBAEV,KAAK,EAAE,MAAM;IAQnB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAiB7D"}
|
|
@@ -1,4 +1,24 @@
|
|
|
1
1
|
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
const MAX_RETRIES = 3;
|
|
3
|
+
const BASE_DELAY_MS = 1000;
|
|
4
|
+
async function withRetry(fn) {
|
|
5
|
+
let lastError;
|
|
6
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
7
|
+
try {
|
|
8
|
+
return await fn();
|
|
9
|
+
}
|
|
10
|
+
catch (err) {
|
|
11
|
+
lastError = err;
|
|
12
|
+
const status = err.status;
|
|
13
|
+
// Only retry on rate limit (429) or server error (5xx)
|
|
14
|
+
if (status !== 429 && !(status && status >= 500))
|
|
15
|
+
throw err;
|
|
16
|
+
const delay = BASE_DELAY_MS * Math.pow(2, attempt);
|
|
17
|
+
await new Promise((res) => setTimeout(res, delay));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
throw lastError;
|
|
21
|
+
}
|
|
2
22
|
export class AnthropicAdapter {
|
|
3
23
|
client;
|
|
4
24
|
model;
|
|
@@ -10,12 +30,12 @@ export class AnthropicAdapter {
|
|
|
10
30
|
this.client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
|
|
11
31
|
}
|
|
12
32
|
async call(prompt, system) {
|
|
13
|
-
const response = await this.client.messages.create({
|
|
33
|
+
const response = await withRetry(() => this.client.messages.create({
|
|
14
34
|
model: this.model,
|
|
15
35
|
max_tokens: 1024,
|
|
16
36
|
...(system && { system }),
|
|
17
37
|
messages: [{ role: 'user', content: prompt }],
|
|
18
|
-
});
|
|
38
|
+
}));
|
|
19
39
|
const content = response.content[0];
|
|
20
40
|
if (content?.type === 'text') {
|
|
21
41
|
return content.text;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../src/adapters/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAG1C,MAAM,OAAO,gBAAgB;IACnB,MAAM,CAAY;IAClB,KAAK,CAAS;IAEtB,YAAY,KAAa;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAe;QACxC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../src/adapters/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAG1C,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,KAAK,UAAU,SAAS,CAAI,EAAoB;IAC9C,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,SAAS,GAAG,GAAG,CAAC;YAChB,MAAM,MAAM,GAAI,GAA2B,CAAC,MAAM,CAAC;YACnD,uDAAuD;YACvD,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,IAAI,GAAG,CAAC;gBAAE,MAAM,GAAG,CAAC;YAC5D,MAAM,KAAK,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,gBAAgB;IACnB,MAAM,CAAY;IAClB,KAAK,CAAS;IAEtB,YAAY,KAAa;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAe;QACxC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI;YAChB,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC;YACzB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C,CAAC,CACH,CAAC;QAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,OAAO,EAAE,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/adapters/openai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/adapters/openai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAsBjD,qBAAa,aAAc,YAAW,eAAe;IACnD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,KAAK,EAAE,MAAM;IAQnB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAc7D"}
|
package/dist/adapters/openai.js
CHANGED
|
@@ -1,4 +1,24 @@
|
|
|
1
1
|
import OpenAI from 'openai';
|
|
2
|
+
const MAX_RETRIES = 3;
|
|
3
|
+
const BASE_DELAY_MS = 1000;
|
|
4
|
+
async function withRetry(fn) {
|
|
5
|
+
let lastError;
|
|
6
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
7
|
+
try {
|
|
8
|
+
return await fn();
|
|
9
|
+
}
|
|
10
|
+
catch (err) {
|
|
11
|
+
lastError = err;
|
|
12
|
+
const status = err.status;
|
|
13
|
+
// Only retry on rate limit (429) or server error (5xx)
|
|
14
|
+
if (status !== 429 && !(status && status >= 500))
|
|
15
|
+
throw err;
|
|
16
|
+
const delay = BASE_DELAY_MS * Math.pow(2, attempt);
|
|
17
|
+
await new Promise((res) => setTimeout(res, delay));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
throw lastError;
|
|
21
|
+
}
|
|
2
22
|
export class OpenAIAdapter {
|
|
3
23
|
client;
|
|
4
24
|
model;
|
|
@@ -15,10 +35,7 @@ export class OpenAIAdapter {
|
|
|
15
35
|
messages.push({ role: 'system', content: system });
|
|
16
36
|
}
|
|
17
37
|
messages.push({ role: 'user', content: prompt });
|
|
18
|
-
const response = await this.client.chat.completions.create({
|
|
19
|
-
model: this.model,
|
|
20
|
-
messages,
|
|
21
|
-
});
|
|
38
|
+
const response = await withRetry(() => this.client.chat.completions.create({ model: this.model, messages }));
|
|
22
39
|
return response.choices[0]?.message?.content ?? '';
|
|
23
40
|
}
|
|
24
41
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/adapters/openai.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,KAAa;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAe;QACxC,MAAM,QAAQ,GAA6C,EAAE,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/adapters/openai.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,KAAK,UAAU,SAAS,CAAI,EAAoB;IAC9C,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,SAAS,GAAG,GAAG,CAAC;YAChB,MAAM,MAAM,GAAI,GAA2B,CAAC,MAAM,CAAC;YACnD,uDAAuD;YACvD,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,IAAI,GAAG,CAAC;gBAAE,MAAM,GAAG,CAAC;YAC5D,MAAM,KAAK,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,KAAa;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAe;QACxC,MAAM,QAAQ,GAA6C,EAAE,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEjD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CACrE,CAAC;QAEF,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IACrD,CAAC;CACF"}
|
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { Command } from 'commander';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
+
import * as os from 'node:os';
|
|
4
5
|
import { parseScenario } from './core/scenario-parser.js';
|
|
5
6
|
import { runScenario } from './core/scenario-runner.js';
|
|
6
7
|
import { writeJsonReport } from './reporters/json-reporter.js';
|
|
@@ -8,13 +9,96 @@ import { printReport } from './reporters/terminal-reporter.js';
|
|
|
8
9
|
import { formatSarif } from './reporters/sarif-reporter.js';
|
|
9
10
|
import { formatJunit } from './reporters/junit-reporter.js';
|
|
10
11
|
import * as fs from 'node:fs';
|
|
11
|
-
import { guard } from '@bilkobibitkov/preflight-license';
|
|
12
|
+
import { guard, validate } from '@bilkobibitkov/preflight-license';
|
|
12
13
|
import { runInit } from './commands/init.js';
|
|
14
|
+
/* ── Usage-based monetization ───────────────────────────────────────── */
|
|
15
|
+
const FREE_MONTHLY_LIMIT = 10;
|
|
16
|
+
const UPGRADE_URL = 'https://buy.stripe.com/3cIbJ3fA8am122VcwE8k804';
|
|
17
|
+
const CONFIG_DIR = path.join(os.homedir(), '.config', 'stepproof');
|
|
18
|
+
const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
|
|
19
|
+
const USAGE_FILE = path.join(CONFIG_DIR, 'usage.json');
|
|
20
|
+
/** Read license key from STEPPROOF_KEY env var or ~/.config/stepproof/config.json */
|
|
21
|
+
function getStepproofKey() {
|
|
22
|
+
const envKey = process.env.STEPPROOF_KEY;
|
|
23
|
+
if (envKey && envKey.trim())
|
|
24
|
+
return envKey.trim();
|
|
25
|
+
try {
|
|
26
|
+
if (fs.existsSync(CONFIG_FILE)) {
|
|
27
|
+
const raw = fs.readFileSync(CONFIG_FILE, 'utf8');
|
|
28
|
+
const parsed = JSON.parse(raw);
|
|
29
|
+
if (parsed.key && parsed.key.trim())
|
|
30
|
+
return parsed.key.trim();
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// Corrupted config — ignore
|
|
35
|
+
}
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
/** Check if user has a valid pro license */
|
|
39
|
+
function isProUser() {
|
|
40
|
+
const key = getStepproofKey();
|
|
41
|
+
if (!key)
|
|
42
|
+
return false;
|
|
43
|
+
const result = validate(key);
|
|
44
|
+
return result.valid && result.tier !== 'free';
|
|
45
|
+
}
|
|
46
|
+
/** Read current month's usage */
|
|
47
|
+
function readUsage() {
|
|
48
|
+
const currentMonth = new Date().toISOString().slice(0, 7); // YYYY-MM
|
|
49
|
+
try {
|
|
50
|
+
if (fs.existsSync(USAGE_FILE)) {
|
|
51
|
+
const raw = fs.readFileSync(USAGE_FILE, 'utf8');
|
|
52
|
+
const parsed = JSON.parse(raw);
|
|
53
|
+
if (parsed.month === currentMonth)
|
|
54
|
+
return parsed;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Corrupted — reset
|
|
59
|
+
}
|
|
60
|
+
return { month: currentMonth, count: 0 };
|
|
61
|
+
}
|
|
62
|
+
/** Write usage record to disk */
|
|
63
|
+
function writeUsage(record) {
|
|
64
|
+
try {
|
|
65
|
+
fs.mkdirSync(CONFIG_DIR, { recursive: true });
|
|
66
|
+
fs.writeFileSync(USAGE_FILE, JSON.stringify(record), 'utf8');
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// Can't write — degrade gracefully
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/** Check free limit before a run. Returns true if allowed, false if blocked. */
|
|
73
|
+
function checkUsageLimit() {
|
|
74
|
+
if (isProUser())
|
|
75
|
+
return true;
|
|
76
|
+
const usage = readUsage();
|
|
77
|
+
if (usage.count >= FREE_MONTHLY_LIMIT) {
|
|
78
|
+
process.stderr.write(`\nYou've used ${FREE_MONTHLY_LIMIT}/${FREE_MONTHLY_LIMIT} free runs this month. ` +
|
|
79
|
+
`Upgrade to Stepproof Pro ($19/mo) for unlimited runs → ${UPGRADE_URL}\n\n`);
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
/** Increment usage after a successful free run and show remaining count */
|
|
85
|
+
function trackUsageAfterRun() {
|
|
86
|
+
if (isProUser())
|
|
87
|
+
return;
|
|
88
|
+
const usage = readUsage();
|
|
89
|
+
usage.count += 1;
|
|
90
|
+
writeUsage(usage);
|
|
91
|
+
const remaining = FREE_MONTHLY_LIMIT - usage.count;
|
|
92
|
+
process.stderr.write(`\n${remaining}/${FREE_MONTHLY_LIMIT} free runs remaining this month.` +
|
|
93
|
+
(remaining <= 3 ? ` Unlock unlimited: ${UPGRADE_URL}` : '') +
|
|
94
|
+
'\n');
|
|
95
|
+
}
|
|
96
|
+
/* ── CLI ────────────────────────────────────────────────────────────── */
|
|
13
97
|
const program = new Command();
|
|
14
98
|
program
|
|
15
99
|
.name('stepproof')
|
|
16
100
|
.description('Regression testing for multi-step AI workflows. Not observability — a CI gate.')
|
|
17
|
-
.version('0.2.
|
|
101
|
+
.version('0.2.3')
|
|
18
102
|
.addHelpText('after', `
|
|
19
103
|
Examples:
|
|
20
104
|
stepproof init scaffold a starter scenario
|
|
@@ -28,7 +112,26 @@ program
|
|
|
28
112
|
runInit(dir);
|
|
29
113
|
});
|
|
30
114
|
program
|
|
31
|
-
.command('
|
|
115
|
+
.command('activate <key>')
|
|
116
|
+
.description('Store a license key for unlimited runs')
|
|
117
|
+
.action((key) => {
|
|
118
|
+
const result = validate(key);
|
|
119
|
+
if (!result.valid) {
|
|
120
|
+
process.stderr.write(`\nInvalid license key: ${result.reason}\n\n`);
|
|
121
|
+
process.exit(1);
|
|
122
|
+
}
|
|
123
|
+
try {
|
|
124
|
+
fs.mkdirSync(CONFIG_DIR, { recursive: true });
|
|
125
|
+
fs.writeFileSync(CONFIG_FILE, JSON.stringify({ key }), 'utf8');
|
|
126
|
+
console.log(`\nLicense activated (${result.tier} — ${result.org}). Unlimited runs enabled.\n`);
|
|
127
|
+
}
|
|
128
|
+
catch (e) {
|
|
129
|
+
process.stderr.write(`\nFailed to save license: ${e.message}\n\n`);
|
|
130
|
+
process.exit(1);
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
program
|
|
134
|
+
.command('run [scenario]')
|
|
32
135
|
.description('Run a scenario YAML file and report pass rates per step')
|
|
33
136
|
.option('-n, --iterations <number>', 'Number of iterations to run (overrides scenario file)', parseInt)
|
|
34
137
|
.option('-o, --output <file>', 'Path for output file (JSON by default; SARIF or JUnit when --format is set)', 'stepproof-report.json')
|
|
@@ -50,9 +153,43 @@ program
|
|
|
50
153
|
if (opts.format === 'sarif' || opts.format === 'junit') {
|
|
51
154
|
guard('team', { feature: `--format ${opts.format}` });
|
|
52
155
|
}
|
|
156
|
+
// Usage limit — check before running (avoid wasted API calls)
|
|
157
|
+
if (!checkUsageLimit()) {
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
53
160
|
// --format implies quiet (suppress terminal output) unless --quiet already set
|
|
54
161
|
const isQuiet = opts.quiet || !!opts.format;
|
|
162
|
+
if (opts.iterations !== undefined) {
|
|
163
|
+
if (!Number.isInteger(opts.iterations) || opts.iterations <= 0) {
|
|
164
|
+
console.error(`\nError: --iterations must be a positive integer, got "${process.argv[process.argv.indexOf('--iterations') + 1] ?? process.argv[process.argv.indexOf('-n') + 1]}"`);
|
|
165
|
+
process.exit(2);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (scenarioPath.includes('\0')) {
|
|
169
|
+
console.error('\nError: Invalid path — null bytes are not allowed');
|
|
170
|
+
process.exit(2);
|
|
171
|
+
}
|
|
172
|
+
if (opts.output && opts.output.includes('\0')) {
|
|
173
|
+
console.error('\nError: Invalid output path — null bytes are not allowed');
|
|
174
|
+
process.exit(2);
|
|
175
|
+
}
|
|
55
176
|
const resolvedPath = path.resolve(process.cwd(), scenarioPath);
|
|
177
|
+
try {
|
|
178
|
+
const stat = fs.statSync(resolvedPath);
|
|
179
|
+
if (stat.isDirectory()) {
|
|
180
|
+
console.error(`\nError: "${scenarioPath}" is a directory.`);
|
|
181
|
+
console.error('Run a specific file: stepproof run ./scenarios/first-test.yaml');
|
|
182
|
+
process.exit(2);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
catch (statErr) {
|
|
186
|
+
if (statErr.code === 'ENOENT') {
|
|
187
|
+
console.error(`\nScenario not found: ${resolvedPath}`);
|
|
188
|
+
console.error("Run 'stepproof init' to scaffold a new scenario, or check the path.");
|
|
189
|
+
process.exit(2);
|
|
190
|
+
}
|
|
191
|
+
// Other stat errors — let parseScenario surface the message
|
|
192
|
+
}
|
|
56
193
|
let scenario;
|
|
57
194
|
try {
|
|
58
195
|
scenario = parseScenario(resolvedPath);
|
|
@@ -116,6 +253,8 @@ program
|
|
|
116
253
|
console.error(`Warning: Could not write JSON report: ${e.message}`);
|
|
117
254
|
}
|
|
118
255
|
}
|
|
256
|
+
// Track usage after successful run completion
|
|
257
|
+
trackUsageAfterRun();
|
|
119
258
|
// Exit 1 if any step below threshold — this is the CI gate
|
|
120
259
|
if (!report.allPassed) {
|
|
121
260
|
process.exit(1);
|
|
@@ -130,5 +269,17 @@ program.action(() => {
|
|
|
130
269
|
}
|
|
131
270
|
program.help(); // exits 0
|
|
132
271
|
});
|
|
272
|
+
// Override commander's default error handler for better UX
|
|
273
|
+
program.exitOverride((err) => {
|
|
274
|
+
if (err.code === 'commander.missingArgument') {
|
|
275
|
+
const cmd = err.message.match(/'([^']+)'/)?.[1] ?? 'scenario';
|
|
276
|
+
process.stderr.write(`\nError: Missing required argument <${cmd}>\n`);
|
|
277
|
+
process.stderr.write(`Usage: stepproof run <scenario>\n`);
|
|
278
|
+
process.stderr.write(`\nQuick start:\n stepproof init scaffold a starter scenario\n stepproof run ./scenarios/first-test.yaml\n\n`);
|
|
279
|
+
process.exit(2);
|
|
280
|
+
}
|
|
281
|
+
// Re-throw other commander errors to default handling
|
|
282
|
+
throw err;
|
|
283
|
+
});
|
|
133
284
|
program.parse(process.argv);
|
|
134
285
|
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,kCAAkC,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,OAAO,EAAE;;;;;qFAK6D,CAAC,CAAC;AAEvF,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,CAAC,GAAY,EAAE,EAAE;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC;AACf,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,IAOpC,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;IAE/D,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,mBAAmB,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACxC,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU;AAC5B,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAiB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,2EAA2E;AAE3E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,WAAW,GAAG,gDAAgD,CAAC;AACrE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AACzD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AAOvD,qFAAqF;AACrF,SAAS,eAAe;IACtB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAElD,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAqB,CAAC;YACnD,IAAI,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE;gBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,4BAA4B;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,eAAe,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;AAChD,CAAC;AAED,iCAAiC;AACjC,SAAS,SAAS;IAChB,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU;IACrE,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;YAC9C,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY;gBAAE,OAAO,MAAM,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,oBAAoB;IACtB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,iCAAiC;AACjC,SAAS,UAAU,CAAC,MAAmB;IACrC,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,mCAAmC;IACrC,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe;IACtB,IAAI,SAAS,EAAE;QAAE,OAAO,IAAI,CAAC;IAE7B,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,IAAI,KAAK,CAAC,KAAK,IAAI,kBAAkB,EAAE,CAAC;QACtC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iBAAiB,kBAAkB,IAAI,kBAAkB,yBAAyB;YAClF,0DAA0D,WAAW,MAAM,CAC5E,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2EAA2E;AAC3E,SAAS,kBAAkB;IACzB,IAAI,SAAS,EAAE;QAAE,OAAO;IAExB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,UAAU,CAAC,KAAK,CAAC,CAAC;IAElB,MAAM,SAAS,GAAG,kBAAkB,GAAG,KAAK,CAAC,KAAK,CAAC;IACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,SAAS,IAAI,kBAAkB,kCAAkC;QACtE,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,sBAAsB,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,IAAI,CACL,CAAC;AACJ,CAAC;AAED,2EAA2E;AAE3E,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,OAAO,EAAE;;;;;qFAK6D,CAAC,CAAC;AAEvF,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,CAAC,GAAY,EAAE,EAAE;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC;AACf,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,wCAAwC,CAAC;KACrD,MAAM,CAAC,CAAC,GAAW,EAAE,EAAE;IACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,GAAG,8BAA8B,CAAC,CAAC;IACjG,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,MAAM,CAAC,CAAC;QAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,yDAAyD,CAAC;KACtE,MAAM,CAAC,2BAA2B,EAAE,uDAAuD,EAAE,QAAQ,CAAC;KACtG,MAAM,CAAC,qBAAqB,EAAE,6EAA6E,EAAE,uBAAuB,CAAC;KACrI,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,SAAS,EAAE,qDAAqD,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,4BAA4B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,YAAgC,EAAE,IAOhD,EAAE,EAAE;IACH,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,8DAA8D;IAC9D,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,0DAA0D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnL,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,YAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAa,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,aAAa,YAAY,mBAAmB,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAK,OAAiC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAC;YACvD,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,4DAA4D;IAC9D,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,UAAU,IAAI,EAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;IAErE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,mBAAmB,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACxC,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;oBACzE,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,6BAA8B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3F,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,yCAA0C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,kBAAkB,EAAE,CAAC;IAErB,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU;AAC5B,CAAC,CAAC,CAAC;AAEH,2DAA2D;AAC3D,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,EAAE;IAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;QAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC1D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0HAA0H,CAAC,CAAC;QACjJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,sDAAsD;IACtD,MAAM,GAAG,CAAC;AACZ,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAuBA,wBAAgB,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAsBhD"}
|
package/dist/commands/init.js
CHANGED
|
@@ -3,17 +3,20 @@ import { resolve } from 'path';
|
|
|
3
3
|
const SCENARIO_SCAFFOLD = `# stepproof scenario — edit this, then run: stepproof run ./scenarios/first-test.yaml
|
|
4
4
|
name: First Test
|
|
5
5
|
description: "Test your AI agent's response quality"
|
|
6
|
-
iterations:
|
|
6
|
+
iterations: 3
|
|
7
7
|
|
|
8
8
|
steps:
|
|
9
9
|
- id: step-1
|
|
10
|
+
provider: anthropic # "anthropic" or "openai"
|
|
11
|
+
model: claude-haiku-4-5-20251001 # model to test
|
|
10
12
|
prompt: "Explain what you do in one sentence."
|
|
11
|
-
|
|
12
|
-
|
|
13
|
+
min_pass_rate: 0.8 # 80% of iterations must pass
|
|
14
|
+
assertions:
|
|
13
15
|
- type: contains
|
|
14
|
-
value: "AI"
|
|
16
|
+
value: "AI" # replace with text you expect in the response
|
|
15
17
|
|
|
16
|
-
#
|
|
18
|
+
# Assertion types: contains, not_contains, regex, json_schema, llm_judge
|
|
19
|
+
# Set ANTHROPIC_API_KEY or OPENAI_API_KEY before running
|
|
17
20
|
# Full docs: https://github.com/StanislavBG/stepproof
|
|
18
21
|
`;
|
|
19
22
|
export function runInit(outputDir) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,MAAM,iBAAiB,GAAG
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;CAkBzB,CAAC;AAEF,MAAM,UAAU,OAAO,CAAC,SAAkB;IACxC,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAE7C,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,8EAA8E,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -6,7 +6,7 @@ export function parseScenario(filePath) {
|
|
|
6
6
|
content = fs.readFileSync(filePath, 'utf-8');
|
|
7
7
|
}
|
|
8
8
|
catch (e) {
|
|
9
|
-
throw new Error(`Cannot read scenario file: ${filePath}
|
|
9
|
+
throw new Error(`Cannot read scenario file: ${filePath}\n Hint: Check the path, or run 'stepproof init' to scaffold a starter scenario.`);
|
|
10
10
|
}
|
|
11
11
|
let raw;
|
|
12
12
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario-parser.js","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAGhC,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,
|
|
1
|
+
{"version":3,"file":"scenario-parser.js","sourceRoot":"","sources":["../../src/core/scenario-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAGhC,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,mFAAmF,CAAC,CAAC;IAC7I,CAAC;IAED,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAG,GAA8B,CAAC;IAEhD,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAE5F,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,UAAU,EAAE,OAAO,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC9E,SAAS,EAAE,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI;YAC9E,CAAC,CAAC,QAAQ,CAAC,SAAmC;YAC9C,CAAC,CAAC,EAAE;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC;IAEhE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAkB,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,8CAA8C,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sCAAsC,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uCAAuC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;IACtF,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,+CAA+C,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,QAAQ,EAAE,IAAI,CAAC,QAAkC;QACjD,KAAK,EAAE,IAAI,CAAC,KAAe;QAC3B,MAAM,EAAE,IAAI,CAAC,MAAgB;QAC7B,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACjE,aAAa,EAAE,WAAW;QAC1B,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,SAAiC,EACjC,WAAmC;IAEnC,OAAO,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAE3B,sDAAsD;QACtD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC5D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAC/B,CAAC;QAED,wDAAwD;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "stepproof",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Regression testing for multi-step AI workflows. Not observability — a CI gate.",
|
|
5
5
|
"main": "dist/cli.js",
|
|
6
6
|
"bin": {
|
|
@@ -57,6 +57,6 @@
|
|
|
57
57
|
"@types/node": "^20.17.0",
|
|
58
58
|
"tsx": "^4.19.0",
|
|
59
59
|
"typescript": "^5.7.0",
|
|
60
|
-
"vitest": "^2.
|
|
60
|
+
"vitest": "^3.2.4"
|
|
61
61
|
}
|
|
62
62
|
}
|