saffron-ai 1.3.5 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1809 -3
- package/README.md +22 -6
- package/dist-pkg/cli.js +475 -432
- package/package.json +49 -4
- package/skills/saffron/SKILL.md +70 -16
- package/skills/saffron/references/api.md +11 -2
- package/skills/saffron/references/config.md +4 -3
- package/skills/saffron/references/database.md +327 -0
- package/skills/saffron/references/syntax.md +51 -14
- package/templates/playwright/saffron-helpers.ts +855 -153
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.1",
|
|
4
4
|
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
"dev": "tsc --watch",
|
|
9
9
|
"e2e": "node scripts/e2e.mjs && node scripts/e2e-eject.mjs",
|
|
10
10
|
"benchmark": "node scripts/benchmark.mjs",
|
|
11
|
-
"
|
|
11
|
+
"gate": "node scripts/gate.mjs",
|
|
12
|
+
"prepublishOnly": "npm run gate",
|
|
12
13
|
"prepack": "npm run bundle",
|
|
13
14
|
"docs": "node scripts/build-compat.mjs && node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
|
|
14
15
|
"bundle": "esbuild src/cli/index.ts --bundle --platform=node --format=esm --packages=external --minify --legal-comments=none --outfile=dist-pkg/cli.js",
|
|
@@ -45,11 +46,20 @@
|
|
|
45
46
|
"saffron": "dist-pkg/cli.js"
|
|
46
47
|
},
|
|
47
48
|
"devDependencies": {
|
|
49
|
+
"@aws-sdk/client-dynamodb": "^3.1166.0",
|
|
48
50
|
"@cursor/sdk": "^1.0.32",
|
|
51
|
+
"@google-cloud/firestore": "^7.11.6",
|
|
49
52
|
"@openai/codex-sdk": "^0.158.0",
|
|
53
|
+
"@types/mssql": "^12.3.0",
|
|
50
54
|
"@types/node": "^26.1.0",
|
|
55
|
+
"@types/pg": "^8.23.1",
|
|
51
56
|
"esbuild": "^0.28.1",
|
|
52
57
|
"marked": "^18.0.5",
|
|
58
|
+
"mongodb": "^7.7.0",
|
|
59
|
+
"mssql": "^12.7.14",
|
|
60
|
+
"mysql2": "^3.24.6",
|
|
61
|
+
"node-sql-parser": "^5.4.0",
|
|
62
|
+
"oracledb": "^6.10.0",
|
|
53
63
|
"typescript": "^6.0.3",
|
|
54
64
|
"vitest": "^4.1.9"
|
|
55
65
|
},
|
|
@@ -62,7 +72,7 @@
|
|
|
62
72
|
"ajv": "^8.17.1",
|
|
63
73
|
"ajv-formats": "^3.0.1",
|
|
64
74
|
"commander": "^15.0.0",
|
|
65
|
-
"playwright": "^1.
|
|
75
|
+
"playwright": "^1.65.1",
|
|
66
76
|
"vscode-languageserver": "^10.1.0",
|
|
67
77
|
"vscode-languageserver-textdocument": "^1.0.12",
|
|
68
78
|
"yaml": "^2.9.1",
|
|
@@ -87,5 +97,40 @@
|
|
|
87
97
|
"README.md",
|
|
88
98
|
"CHANGELOG.md",
|
|
89
99
|
"LICENSE"
|
|
90
|
-
]
|
|
100
|
+
],
|
|
101
|
+
"optionalDependencies": {
|
|
102
|
+
"pg": "^8.23.1"
|
|
103
|
+
},
|
|
104
|
+
"peerDependencies": {
|
|
105
|
+
"@aws-sdk/client-dynamodb": "*",
|
|
106
|
+
"@google-cloud/firestore": "*",
|
|
107
|
+
"mongodb": "*",
|
|
108
|
+
"mssql": "*",
|
|
109
|
+
"mysql2": "*",
|
|
110
|
+
"node-sql-parser": "*",
|
|
111
|
+
"oracledb": "*"
|
|
112
|
+
},
|
|
113
|
+
"peerDependenciesMeta": {
|
|
114
|
+
"node-sql-parser": {
|
|
115
|
+
"optional": true
|
|
116
|
+
},
|
|
117
|
+
"mssql": {
|
|
118
|
+
"optional": true
|
|
119
|
+
},
|
|
120
|
+
"mysql2": {
|
|
121
|
+
"optional": true
|
|
122
|
+
},
|
|
123
|
+
"mongodb": {
|
|
124
|
+
"optional": true
|
|
125
|
+
},
|
|
126
|
+
"oracledb": {
|
|
127
|
+
"optional": true
|
|
128
|
+
},
|
|
129
|
+
"@aws-sdk/client-dynamodb": {
|
|
130
|
+
"optional": true
|
|
131
|
+
},
|
|
132
|
+
"@google-cloud/firestore": {
|
|
133
|
+
"optional": true
|
|
134
|
+
}
|
|
135
|
+
}
|
|
91
136
|
}
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -3,14 +3,14 @@ name: saffron
|
|
|
3
3
|
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
|
-
saffron-ai: "1.
|
|
6
|
+
saffron-ai: "1.4.1"
|
|
7
7
|
author: saffron-ai
|
|
8
8
|
homepage: https://saffron-ai.io
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
# Saffron
|
|
12
12
|
|
|
13
|
-
This skill describes saffron-ai 1.
|
|
13
|
+
This skill describes saffron-ai 1.4.1 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
|
|
14
14
|
|
|
15
15
|
Apply this workflow only when asked to create, translate, or edit Saffron
|
|
16
16
|
scenarios in `.saffron` or `.feature` files. A Saffron mention or an
|
|
@@ -52,7 +52,7 @@ replays for free.
|
|
|
52
52
|
**Before writing any step, look at what the project already knows:**
|
|
53
53
|
|
|
54
54
|
```bash
|
|
55
|
-
npx saffron steps # every step: ● recorded,
|
|
55
|
+
npx saffron steps # every step: ● recorded, ◐ divergent, ○ unrecorded
|
|
56
56
|
npx saffron steps "login" # search
|
|
57
57
|
npx saffron steps --json # machine-readable
|
|
58
58
|
```
|
|
@@ -96,14 +96,17 @@ end up with something recorded against it. Put
|
|
|
96
96
|
the final assertions **last**; the trailing block of `Then` steps is
|
|
97
97
|
strict under every policy. Use `Given`/`When` for actions. A text check
|
|
98
98
|
passes only on text the page shows: hidden elements, hidden parts of an
|
|
99
|
-
element
|
|
100
|
-
means visible.
|
|
99
|
+
element, hidden dropdown options and frames the page hides never count,
|
|
100
|
+
so "should be shown" means visible. One narrow exception to "never healed": a database check
|
|
101
|
+
whose query fails after a schema change may have its query (table,
|
|
102
|
+
columns, joins) repaired by the agent; its binds, expected rows and row
|
|
103
|
+
count never change.
|
|
101
104
|
|
|
102
105
|
## 3. Never bake volatile or secret values into files
|
|
103
106
|
|
|
104
107
|
- Secrets: `{env:VAR}`, as in `When I enter "{env:ADMIN_PASSWORD}" in the password field`.
|
|
105
108
|
- Shared test data (accounts, names, enum values): put it in `data/*.json` and write `{data:users.admin.email}` (file, then dotted path; one value, not a list). The recording stores the token, so editing the file changes the next replay at zero tokens. Never for secrets: data files are committed. To check a whole list (an enum), point a Then step at it: `Then the status filter should list every {data:enums.OrderStatus}`; replay iterates the file's list, so adding a value needs no re-record. `Examples: {data:roles}` reads outline rows from `data/roles.csv` or a JSON list (`.saffron` only). A value that must be new on every run (an email for a sign-up) is `{unique:name}`: fresh per run, the same within it.
|
|
106
|
-
- State the application must be in before the suite (seeded users, a reset database): do not script it through the UI in every scenario. The project's `saffron.config.json` takes `"setup"` and `"teardown"` shell commands, run once around `saffron run` (teardown always runs). Propose a seed script there; keep scenario steps for the behaviour under test.
|
|
109
|
+
- State the application must be in before the suite (seeded users, a reset database): do not script it through the UI in every scenario. The project's `saffron.config.json` takes `"setup"` and `"teardown"` shell commands, run once around `saffron run` (teardown always runs). Propose a seed script there; keep scenario steps for the behaviour under test. Data one scenario needs can be seeded inside it instead with a database seed step (`Given a customer "ann-{unique:c}@example.test" on the "basic" tier`), when the project configures `db` with writes on; see Database steps below.
|
|
107
110
|
- Dates: write intent when the day should move ("1 day from today" records as `{date+1}`); a date you write out (`I enter 2026-10-10`, in a table, a data file or `{env:...}`) is entered as written on every run. An Examples cell such as `{date+2:DD.MM.YYYY}` gives each row its own day. A date the app shows in words or as a range is a token too (`{date+7:D. MMM:de-DE}`, `{date+7 - +9:MMM D, YYYY}`), never a setup script.
|
|
108
111
|
- Dynamic display values: capture and compare, e.g. `I record the total as "first"` … `"first" should differ from the displayed total`.
|
|
109
112
|
|
|
@@ -185,8 +188,10 @@ means visible.
|
|
|
185
188
|
`I record the basket total as "before"`, `I record the breakfast price as
|
|
186
189
|
"breakfast"`, then `Then the basket total should be "before" plus
|
|
187
190
|
"breakfast"`. Replay reads the one number each element shows, in either
|
|
188
|
-
decimal style, and compares to the cent
|
|
189
|
-
|
|
191
|
+
decimal style, and compares to the cent, or to the decimals the element
|
|
192
|
+
shows when it shows more: a rate shown as `0.1200` is not `0.1204`.
|
|
193
|
+
Minus and "times 2" work too; percentages, rounding rules and taxes do
|
|
194
|
+
not.
|
|
190
195
|
- **Check the route, not only the destination.** When the requirement is
|
|
191
196
|
a path (a 3DS challenge, a redirect, a confirmation dialog), add a
|
|
192
197
|
mid-scenario `Then` for each checkpoint, such as `Then I should see the
|
|
@@ -208,7 +213,9 @@ means visible.
|
|
|
208
213
|
```
|
|
209
214
|
|
|
210
215
|
Each cell records as its `<table:key>`, so editing a value replays at
|
|
211
|
-
zero AI
|
|
216
|
+
zero AI, unless the old value also stands inside a word the recording
|
|
217
|
+
holds (`Pro` in "Products"), which records the step once more; renaming
|
|
218
|
+
or adding a key re-records the step. Use a header
|
|
212
219
|
table for a list of records, and a `"""` doc string for free text or a
|
|
213
220
|
whole JSON body. Keep one or two values inline when they read as part of
|
|
214
221
|
the sentence (`When I read customer "cust-42" through the API`).
|
|
@@ -257,9 +264,11 @@ expected behavior in its input. Its output must still go through steps
|
|
|
257
264
|
|
|
258
265
|
```gherkin
|
|
259
266
|
Given As "seller" I am on the orders page
|
|
260
|
-
|
|
267
|
+
# the buyer's own browser
|
|
268
|
+
When As "buyer" I order the blue lamp
|
|
261
269
|
Then As "seller" I see an order for the blue lamp
|
|
262
|
-
|
|
270
|
+
# still the seller
|
|
271
|
+
And the order is marked new
|
|
263
272
|
```
|
|
264
273
|
|
|
265
274
|
Each name is its own browser session (cookies, login, tabs). It carries
|
|
@@ -296,13 +305,41 @@ or "should match the contract", and name the status of an error case.
|
|
|
296
305
|
Pin the contract first (`npx saffron api pin <url or file>`), and read
|
|
297
306
|
[API tests](references/api.md) for the recording, review and run details.
|
|
298
307
|
|
|
308
|
+
## Database steps (beta)
|
|
309
|
+
|
|
310
|
+
```gherkin
|
|
311
|
+
Scenario: An upgrade is saved
|
|
312
|
+
Given a customer "ann-{unique:c}@example.test" on the "basic" tier
|
|
313
|
+
When I upgrade "ann-{unique:c}@example.test" to "gold" on the account page
|
|
314
|
+
Then the customer "ann-{unique:c}@example.test" should be on the "gold" tier in the database
|
|
315
|
+
And no refund should exist in the database for "ann-{unique:c}@example.test"
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
With `db` in `saffron.config.json` (PostgreSQL, MySQL, SQL Server, Oracle, SQLite, MariaDB, CockroachDB, MongoDB, DynamoDB or Firestore), a `Then` can check
|
|
319
|
+
what the database holds and a `Given` or `When` can seed data, in browser and `@api`
|
|
320
|
+
scenarios alike. Name the database in a check ("in the database") and name
|
|
321
|
+
the record by a value the scenario wrote: the step's text, an Examples
|
|
322
|
+
column, a `{unique:...}` value, or a value an earlier step kept. A check
|
|
323
|
+
bound to nothing ("a customer exists") is refused while recording. A
|
|
324
|
+
"not in the database" check waits out `db.timeout` (3 s by default) before
|
|
325
|
+
it passes. Seeding needs `"writes": true` and the database's host or name
|
|
326
|
+
in `allow` (a SQLite file's name, a DynamoDB region, a Firestore project);
|
|
327
|
+
seed what the app gives no way to set, with `{unique:...}` values, and the
|
|
328
|
+
rows are undone when the scenario ends. A step never holds a query: the
|
|
329
|
+
recording agent writes it, as SQL or, on MongoDB, DynamoDB and Firestore,
|
|
330
|
+
as one JSON command. On DynamoDB name the item by the value of its key
|
|
331
|
+
(or of an index's key), since there is no scan; on Firestore name the
|
|
332
|
+
document by its id or by a field's value. Read
|
|
333
|
+
[Database steps](references/database.md) for the details.
|
|
334
|
+
|
|
299
335
|
## Repeated step groups → StepSets (`.saffron` files only)
|
|
300
336
|
|
|
301
337
|
`.saffron` is a superset of `.feature`: rename the file, nothing else
|
|
302
338
|
changes, and you gain the `StepSet:` keyword.
|
|
303
339
|
|
|
304
340
|
```gherkin
|
|
305
|
-
|
|
341
|
+
# define: colon, like Scenario:
|
|
342
|
+
StepSet: Complete guest information
|
|
306
343
|
Given I am on the guest information page
|
|
307
344
|
When I enter guest name "Chathuranga"
|
|
308
345
|
And I click the continue button
|
|
@@ -310,12 +347,16 @@ StepSet: Complete guest information # define: colon, like Scenario:
|
|
|
310
347
|
|
|
311
348
|
Scenario: Checkout happy path
|
|
312
349
|
Given I am on the cart page
|
|
313
|
-
|
|
350
|
+
# invoke: no colon, like a step
|
|
351
|
+
StepSet Complete guest information
|
|
314
352
|
Then I am on the checkout page
|
|
315
353
|
```
|
|
316
354
|
|
|
317
|
-
- Definition: `StepSet: <name>`. Invocation: `StepSet <name
|
|
318
|
-
name without the keyword is a
|
|
355
|
+
- Definition: `StepSet: <name>`. Invocation: `StepSet <name>`. The bare
|
|
356
|
+
name without the keyword is refused after a step, and read as
|
|
357
|
+
description, so left out, above a scenario's first step.
|
|
358
|
+
- A comment is a line of its own starting with `#`. After a step on the
|
|
359
|
+
same line it is part of the step's text.
|
|
319
360
|
- Names are project-wide unique; sets resolve across files; no nesting.
|
|
320
361
|
- Application-wide flows (login, cookie banner) live in a sets-only
|
|
321
362
|
library file: `features/shared.steps.saffron` (still needs a
|
|
@@ -353,7 +394,12 @@ refused (red, at zero tokens) with guidance naming the column and the
|
|
|
353
394
|
step. That includes dates without a year and one- or two-character
|
|
354
395
|
values. A blank cell is refused the same way, or, when its step types
|
|
355
396
|
`<column>`, the first filled row records once more and serves the blank
|
|
356
|
-
rows too.
|
|
397
|
+
rows too. A value the page sets in its own way (a phone or IBAN in
|
|
398
|
+
blocks, an id zero-padded inside a longer one like `INV-000104`, a date
|
|
399
|
+
cell shown in another format, an amount shown with other decimals, a data
|
|
400
|
+
row's value with outer spaces shown trimmed) binds the recording to its
|
|
401
|
+
row the same way: give the cell the value as the page shows it, or give
|
|
402
|
+
the shown value an Examples column of its own. Keep a check that lists all rows' values out of the Outline.
|
|
357
403
|
Tests live in the folder named by `features` in `saffron.config.json`
|
|
358
404
|
(`features/` by default).
|
|
359
405
|
|
|
@@ -392,6 +438,11 @@ Use Saffron itself first: `{date...}` tokens (ranges and locales included), capt
|
|
|
392
438
|
| Create a fixed record (`"cust-1"`) in an API step | `{unique:name}`: every run, and the proof replay, creates its own |
|
|
393
439
|
| Check an API response's status in a `When` | Name the status in a `Then` ("should answer 404") |
|
|
394
440
|
| One long step line carrying an API key, ids and several quoted parameters | A short step and a 2-column data table under it, one value per row |
|
|
441
|
+
| A database check bound to nothing ("Then a customer exists in the database") | Name the record the scenario made: `Then the customer "ann-{unique:c}@example.test" should be saved in the database` |
|
|
442
|
+
| SQL or a JSON command written in a step | Say what should be stored, in plain words; the recording agent writes the query for the project's database |
|
|
443
|
+
| `scenario:` in lower case, `Scenario :`, or a tab after a step keyword | Spell headers and keywords exactly: Saffron refuses the file, where Gherkin would drop the scenario silently |
|
|
444
|
+
| `When StepSet Login` | A call is a line of its own: `StepSet Login` |
|
|
445
|
+
| `--filter "@smoke && !@wip"` | `--filter "@smoke and not @wip"`: the operators are the words |
|
|
395
446
|
|
|
396
447
|
## References
|
|
397
448
|
|
|
@@ -403,3 +454,6 @@ Use Saffron itself first: `{date...}` tokens (ranges and locales included), capt
|
|
|
403
454
|
heal model, secrets setup, the API settings.
|
|
404
455
|
- [API tests](references/api.md): `@api` scenarios, the pinned contract,
|
|
405
456
|
writing and reviewing request steps, API calls from a browser scenario.
|
|
457
|
+
- [Database steps](references/database.md): checks and seeds against the
|
|
458
|
+
configured database, what is recorded and refused, cleanup, and the one
|
|
459
|
+
Then whose query may be repaired.
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
A scenario tagged `@api` tests an HTTP API with no browser. The agent
|
|
4
4
|
records each request once, every later run replays it at zero tokens, and
|
|
5
|
-
`Then` steps are never healed, as for browser scenarios
|
|
5
|
+
`Then` steps are never healed, as for browser scenarios (the one
|
|
6
|
+
exception is a database check's query after a schema change, see
|
|
7
|
+
[Database steps](database.md)). A browser
|
|
6
8
|
scenario can also call the API in a step; see the end of this page.
|
|
7
9
|
|
|
8
10
|
## Before writing
|
|
@@ -86,10 +88,17 @@ Feature: Customers API
|
|
|
86
88
|
- **Name the status of an error case** in the request's step or the
|
|
87
89
|
`Then` right after it ("should answer
|
|
88
90
|
404", "is refused with 409", "I get a 404", "the customer is not
|
|
89
|
-
found"): a number elsewhere ("I order 450 units") is no status. A
|
|
91
|
+
found"): a number elsewhere ("I order 450 units") is no status. A status
|
|
92
|
+
the step names only to rule it out ("should not answer 500") is not
|
|
93
|
+
named: a response with it fails the step. A response that is no
|
|
90
94
|
2xx is recorded only when its step names that status; otherwise the
|
|
91
95
|
step fails, so a failure seen while recording never replays as the
|
|
92
96
|
expected result.
|
|
97
|
+
- **A `Then` says what it checks.** A `Then` whose only check would be
|
|
98
|
+
the 2xx class, or the status its request already required, is refused
|
|
99
|
+
(it could never fail): name a field or "should match the contract". A
|
|
100
|
+
time limit ("within 2 seconds") or a poll's limit is recorded exactly as
|
|
101
|
+
the step states it in figures.
|
|
93
102
|
- **Secrets stay `{env:VAR}` tokens**, typically in a step that names the
|
|
94
103
|
credential: `Given I use the API token "{env:API_TOKEN}"`. The request
|
|
95
104
|
sends the value and the recording keeps the token.
|
|
@@ -44,7 +44,7 @@ your-project/
|
|
|
44
44
|
|
|
45
45
|
| Key | Meaning |
|
|
46
46
|
|---|---|
|
|
47
|
-
| `dataDir` | Folder of JSON files read by `{data:file.key}` tokens (default `data`). Committed; not for secrets |
|
|
47
|
+
| `dataDir` | Folder of JSON and CSV files read by `{data:file.key}` tokens and by `Examples: {data:file}` (default `data`). Committed; not for secrets |
|
|
48
48
|
| `baseURL` | App under test; steps say "the login page", not full URLs |
|
|
49
49
|
| `storageState` | Playwright storage-state JSON so replays and the agent start authenticated (`npx playwright open --save-storage=.auth/state.json <url>`) |
|
|
50
50
|
| `sessions` | Per browser session (`As "buyer" ...`): `{ "buyer": { "storageState": ".auth/buyer.json" } }` starts that session signed in as its own person; others, and the default session, take `storageState` |
|
|
@@ -72,7 +72,8 @@ your-project/
|
|
|
72
72
|
| `healModel` | Cheaper model for heal sessions only |
|
|
73
73
|
| `antigravityApiKey` | `true`: Antigravity signs in with `GEMINI_API_KEY` even beside an agy login (default: a login wins) |
|
|
74
74
|
| `ci` | Where `saffron runs` and `import --run` find CI runs: `{"provider": "github" \| "azure", "organization": ..., "project": ...}`. Unset, an Azure Repos remote means Azure Pipelines (through `az`), anything else GitHub Actions (through `gh`). `organization` and `project` are for a GitHub repository built by Azure Pipelines. `cacheRuns` and `cacheMB` cap the runs viewed into `.saffron/ci/` (default 10 and 1024) |
|
|
75
|
-
| `
|
|
75
|
+
| `db` | The database that database steps check and seed (PostgreSQL, MySQL, SQL Server, Oracle, SQLite, MariaDB, CockroachDB, MongoDB, DynamoDB or Firestore, by the URL's scheme; `npm install node-sql-parser` and `pg`, `mysql2`, `mssql` or `oracledb@6` for a SQL engine, none for SQLite's driver, or `mongodb`, `@aws-sdk/client-dynamodb` or `@google-cloud/firestore@7` alone): `{"url": "{env:DATABASE_URL}", "writes": false, "allow": ["localhost"]}`. `url` must be an `{env:...}` reference; `writes` (default `false`) lets `Given` and `When` steps insert and update with `dbWrite`; `allow` lists the hosts or database names a write may go to; `timeout` (default 3000 ms) is how long a check reads again, and a "not in the database" check must hold that long; `cleanup` is `"all"` (default: delete inserts, put updated rows back), `"inserts"` or `"none"`. A `Then` records `expectRows`: one SELECT, values as `$n` binds bound to a scenario value, checks on its rows; a check whose query fails on the schema may have its query repaired by the agent, its binds, checks and row count unchanged |
|
|
76
|
+
| `api` | The API under test: `{"baseURL": ..., "contract": "api/openapi.json", "transcript": "on-failure"}`. `baseURL` is where API steps' relative paths go (unset: the run's `baseURL`); `contract` is the pinned OpenAPI 3 document schema checks validate against (`saffron api pin` writes it), and a change to it, or to which file it names, makes the recordings checked against it stale; `transcript` keeps each scenario's requests and responses for the report and `saffron trace`: `"on-failure"` (default), `"on"` or `"off"` (statuses and timings only); `services` names more APIs, each `{"baseURL": ..., "contract": ...}`, which a request names instead of a host (`--api-url name=<url>` points one elsewhere for a run). A key Saffron does not read under `api` or a service is refused. See [API tests](api.md) |
|
|
76
77
|
|
|
77
78
|
## CLI
|
|
78
79
|
|
|
@@ -90,7 +91,7 @@ your-project/
|
|
|
90
91
|
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `run_plan`, `api_contract`, `verify`) for AI assistants / language server for editors |
|
|
91
92
|
| `saffron api pin <source> [--out file] [--service name]` | Pin the API contract: fetch an OpenAPI 3 document from a Swagger UI page, a document URL or a file, and write it as JSON (default `api/openapi.json`, or `api/<name>.json` with `--service`). Add it to the config as `api.contract`, or as `api.services.<name>.contract` |
|
|
92
93
|
| `saffron init [--examples] [--agents list]` | Install this skill into the project's agent directories, register MCP, add an AGENTS.md block, scaffold config; `--examples` adds the Saucedemo example suite |
|
|
93
|
-
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs and 1 GB unless `ci.cacheRuns` and `ci.cacheMB` say otherwise), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
|
|
94
|
+
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line; it exits 1 when the merged run has a red (or a pending review with `strict` on). `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs and 1 GB unless `ci.cacheRuns` and `ci.cacheMB` say otherwise), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
|
|
94
95
|
| `saffron dashboard [--stdout] [--no-open]` | The suite's quality from the run history (failing and for how long, flaky, repeatedly healed, never run, slowest, trends, per tag); writes and opens `.saffron/reports/dashboard.html`, `--stdout` prints it |
|
|
95
96
|
| `saffron export [-o file] [--branch name [--base branch]] [--report-only]` | Pack the pending proposals, the run's report, screenshots and traces into one bundle (default `.saffron/reports/saffron-bundle.zip`). `--report-only` writes a summary instead (report and screenshots only, `saffron-summary.zip`): upload it as a `saffron-summary*` artifact on every run so any run's results can be viewed; a run that left only summaries has nothing to import. In CI write it with `--out <file outside .saffron>` and upload that file as an artifact (`upload-artifact` skips hidden folders). `--branch` also pushes the proposals as a new branch and opens a draft pull request (GitHub through `gh`, Azure DevOps through `az`); exit 1 only when the branch is pushed but the pull request could not be opened |
|
|
96
97
|
| `saffron runs [--branch b] [--limit n] [--provider github\|azure] [--json]` | The branch's CI runs, with how many of each run's proposals are pending here; exit 2 when `gh` or `az` is missing or signed out |
|