saffron-ai 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +356 -0
- package/README.md +23 -10
- package/dist-pkg/cli.js +533 -379
- package/package.json +29 -19
- package/skills/saffron/SKILL.md +42 -4
- package/skills/saffron/references/api.md +151 -0
- package/skills/saffron/references/config.md +12 -6
- package/skills/saffron/references/syntax.md +4 -0
- package/templates/playwright/saffron-helpers.ts +626 -7
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.1
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.2.1",
|
|
4
|
+
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
|
7
7
|
"build": "tsc",
|
|
@@ -14,15 +14,30 @@
|
|
|
14
14
|
"plugin": "node scripts/build-claude-plugin.mjs"
|
|
15
15
|
},
|
|
16
16
|
"keywords": [
|
|
17
|
+
"e2e",
|
|
18
|
+
"e2e-testing",
|
|
19
|
+
"end-to-end-testing",
|
|
17
20
|
"testing",
|
|
18
|
-
"
|
|
19
|
-
"bdd",
|
|
21
|
+
"test-automation",
|
|
20
22
|
"playwright",
|
|
21
|
-
"ai",
|
|
23
|
+
"ai-testing",
|
|
22
24
|
"self-healing",
|
|
23
|
-
"
|
|
25
|
+
"self-healing-tests",
|
|
26
|
+
"natural-language",
|
|
27
|
+
"plain-language",
|
|
28
|
+
"bdd",
|
|
29
|
+
"gherkin",
|
|
30
|
+
"cucumber",
|
|
31
|
+
"feature-files",
|
|
32
|
+
"no-step-definitions",
|
|
33
|
+
"browser-testing",
|
|
34
|
+
"regression-testing",
|
|
35
|
+
"claude",
|
|
36
|
+
"codex",
|
|
37
|
+
"cursor",
|
|
38
|
+
"mcp"
|
|
24
39
|
],
|
|
25
|
-
"author": "",
|
|
40
|
+
"author": "Chathuranga Jayasinghe",
|
|
26
41
|
"license": "SEE LICENSE IN LICENSE",
|
|
27
42
|
"type": "module",
|
|
28
43
|
"bin": {
|
|
@@ -43,25 +58,20 @@
|
|
|
43
58
|
"@cucumber/messages": "^33.0.4",
|
|
44
59
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
45
60
|
"@playwright/mcp": "^0.0.77",
|
|
61
|
+
"ajv": "^8.17.1",
|
|
62
|
+
"ajv-formats": "^3.0.1",
|
|
46
63
|
"commander": "^15.0.0",
|
|
47
64
|
"playwright": "^1.63.0",
|
|
48
65
|
"vscode-languageserver": "^10.1.0",
|
|
49
66
|
"vscode-languageserver-textdocument": "^1.0.12",
|
|
67
|
+
"yaml": "^2.9.1",
|
|
50
68
|
"zod": "^4.4.3"
|
|
51
69
|
},
|
|
52
|
-
"peerDependencies": {
|
|
53
|
-
"@cursor/sdk": "^1.0.32",
|
|
54
|
-
"@openai/codex-sdk": ">=0.158.0"
|
|
55
|
-
},
|
|
56
|
-
"peerDependenciesMeta": {
|
|
57
|
-
"@cursor/sdk": {
|
|
58
|
-
"optional": true
|
|
59
|
-
},
|
|
60
|
-
"@openai/codex-sdk": {
|
|
61
|
-
"optional": true
|
|
62
|
-
}
|
|
63
|
-
},
|
|
64
70
|
"homepage": "https://saffron-ai.io",
|
|
71
|
+
"repository": {
|
|
72
|
+
"type": "git",
|
|
73
|
+
"url": "git+https://github.com/s-chathuranga-j/saffron-community.git"
|
|
74
|
+
},
|
|
65
75
|
"bugs": {
|
|
66
76
|
"url": "https://github.com/s-chathuranga-j/saffron-community/issues"
|
|
67
77
|
},
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: saffron
|
|
3
|
-
description: Create, translate, or edit Saffron
|
|
3
|
+
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
6
|
author: saffron-ai
|
|
@@ -29,6 +29,13 @@ on Playwright's browser engine; Playwright MCP is used only by the agent
|
|
|
29
29
|
while recording and healing.) When the UI drifts, the agent heals the failing
|
|
30
30
|
step mid-run and files a reviewable proposal.
|
|
31
31
|
|
|
32
|
+
A scenario tagged `@api` tests an HTTP API instead, with no browser: the
|
|
33
|
+
agent records each request and the checks on its response, against the
|
|
34
|
+
project's pinned OpenAPI contract. The rules and workflow below hold for
|
|
35
|
+
both; [API tests](references/api.md) has what is specific to them. Read
|
|
36
|
+
it before writing or reviewing an `@api` scenario, or a step that calls
|
|
37
|
+
the API from a browser scenario.
|
|
38
|
+
|
|
32
39
|
Your job when writing tests is to make that economy work. Three rules
|
|
33
40
|
matter more than everything else:
|
|
34
41
|
|
|
@@ -54,7 +61,8 @@ names the old value elsewhere (a `delete-bob` test id, a `#row-bob`
|
|
|
54
61
|
selector) is not reused for the new one: that step records afresh. `saffron init` registers the
|
|
55
62
|
`saffron` MCP server with Claude Code, Cursor and VS Code; its
|
|
56
63
|
`search_steps` / `list_step_sets` / `project_status` tools give the same
|
|
57
|
-
answers without a shell
|
|
64
|
+
answers without a shell, and `api_contract` lists the operations of the
|
|
65
|
+
pinned API contract for `@api` scenarios.
|
|
58
66
|
|
|
59
67
|
**Know the project before touching it:**
|
|
60
68
|
|
|
@@ -106,7 +114,10 @@ means visible.
|
|
|
106
114
|
flow, but do not establish its current UI behavior. Reuse recent,
|
|
107
115
|
relevant browser evidence instead of exploring the whole application.
|
|
108
116
|
If access or test data is missing, identify the gap and produce a draft
|
|
109
|
-
with explicit assumptions; do not invent observations.
|
|
117
|
+
with explicit assumptions; do not invent observations. For an `@api`
|
|
118
|
+
scenario, read the pinned contract (`api_contract`) instead of the
|
|
119
|
+
page, and where access allows, send the request once to see a real
|
|
120
|
+
response; pin a contract first if the project has none.
|
|
110
121
|
3. **Design one behavior per scenario.** When translating Cypress or other
|
|
111
122
|
tests, preserve the intended coverage, reconsider assumptions, and
|
|
112
123
|
split independent changes. Compare the same target immediately before
|
|
@@ -236,6 +247,27 @@ it, and never copy one person's prefix onto another's step: a recording
|
|
|
236
247
|
is reused only within its own session, so the buyer's typed address
|
|
237
248
|
never seeds the seller's step.
|
|
238
249
|
|
|
250
|
+
## API tests: `@api`
|
|
251
|
+
|
|
252
|
+
```gherkin
|
|
253
|
+
@api
|
|
254
|
+
Feature: Customers API
|
|
255
|
+
|
|
256
|
+
Scenario: Subscribe a new customer to a plan
|
|
257
|
+
When I create a customer named "cust-{unique:name}" through the API
|
|
258
|
+
And I put the customer on the "Basic" plan for 3 months
|
|
259
|
+
Then the customer should be on the "Basic" plan for 3 months
|
|
260
|
+
And the customer should match the contract
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
One request intent per `When` step, its values in the step text. Records
|
|
264
|
+
that must be new each run take `{unique:name}`. Refer to an earlier
|
|
265
|
+
result by intent ("the customer"), never by an id you saw: the agent keeps
|
|
266
|
+
the fields later steps need. `Then` steps state a status, a field's value
|
|
267
|
+
or "should match the contract", and name the status of an error case.
|
|
268
|
+
Pin the contract first (`npx saffron api pin <url or file>`), and read
|
|
269
|
+
[API tests](references/api.md) for the recording, review and run details.
|
|
270
|
+
|
|
239
271
|
## Repeated step groups → StepSets (`.saffron` files only)
|
|
240
272
|
|
|
241
273
|
`.saffron` is a superset of `.feature`: rename the file, nothing else
|
|
@@ -271,6 +303,7 @@ npx saffron trace "<scenario>" # inspect a failed or adapted step (
|
|
|
271
303
|
npx saffron accept <proposal-file> # accept a reviewed, verified proposal
|
|
272
304
|
npx saffron run features/login.saffron:12 --no-agent # prove cached replay
|
|
273
305
|
npx saffron status --json # confirm affected scenario state
|
|
306
|
+
npx saffron api pin <url or file> # pin the API contract @api scenarios check against
|
|
274
307
|
```
|
|
275
308
|
|
|
276
309
|
Results: **green** = cached pass · **yellow** = pending review · **red** =
|
|
@@ -316,6 +349,9 @@ setup, and other configuration needed by the requested test, read
|
|
|
316
349
|
| Sleep/wait "for 3 seconds" | Wait for a UI signal or an API response |
|
|
317
350
|
| Several independent changes and all-pairs comparisons in one scenario | One behavior and its own baseline per scenario; reuse setup with StepSets in `.saffron` |
|
|
318
351
|
| Edit cache JSON to fix a test | Edit the scenario text and re-run, or `saffron run --rerecord` if the recording itself is wrong |
|
|
352
|
+
| Write an id you saw in a response into a later API step | Refer to it by intent ("the customer I created"); the recording keeps it as `{captured:name}` |
|
|
353
|
+
| Create a fixed record (`"cust-1"`) in an API step | `{unique:name}`: every run, and the proof replay, creates its own |
|
|
354
|
+
| Check an API response's status in a `When` | Name the status in a `Then` ("should answer 404") |
|
|
319
355
|
|
|
320
356
|
## References
|
|
321
357
|
|
|
@@ -323,4 +359,6 @@ setup, and other configuration needed by the requested test, read
|
|
|
323
359
|
outlines, secrets, dates, network waits, page furniture, StepSets.
|
|
324
360
|
- [Configuration & CI](references/config.md): `saffron.config.json`,
|
|
325
361
|
flags, `--strict`, cross-browser and browser matrix, devices, workers
|
|
326
|
-
and agent workers, heal model, secrets setup.
|
|
362
|
+
and agent workers, heal model, secrets setup, the API settings.
|
|
363
|
+
- [API tests](references/api.md): `@api` scenarios, the pinned contract,
|
|
364
|
+
writing and reviewing request steps, API calls from a browser scenario.
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# API tests (`@api`)
|
|
2
|
+
|
|
3
|
+
A scenario tagged `@api` tests an HTTP API with no browser. The agent
|
|
4
|
+
records each request once, every later run replays it at zero tokens, and
|
|
5
|
+
`Then` steps are never healed, as for browser scenarios. A browser
|
|
6
|
+
scenario can also call the API in a step; see the end of this page.
|
|
7
|
+
|
|
8
|
+
## Before writing
|
|
9
|
+
|
|
10
|
+
1. **Pin the contract.** Schema checks validate responses against an
|
|
11
|
+
OpenAPI 3 document committed to the project, never one fetched at run
|
|
12
|
+
time:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
npx saffron api pin https://api.example.test/docs # a Swagger UI page, a document URL, or a file
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
It writes `api/openapi.json`. Add it to `saffron.config.json`, with the
|
|
19
|
+
API's address when it differs from the app's:
|
|
20
|
+
|
|
21
|
+
```json
|
|
22
|
+
{ "api": { "baseURL": "https://api.example.test", "contract": "api/openapi.json" } }
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
2. **Read what the API offers.** The `api_contract` MCP tool (or the
|
|
26
|
+
pinned file) lists every operation with its parameters, body fields and
|
|
27
|
+
response codes. Write steps for operations the API has, with the fields
|
|
28
|
+
it accepts and the statuses it answers. Do not invent endpoints.
|
|
29
|
+
3. **Several APIs?** The default one is `api.baseURL`; others are named
|
|
30
|
+
under `api.services` (`"orders": { "baseURL": ..., "contract":
|
|
31
|
+
"api/orders.json" }`, pinned with `npx saffron api pin <url> --service
|
|
32
|
+
orders`). `api_contract` lists each. A step that calls a named one says
|
|
33
|
+
so ("through the orders API"); the recording keeps the name, never the
|
|
34
|
+
host.
|
|
35
|
+
4. **Reuse the vocabulary**, as for any scenario: `npx saffron steps
|
|
36
|
+
"through the API"` and `search_steps` show the wordings already recorded.
|
|
37
|
+
|
|
38
|
+
## Writing API steps
|
|
39
|
+
|
|
40
|
+
```gherkin
|
|
41
|
+
@api
|
|
42
|
+
Feature: Customers API
|
|
43
|
+
|
|
44
|
+
Scenario: Subscribe a new customer to a plan
|
|
45
|
+
When I create a customer named "cust-{unique:name}" through the API
|
|
46
|
+
And I put the customer on the "Basic" plan for 3 months
|
|
47
|
+
Then the customer should be on the "Basic" plan for 3 months
|
|
48
|
+
And the customer should match the contract
|
|
49
|
+
|
|
50
|
+
Scenario: A missing customer answers 404
|
|
51
|
+
When I read customer 999999 through the API
|
|
52
|
+
Then the response status should be 404
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
- **One request intent per `When` step**, with the values it sends in
|
|
56
|
+
the step text: quoted strings, numbers, `<column>`s, `{data:...}`. The
|
|
57
|
+
recording keeps them as the step writes them, and a number stays a
|
|
58
|
+
number in the request body on every Examples row.
|
|
59
|
+
- **Records that must be new each run** take `{unique:name}`. A proof
|
|
60
|
+
replay is a second execution, and every later run is another: each
|
|
61
|
+
creates its own customer instead of colliding with the last one. Clean
|
|
62
|
+
up with the project's `teardown` command, not with steps.
|
|
63
|
+
- **Refer to earlier results by intent** ("the customer", "the order I
|
|
64
|
+
placed"), never by an id you saw while exploring. The agent keeps the
|
|
65
|
+
response fields later steps need and records them as `{captured:name}`,
|
|
66
|
+
so replay uses the id its own request got.
|
|
67
|
+
- **`Then` steps say what must be true of a response**: its status, a
|
|
68
|
+
field's value, or "should match the contract" for a schema check. A
|
|
69
|
+
`Then` that needs fresh data reads it itself (the agent sends a GET
|
|
70
|
+
under that step). Never put a write in a `Then`.
|
|
71
|
+
- **Name the status of an error case** in the request's step or the
|
|
72
|
+
`Then` right after it ("should answer
|
|
73
|
+
404", "is refused with 409", "I get a 404", "the customer is not
|
|
74
|
+
found"): a number elsewhere ("I order 450 units") is no status. A response that is no
|
|
75
|
+
2xx is recorded only when its step names that status; otherwise the
|
|
76
|
+
step fails, so a failure seen while recording never replays as the
|
|
77
|
+
expected result.
|
|
78
|
+
- **Secrets stay `{env:VAR}` tokens**, typically in a step that names the
|
|
79
|
+
credential: `Given I use the API token "{env:API_TOKEN}"`. The request
|
|
80
|
+
sends the value and the recording keeps the token.
|
|
81
|
+
- **Outlines** work as for browser scenarios: write `<column>` in every
|
|
82
|
+
step whose request or check depends on the row.
|
|
83
|
+
|
|
84
|
+
## What gets recorded, and what to check in `saffron diff`
|
|
85
|
+
|
|
86
|
+
| Action | Holds |
|
|
87
|
+
|---|---|
|
|
88
|
+
| `request` | method, path, the named API it goes to (`service`, absent for the default), headers, the JSON body, the fields it keeps (`extract`: name to JSON Pointer, a segment spelling a column the step names written as `<column>`), and the status it expects |
|
|
89
|
+
| `expectStatus` | the status of the latest response in that step's session, exact (`404`) or a class (`2xx`) |
|
|
90
|
+
| `expectJson` | field checks by JSON Pointer, typed (`/months` equals `3` as a number) or present/absent; an entry of a list with `item` (`{"path": "/messages", "item": [{"path": "/name", "equals": "ann"}]}`), wherever it stands |
|
|
91
|
+
| `expectSchema` | the latest response against the pinned contract: the operation's response schema, or a named component schema |
|
|
92
|
+
|
|
93
|
+
Before accepting a proposal, check that:
|
|
94
|
+
|
|
95
|
+
- no id, token or timestamp from the recording run is written literally
|
|
96
|
+
in a path, header or body (a kept value reads `{captured:name}`);
|
|
97
|
+
- a request to a named API carries `"service"` and a relative path, never
|
|
98
|
+
that API's host;
|
|
99
|
+
- secrets read `{env:...}`, and values the step wrote keep their tokens;
|
|
100
|
+
- every `Then` has a status, field or schema check that states the step,
|
|
101
|
+
and "should match the contract" became an `expectSchema`;
|
|
102
|
+
- an error case expects the status the step names;
|
|
103
|
+
- an entry of a list is an `item` check, not a position (`/0/name`),
|
|
104
|
+
unless the step names the position ("the first order").
|
|
105
|
+
|
|
106
|
+
The agent verifies each check against the response before it records it,
|
|
107
|
+
so a recorded check held at recording time; review whether it is the
|
|
108
|
+
check the step asks for.
|
|
109
|
+
|
|
110
|
+
## Running and reading results
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
npx saffron run --filter @api # API scenarios alone: no browser needed
|
|
114
|
+
npx saffron run features/customers-api.saffron:4 --no-agent
|
|
115
|
+
npx saffron trace "A missing customer answers 404" # the requests and responses, per step
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
- A write (`POST`, `PUT`, `PATCH`, `DELETE`) is never retried on replay:
|
|
119
|
+
a timeout fails the step instead of creating a second record. Reads are
|
|
120
|
+
retried as usual.
|
|
121
|
+
- A recording with a schema check is checked again when the pinned
|
|
122
|
+
contract changes, or when `api.contract` names another file: a zero-AI
|
|
123
|
+
replay of the same recording, filed for review (yellow) when it passes,
|
|
124
|
+
red when the API breaks the new contract, and no agent session either way.
|
|
125
|
+
It never passes green over a contract it was not checked against.
|
|
126
|
+
`saffron run --rerecord` records it again.
|
|
127
|
+
- `api.transcript` sets what the request transcript keeps: `"on-failure"`
|
|
128
|
+
(default), `"on"` (green scenarios too) or `"off"` (statuses and timings
|
|
129
|
+
only). The report lists each scenario's requests per step, and
|
|
130
|
+
`saffron trace` replays an API scenario from its transcript. Nothing is
|
|
131
|
+
drawn while a scenario runs.
|
|
132
|
+
|
|
133
|
+
## API calls in a browser scenario
|
|
134
|
+
|
|
135
|
+
A browser scenario (no `@api` tag) can call the API in a step, for setup
|
|
136
|
+
the UI would make slow or for a check the page does not show:
|
|
137
|
+
|
|
138
|
+
```gherkin
|
|
139
|
+
Scenario: A customer created through the API shows in the admin list
|
|
140
|
+
Given I am signed in as an admin
|
|
141
|
+
When I create a customer named "cust-{unique:name}" through the API
|
|
142
|
+
And I open the customers page
|
|
143
|
+
Then the customer I created should be listed
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
The request goes through the browser session's own client, so it carries
|
|
147
|
+
the cookies the UI signed in with, and a value it kept can be checked on
|
|
148
|
+
the page. Keep each step either a browser step or an API step. The agent
|
|
149
|
+
has the API tools in a browser scenario only when `saffron.config.json`
|
|
150
|
+
sets `api` (`"api": {}` is enough), or once the scenario's replay has
|
|
151
|
+
called the API.
|
|
@@ -69,7 +69,9 @@ your-project/
|
|
|
69
69
|
| `provider` | Whose agent records and heals, on its login on this machine: `claude` (default), `codex` (ChatGPT plan), `antigravity` (Google AI Pro/Ultra via the `agy` CLI, experimental), `cursor`. Also `--provider` |
|
|
70
70
|
| `model` | Model in the provider's own names (unset: the provider's default). A `--provider` run on another provider ignores it |
|
|
71
71
|
| `healModel` | Cheaper model for heal sessions only |
|
|
72
|
+
| `antigravityApiKey` | `true`: Antigravity signs in with `GEMINI_API_KEY` even beside an agy login (default: a login wins) |
|
|
72
73
|
| `ci` | Where `saffron runs` and `import --run` find CI runs: `{"provider": "github" \| "azure", "organization": ..., "project": ...}`. Unset, an Azure Repos remote means Azure Pipelines (through `az`), anything else GitHub Actions (through `gh`). `organization` and `project` are for a GitHub repository built by Azure Pipelines |
|
|
74
|
+
| `api` | The API under test: `{"baseURL": ..., "contract": "api/openapi.json", "transcript": "on-failure"}`. `baseURL` is where API steps' relative paths go (unset: the run's `baseURL`); `contract` is the pinned OpenAPI 3 document schema checks validate against (`saffron api pin` writes it), and a change to it, or to which file it names, makes the recordings checked against it stale; `transcript` keeps each scenario's requests and responses for the report and `saffron trace`: `"on-failure"` (default), `"on"` or `"off"` (statuses and timings only); `services` names more APIs, each `{"baseURL": ..., "contract": ...}`, which a request names instead of a host (`--api-url name=<url>` points one elsewhere for a run). See [API tests](api.md) |
|
|
73
75
|
|
|
74
76
|
## CLI
|
|
75
77
|
|
|
@@ -84,7 +86,8 @@ your-project/
|
|
|
84
86
|
| `saffron steps [search] [--json] [--snippets]` | The step vocabulary with recorded/divergent/unrecorded badges |
|
|
85
87
|
| `saffron author <prose-file> [--provider p]` | Draft a `.saffron` file from plain-paragraph requirements using the project vocabulary |
|
|
86
88
|
| `saffron login [provider]` | Check that a provider can record and heal here and list what is missing; `saffron login cursor` also runs Cursor's one-time sign-in, `saffron login antigravity` also allows Saffron's tools in agy |
|
|
87
|
-
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`) for AI assistants / language server for editors |
|
|
89
|
+
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `api_contract`) for AI assistants / language server for editors |
|
|
90
|
+
| `saffron api pin <source> [--out file] [--service name]` | Pin the API contract: fetch an OpenAPI 3 document from a Swagger UI page, a document URL or a file, and write it as JSON (default `api/openapi.json`, or `api/<name>.json` with `--service`). Add it to the config as `api.contract`, or as `api.services.<name>.contract` |
|
|
88
91
|
| `saffron init [--examples] [--agents list]` | Install this skill into the project's agent directories, register MCP, add an AGENTS.md block, scaffold config; `--examples` adds the Saucedemo example suite |
|
|
89
92
|
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs, 1 GB, fixed), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
|
|
90
93
|
| `saffron dashboard [--stdout] [--no-open]` | The suite's quality from the run history (failing and for how long, flaky, repeatedly healed, never run, slowest, trends, per tag); writes and opens `.saffron/reports/dashboard.html`, `--stdout` prints it |
|
|
@@ -98,11 +101,11 @@ one line (`SAFFRON_DEBUG=1` shows the stack).
|
|
|
98
101
|
|
|
99
102
|
## AI access
|
|
100
103
|
|
|
101
|
-
Recording and healing need the configured provider's login on the machine
|
|
102
|
-
Claude (default): `ANTHROPIC_API_KEY`, or a Claude Code login. Codex: a
|
|
103
|
-
ChatGPT-plan `npx @openai/codex login` plus `npm i -D @openai/codex-sdk`. Antigravity:
|
|
104
|
-
Google's `agy` CLI, signed in once
|
|
105
|
-
(allows Saffron's tools in agy's settings). Cursor: `npm i -D @cursor/sdk`
|
|
104
|
+
Recording and healing need the configured provider's login on the machine,
|
|
105
|
+
or its API key. Claude (default): `ANTHROPIC_API_KEY`, or a Claude Code login. Codex: a
|
|
106
|
+
ChatGPT-plan `npx @openai/codex login`, or `CODEX_API_KEY` (or `OPENAI_API_KEY`), plus `npm i -D "@openai/codex-sdk@>=0.158.0"`. Antigravity:
|
|
107
|
+
Google's `agy` CLI, signed in once or with `GEMINI_API_KEY`, then `npx saffron login antigravity`
|
|
108
|
+
(allows Saffron's tools in agy's settings). A Codex or Antigravity login wins over a key. Cursor: `npm i -D @cursor/sdk`
|
|
106
109
|
and `npx saffron login cursor`. `npx saffron login` lists what is missing.
|
|
107
110
|
Replay-only runs (`--no-agent`) need none: that is the normal CI mode once
|
|
108
111
|
caches are committed. With no `model` configured the provider's default
|
|
@@ -126,6 +129,9 @@ npx playwright install chromium
|
|
|
126
129
|
npx saffron run --no-agent --strict # replay committed caches; no AI, no surprises
|
|
127
130
|
```
|
|
128
131
|
|
|
132
|
+
A job that runs only `@api` scenarios (`--filter @api`) starts no browser,
|
|
133
|
+
so it can skip `npx playwright install`.
|
|
134
|
+
|
|
129
135
|
Record new scenarios on developer machines. A CI job that may heal (it
|
|
130
136
|
has the provider's login or key) runs without `--no-agent`, and its heals
|
|
131
137
|
travel to the reviewer as a bundle, never lost with the CI workspace:
|
|
@@ -187,6 +187,10 @@ Recorded as URL-pattern + method + status (+ body pattern) matchers;
|
|
|
187
187
|
waits are satisfied by responses from the triggering step onward, so
|
|
188
188
|
"click and wait" never races. Prefer these over any fixed sleep.
|
|
189
189
|
|
|
190
|
+
These watch the requests the page makes. A step that sends a request
|
|
191
|
+
itself ("When I create a customer through the API") is an API step: see
|
|
192
|
+
[API tests](api.md).
|
|
193
|
+
|
|
190
194
|
JSON response fields support typed equality and presence checks:
|
|
191
195
|
|
|
192
196
|
```gherkin
|