saffron-ai 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1159 -2
- package/README.md +7 -5
- package/dist-pkg/cli.js +473 -451
- package/package.json +27 -6
- package/skills/saffron/SKILL.md +40 -18
- package/skills/saffron/references/api.md +3 -1
- package/skills/saffron/references/config.md +1 -1
- package/skills/saffron/references/database.md +173 -10
- package/skills/saffron/references/syntax.md +35 -12
- package/templates/playwright/saffron-helpers.ts +608 -110
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.1",
|
|
4
4
|
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
"dev": "tsc --watch",
|
|
9
9
|
"e2e": "node scripts/e2e.mjs && node scripts/e2e-eject.mjs",
|
|
10
10
|
"benchmark": "node scripts/benchmark.mjs",
|
|
11
|
-
"
|
|
11
|
+
"gate": "node scripts/gate.mjs",
|
|
12
|
+
"prepublishOnly": "npm run gate",
|
|
12
13
|
"prepack": "npm run bundle",
|
|
13
14
|
"docs": "node scripts/build-compat.mjs && node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
|
|
14
15
|
"bundle": "esbuild src/cli/index.ts --bundle --platform=node --format=esm --packages=external --minify --legal-comments=none --outfile=dist-pkg/cli.js",
|
|
@@ -45,16 +46,20 @@
|
|
|
45
46
|
"saffron": "dist-pkg/cli.js"
|
|
46
47
|
},
|
|
47
48
|
"devDependencies": {
|
|
49
|
+
"@aws-sdk/client-dynamodb": "^3.1166.0",
|
|
48
50
|
"@cursor/sdk": "^1.0.32",
|
|
51
|
+
"@google-cloud/firestore": "^7.11.6",
|
|
49
52
|
"@openai/codex-sdk": "^0.158.0",
|
|
50
53
|
"@types/mssql": "^12.3.0",
|
|
51
54
|
"@types/node": "^26.1.0",
|
|
52
55
|
"@types/pg": "^8.23.1",
|
|
53
56
|
"esbuild": "^0.28.1",
|
|
54
57
|
"marked": "^18.0.5",
|
|
58
|
+
"mongodb": "^7.7.0",
|
|
55
59
|
"mssql": "^12.7.14",
|
|
56
60
|
"mysql2": "^3.24.6",
|
|
57
61
|
"node-sql-parser": "^5.4.0",
|
|
62
|
+
"oracledb": "^6.10.0",
|
|
58
63
|
"typescript": "^6.0.3",
|
|
59
64
|
"vitest": "^4.1.9"
|
|
60
65
|
},
|
|
@@ -67,7 +72,7 @@
|
|
|
67
72
|
"ajv": "^8.17.1",
|
|
68
73
|
"ajv-formats": "^3.0.1",
|
|
69
74
|
"commander": "^15.0.0",
|
|
70
|
-
"playwright": "^1.
|
|
75
|
+
"playwright": "^1.65.1",
|
|
71
76
|
"vscode-languageserver": "^10.1.0",
|
|
72
77
|
"vscode-languageserver-textdocument": "^1.0.12",
|
|
73
78
|
"yaml": "^2.9.1",
|
|
@@ -97,9 +102,13 @@
|
|
|
97
102
|
"pg": "^8.23.1"
|
|
98
103
|
},
|
|
99
104
|
"peerDependencies": {
|
|
100
|
-
"
|
|
101
|
-
"
|
|
102
|
-
"
|
|
105
|
+
"@aws-sdk/client-dynamodb": "*",
|
|
106
|
+
"@google-cloud/firestore": "*",
|
|
107
|
+
"mongodb": "*",
|
|
108
|
+
"mssql": "*",
|
|
109
|
+
"mysql2": "*",
|
|
110
|
+
"node-sql-parser": "*",
|
|
111
|
+
"oracledb": "*"
|
|
103
112
|
},
|
|
104
113
|
"peerDependenciesMeta": {
|
|
105
114
|
"node-sql-parser": {
|
|
@@ -110,6 +119,18 @@
|
|
|
110
119
|
},
|
|
111
120
|
"mysql2": {
|
|
112
121
|
"optional": true
|
|
122
|
+
},
|
|
123
|
+
"mongodb": {
|
|
124
|
+
"optional": true
|
|
125
|
+
},
|
|
126
|
+
"oracledb": {
|
|
127
|
+
"optional": true
|
|
128
|
+
},
|
|
129
|
+
"@aws-sdk/client-dynamodb": {
|
|
130
|
+
"optional": true
|
|
131
|
+
},
|
|
132
|
+
"@google-cloud/firestore": {
|
|
133
|
+
"optional": true
|
|
113
134
|
}
|
|
114
135
|
}
|
|
115
136
|
}
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -3,14 +3,14 @@ name: saffron
|
|
|
3
3
|
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
|
-
saffron-ai: "1.4.
|
|
6
|
+
saffron-ai: "1.4.1"
|
|
7
7
|
author: saffron-ai
|
|
8
8
|
homepage: https://saffron-ai.io
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
# Saffron
|
|
12
12
|
|
|
13
|
-
This skill describes saffron-ai 1.4.
|
|
13
|
+
This skill describes saffron-ai 1.4.1 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
|
|
14
14
|
|
|
15
15
|
Apply this workflow only when asked to create, translate, or edit Saffron
|
|
16
16
|
scenarios in `.saffron` or `.feature` files. A Saffron mention or an
|
|
@@ -52,7 +52,7 @@ replays for free.
|
|
|
52
52
|
**Before writing any step, look at what the project already knows:**
|
|
53
53
|
|
|
54
54
|
```bash
|
|
55
|
-
npx saffron steps # every step: ● recorded,
|
|
55
|
+
npx saffron steps # every step: ● recorded, ◐ divergent, ○ unrecorded
|
|
56
56
|
npx saffron steps "login" # search
|
|
57
57
|
npx saffron steps --json # machine-readable
|
|
58
58
|
```
|
|
@@ -188,8 +188,10 @@ count never change.
|
|
|
188
188
|
`I record the basket total as "before"`, `I record the breakfast price as
|
|
189
189
|
"breakfast"`, then `Then the basket total should be "before" plus
|
|
190
190
|
"breakfast"`. Replay reads the one number each element shows, in either
|
|
191
|
-
decimal style, and compares to the cent
|
|
192
|
-
|
|
191
|
+
decimal style, and compares to the cent, or to the decimals the element
|
|
192
|
+
shows when it shows more: a rate shown as `0.1200` is not `0.1204`.
|
|
193
|
+
Minus and "times 2" work too; percentages, rounding rules and taxes do
|
|
194
|
+
not.
|
|
193
195
|
- **Check the route, not only the destination.** When the requirement is
|
|
194
196
|
a path (a 3DS challenge, a redirect, a confirmation dialog), add a
|
|
195
197
|
mid-scenario `Then` for each checkpoint, such as `Then I should see the
|
|
@@ -211,7 +213,9 @@ count never change.
|
|
|
211
213
|
```
|
|
212
214
|
|
|
213
215
|
Each cell records as its `<table:key>`, so editing a value replays at
|
|
214
|
-
zero AI
|
|
216
|
+
zero AI, unless the old value also stands inside a word the recording
|
|
217
|
+
holds (`Pro` in "Products"), which records the step once more; renaming
|
|
218
|
+
or adding a key re-records the step. Use a header
|
|
215
219
|
table for a list of records, and a `"""` doc string for free text or a
|
|
216
220
|
whole JSON body. Keep one or two values inline when they read as part of
|
|
217
221
|
the sentence (`When I read customer "cust-42" through the API`).
|
|
@@ -260,9 +264,11 @@ expected behavior in its input. Its output must still go through steps
|
|
|
260
264
|
|
|
261
265
|
```gherkin
|
|
262
266
|
Given As "seller" I am on the orders page
|
|
263
|
-
|
|
267
|
+
# the buyer's own browser
|
|
268
|
+
When As "buyer" I order the blue lamp
|
|
264
269
|
Then As "seller" I see an order for the blue lamp
|
|
265
|
-
|
|
270
|
+
# still the seller
|
|
271
|
+
And the order is marked new
|
|
266
272
|
```
|
|
267
273
|
|
|
268
274
|
Each name is its own browser session (cookies, login, tabs). It carries
|
|
@@ -309,7 +315,7 @@ Scenario: An upgrade is saved
|
|
|
309
315
|
And no refund should exist in the database for "ann-{unique:c}@example.test"
|
|
310
316
|
```
|
|
311
317
|
|
|
312
|
-
With `db` in `saffron.config.json` (PostgreSQL, MySQL
|
|
318
|
+
With `db` in `saffron.config.json` (PostgreSQL, MySQL, SQL Server, Oracle, SQLite, MariaDB, CockroachDB, MongoDB, DynamoDB or Firestore), a `Then` can check
|
|
313
319
|
what the database holds and a `Given` or `When` can seed data, in browser and `@api`
|
|
314
320
|
scenarios alike. Name the database in a check ("in the database") and name
|
|
315
321
|
the record by a value the scenario wrote: the step's text, an Examples
|
|
@@ -317,8 +323,13 @@ column, a `{unique:...}` value, or a value an earlier step kept. A check
|
|
|
317
323
|
bound to nothing ("a customer exists") is refused while recording. A
|
|
318
324
|
"not in the database" check waits out `db.timeout` (3 s by default) before
|
|
319
325
|
it passes. Seeding needs `"writes": true` and the database's host or name
|
|
320
|
-
in `allow
|
|
321
|
-
|
|
326
|
+
in `allow` (a SQLite file's name, a DynamoDB region, a Firestore project);
|
|
327
|
+
seed what the app gives no way to set, with `{unique:...}` values, and the
|
|
328
|
+
rows are undone when the scenario ends. A step never holds a query: the
|
|
329
|
+
recording agent writes it, as SQL or, on MongoDB, DynamoDB and Firestore,
|
|
330
|
+
as one JSON command. On DynamoDB name the item by the value of its key
|
|
331
|
+
(or of an index's key), since there is no scan; on Firestore name the
|
|
332
|
+
document by its id or by a field's value. Read
|
|
322
333
|
[Database steps](references/database.md) for the details.
|
|
323
334
|
|
|
324
335
|
## Repeated step groups → StepSets (`.saffron` files only)
|
|
@@ -327,7 +338,8 @@ values, and the rows are undone when the scenario ends. Read
|
|
|
327
338
|
changes, and you gain the `StepSet:` keyword.
|
|
328
339
|
|
|
329
340
|
```gherkin
|
|
330
|
-
|
|
341
|
+
# define: colon, like Scenario:
|
|
342
|
+
StepSet: Complete guest information
|
|
331
343
|
Given I am on the guest information page
|
|
332
344
|
When I enter guest name "Chathuranga"
|
|
333
345
|
And I click the continue button
|
|
@@ -335,12 +347,16 @@ StepSet: Complete guest information # define: colon, like Scenario:
|
|
|
335
347
|
|
|
336
348
|
Scenario: Checkout happy path
|
|
337
349
|
Given I am on the cart page
|
|
338
|
-
|
|
350
|
+
# invoke: no colon, like a step
|
|
351
|
+
StepSet Complete guest information
|
|
339
352
|
Then I am on the checkout page
|
|
340
353
|
```
|
|
341
354
|
|
|
342
|
-
- Definition: `StepSet: <name>`. Invocation: `StepSet <name
|
|
343
|
-
name without the keyword is a
|
|
355
|
+
- Definition: `StepSet: <name>`. Invocation: `StepSet <name>`. The bare
|
|
356
|
+
name without the keyword is refused after a step, and read as
|
|
357
|
+
description, so left out, above a scenario's first step.
|
|
358
|
+
- A comment is a line of its own starting with `#`. After a step on the
|
|
359
|
+
same line it is part of the step's text.
|
|
344
360
|
- Names are project-wide unique; sets resolve across files; no nesting.
|
|
345
361
|
- Application-wide flows (login, cookie banner) live in a sets-only
|
|
346
362
|
library file: `features/shared.steps.saffron` (still needs a
|
|
@@ -379,9 +395,11 @@ step. That includes dates without a year and one- or two-character
|
|
|
379
395
|
values. A blank cell is refused the same way, or, when its step types
|
|
380
396
|
`<column>`, the first filled row records once more and serves the blank
|
|
381
397
|
rows too. A value the page sets in its own way (a phone or IBAN in
|
|
382
|
-
blocks, an id zero-padded inside a longer one like `INV-000104
|
|
383
|
-
|
|
384
|
-
|
|
398
|
+
blocks, an id zero-padded inside a longer one like `INV-000104`, a date
|
|
399
|
+
cell shown in another format, an amount shown with other decimals, a data
|
|
400
|
+
row's value with outer spaces shown trimmed) binds the recording to its
|
|
401
|
+
row the same way: give the cell the value as the page shows it, or give
|
|
402
|
+
the shown value an Examples column of its own. Keep a check that lists all rows' values out of the Outline.
|
|
385
403
|
Tests live in the folder named by `features` in `saffron.config.json`
|
|
386
404
|
(`features/` by default).
|
|
387
405
|
|
|
@@ -421,6 +439,10 @@ Use Saffron itself first: `{date...}` tokens (ranges and locales included), capt
|
|
|
421
439
|
| Check an API response's status in a `When` | Name the status in a `Then` ("should answer 404") |
|
|
422
440
|
| One long step line carrying an API key, ids and several quoted parameters | A short step and a 2-column data table under it, one value per row |
|
|
423
441
|
| A database check bound to nothing ("Then a customer exists in the database") | Name the record the scenario made: `Then the customer "ann-{unique:c}@example.test" should be saved in the database` |
|
|
442
|
+
| SQL or a JSON command written in a step | Say what should be stored, in plain words; the recording agent writes the query for the project's database |
|
|
443
|
+
| `scenario:` in lower case, `Scenario :`, or a tab after a step keyword | Spell headers and keywords exactly: Saffron refuses the file, where Gherkin would drop the scenario silently |
|
|
444
|
+
| `When StepSet Login` | A call is a line of its own: `StepSet Login` |
|
|
445
|
+
| `--filter "@smoke && !@wip"` | `--filter "@smoke and not @wip"`: the operators are the words |
|
|
424
446
|
|
|
425
447
|
## References
|
|
426
448
|
|
|
@@ -88,7 +88,9 @@ Feature: Customers API
|
|
|
88
88
|
- **Name the status of an error case** in the request's step or the
|
|
89
89
|
`Then` right after it ("should answer
|
|
90
90
|
404", "is refused with 409", "I get a 404", "the customer is not
|
|
91
|
-
found"): a number elsewhere ("I order 450 units") is no status. A
|
|
91
|
+
found"): a number elsewhere ("I order 450 units") is no status. A status
|
|
92
|
+
the step names only to rule it out ("should not answer 500") is not
|
|
93
|
+
named: a response with it fails the step. A response that is no
|
|
92
94
|
2xx is recorded only when its step names that status; otherwise the
|
|
93
95
|
step fails, so a failure seen while recording never replays as the
|
|
94
96
|
expected result.
|
|
@@ -72,7 +72,7 @@ your-project/
|
|
|
72
72
|
| `healModel` | Cheaper model for heal sessions only |
|
|
73
73
|
| `antigravityApiKey` | `true`: Antigravity signs in with `GEMINI_API_KEY` even beside an agy login (default: a login wins) |
|
|
74
74
|
| `ci` | Where `saffron runs` and `import --run` find CI runs: `{"provider": "github" \| "azure", "organization": ..., "project": ...}`. Unset, an Azure Repos remote means Azure Pipelines (through `az`), anything else GitHub Actions (through `gh`). `organization` and `project` are for a GitHub repository built by Azure Pipelines. `cacheRuns` and `cacheMB` cap the runs viewed into `.saffron/ci/` (default 10 and 1024) |
|
|
75
|
-
| `db` | The database that database steps check and seed (PostgreSQL, MySQL
|
|
75
|
+
| `db` | The database that database steps check and seed (PostgreSQL, MySQL, SQL Server, Oracle, SQLite, MariaDB, CockroachDB, MongoDB, DynamoDB or Firestore, by the URL's scheme; `npm install node-sql-parser` and `pg`, `mysql2`, `mssql` or `oracledb@6` for a SQL engine, none for SQLite's driver, or `mongodb`, `@aws-sdk/client-dynamodb` or `@google-cloud/firestore@7` alone): `{"url": "{env:DATABASE_URL}", "writes": false, "allow": ["localhost"]}`. `url` must be an `{env:...}` reference; `writes` (default `false`) lets `Given` and `When` steps insert and update with `dbWrite`; `allow` lists the hosts or database names a write may go to; `timeout` (default 3000 ms) is how long a check reads again, and a "not in the database" check must hold that long; `cleanup` is `"all"` (default: delete inserts, put updated rows back), `"inserts"` or `"none"`. A `Then` records `expectRows`: one SELECT, values as `$n` binds bound to a scenario value, checks on its rows; a check whose query fails on the schema may have its query repaired by the agent, its binds, checks and row count unchanged |
|
|
76
76
|
| `api` | The API under test: `{"baseURL": ..., "contract": "api/openapi.json", "transcript": "on-failure"}`. `baseURL` is where API steps' relative paths go (unset: the run's `baseURL`); `contract` is the pinned OpenAPI 3 document schema checks validate against (`saffron api pin` writes it), and a change to it, or to which file it names, makes the recordings checked against it stale; `transcript` keeps each scenario's requests and responses for the report and `saffron trace`: `"on-failure"` (default), `"on"` or `"off"` (statuses and timings only); `services` names more APIs, each `{"baseURL": ..., "contract": ...}`, which a request names instead of a host (`--api-url name=<url>` points one elsewhere for a run). A key Saffron does not read under `api` or a service is refused. See [API tests](api.md) |
|
|
77
77
|
|
|
78
78
|
## CLI
|
|
@@ -5,8 +5,11 @@ starts from, with no query written by hand. While recording, the agent
|
|
|
5
5
|
reads the schema, writes one parameterized statement, runs it, and records
|
|
6
6
|
it with the step's values as binds; every later run replays it at zero
|
|
7
7
|
tokens. They work in browser scenarios and in `@api` scenarios alike.
|
|
8
|
-
PostgreSQL, MySQL
|
|
9
|
-
|
|
8
|
+
PostgreSQL, MySQL, SQL Server, Oracle, SQLite, MariaDB and CockroachDB
|
|
9
|
+
with SQL, and MongoDB, DynamoDB and Firestore with one JSON command per
|
|
10
|
+
step, named by the URL's scheme (`postgres://`, `mysql://`, `mssql://`,
|
|
11
|
+
`oracle://`, `sqlite:path/to/app.db`, `mariadb://`, `cockroachdb://`,
|
|
12
|
+
`mongodb://`, `dynamodb://<region>`, `firestore://<project-id>`).
|
|
10
13
|
|
|
11
14
|
## Setup
|
|
12
15
|
|
|
@@ -16,8 +19,11 @@ PostgreSQL, MySQL and SQL Server, named by the URL's scheme
|
|
|
16
19
|
}
|
|
17
20
|
```
|
|
18
21
|
|
|
19
|
-
- `npm install node-sql-parser` and the engine's driver (`pg`, `mysql2
|
|
20
|
-
`mssql`
|
|
22
|
+
- `npm install node-sql-parser` and the engine's driver (`pg`, `mysql2`,
|
|
23
|
+
`mssql` or `oracledb@6`; none for SQLite, which Node has built in) in the
|
|
24
|
+
project; for MongoDB, DynamoDB or Firestore, their driver alone
|
|
25
|
+
(`mongodb`, `@aws-sdk/client-dynamodb`, `@google-cloud/firestore@7`). They
|
|
26
|
+
load only when `db` is set.
|
|
21
27
|
- Every engine takes `$1`, `$2` binds; write the engine's own SQL
|
|
22
28
|
otherwise. On MySQL and SQL Server an `INSERT` with an auto-numbered key
|
|
23
29
|
writes one row (several rows give the key as binds), and an `UPDATE`
|
|
@@ -51,6 +57,12 @@ Scenario: A new customer is saved
|
|
|
51
57
|
left behind, and is refused while recording. A value spelled in the
|
|
52
58
|
step is the same every run: where the check must prove this run saved
|
|
53
59
|
the row, use a `{unique:...}` value or an id an earlier step kept.
|
|
60
|
+
- A step says what should be stored, never how to query it. The agent
|
|
61
|
+
writes SQL on a SQL engine and one JSON command on MongoDB, DynamoDB
|
|
62
|
+
and Firestore. On DynamoDB the value that names the record must be its
|
|
63
|
+
key, or an index's key (there is no scan): "the order "A-{unique:o}" of
|
|
64
|
+
customer "ann@example.test"". On Firestore it is the document's id or a
|
|
65
|
+
field's value.
|
|
54
66
|
- Seed with `{unique:...}` values where a column is unique: parallel
|
|
55
67
|
workers and proof replays each write their own.
|
|
56
68
|
- A seed `Given` states the data, not how to insert it. Seed only what the
|
|
@@ -76,7 +88,14 @@ Scenario: A new customer is saved
|
|
|
76
88
|
range alone, a bind in `LIMIT`, and `UNION` are refused, and a subquery
|
|
77
89
|
must tie its own rows. A `LIKE` value made only of `%` or `_` (an
|
|
78
90
|
Examples cell too) matches every row, so it is refused unless another
|
|
79
|
-
condition ties the rows.
|
|
91
|
+
condition ties the rows. In a "not in the database" check a `LIKE` value
|
|
92
|
+
holding a backslash (a `[` on SQL Server) is refused: compare that text
|
|
93
|
+
with `=`. Placeholders start at `$1`. Such a check is red when rows are
|
|
94
|
+
found once a bind is sent as the other type (text as a number, `true` as
|
|
95
|
+
1 or as a boolean, 1 as a boolean, `0x` hex as bytes, a JSON bind's
|
|
96
|
+
values, a value read as JSON as a JSON string): give each bind the type
|
|
97
|
+
the database keeps, as the refusal says. A bind only a negation compares
|
|
98
|
+
(`<> $2`) is left as sent. A JSON field can tie a check
|
|
80
99
|
(`data->>'orderRef' = $1`). Cast with `CAST($1 AS type)`, not `::`. A
|
|
81
100
|
`WITH` that selects a bind as a column is refused like a `SELECT $1`.
|
|
82
101
|
On MySQL join texts with `CONCAT()`: `||` runs as `OR` there and is
|
|
@@ -87,12 +106,29 @@ Scenario: A new customer is saved
|
|
|
87
106
|
refused, unless every row holds it alike. An aggregate with no `GROUP BY` is always one row: check its
|
|
88
107
|
value (`/0/n equals 1`, or a count on a list it holds), never a count
|
|
89
108
|
or an exists on the row itself, also when the aggregate is in the
|
|
90
|
-
query's only `FROM` or `WITH` source. A count above 0,
|
|
91
|
-
there is no second row, under `LIMIT`, `TOP`, `FETCH`, `DISTINCT
|
|
109
|
+
query's only `FROM` or `WITH` source. A count above 0, an `every`
|
|
110
|
+
check on the rows, or a check that there is no second row, under `LIMIT`, `TOP`, `FETCH`, `DISTINCT`, a
|
|
111
|
+
WHERE that picks one of the table's own rows by a subquery (`id = (SELECT
|
|
112
|
+
max(id) FROM orders WHERE ref = $1)`, `>= ALL (...)`), a WHERE that
|
|
113
|
+
keeps one row of each group (`max(id) ... GROUP BY`, a `row_number()`
|
|
114
|
+
column) or a
|
|
92
115
|
`GROUP BY` with no aggregate is refused, also when they sit in a query
|
|
93
116
|
read `FROM` or `WITH`, and so is any check under `OFFSET`, and `LIMIT
|
|
94
|
-
0`, `TOP 0` and `FETCH ... 0`. A
|
|
95
|
-
|
|
117
|
+
0`, `TOP 0` and `FETCH ... 0`. A row holds every column it selects,
|
|
118
|
+
NULL too: a column that must be empty is `discount IS NULL` in the
|
|
119
|
+
`WHERE`, never `/0/discount` exists false, which is refused. A field
|
|
120
|
+
that must not be in a JSON column (`/0/data/discount`) holds only when
|
|
121
|
+
that row is there.
|
|
122
|
+
- A "differs" check reads text that spells a number as the number (a
|
|
123
|
+
decimal, a bigint and a count come back as text): `12.50` does not
|
|
124
|
+
differ from `12.5`. Where the text itself matters (a version `1.10`),
|
|
125
|
+
give the check `"type": "string"`; a code with a leading zero (`01234`)
|
|
126
|
+
is compared as text already. An "at most" check with an empty or null
|
|
127
|
+
bind is refused, as a "not in the database" check is: to count rows
|
|
128
|
+
whose column is rightly empty, compare its length with the number 0
|
|
129
|
+
(`length(note) = $1`); on Oracle an empty text is NULL, so write
|
|
130
|
+
`note IS NULL` there, beside a column compared with a bind
|
|
131
|
+
(`note IS NULL AND email = $1`).
|
|
96
132
|
- A check reads in a transaction that is rolled back (read-only on
|
|
97
133
|
PostgreSQL and MySQL). A date, a timestamp, a bigint or a numeric reads
|
|
98
134
|
as text in UTC (`2026-11-07`, `2026-11-07 09:00:00+00` on PostgreSQL),
|
|
@@ -158,7 +194,134 @@ replay; the step is reported healed and a proposal names the old and new
|
|
|
158
194
|
SQL. A query that runs and finds other rows is plain red, and no agent is
|
|
159
195
|
started.
|
|
160
196
|
|
|
197
|
+
## MongoDB
|
|
198
|
+
|
|
199
|
+
A MongoDB step holds one JSON command where SQL has a statement, every
|
|
200
|
+
value a placeholder bound from the step:
|
|
201
|
+
|
|
202
|
+
```json
|
|
203
|
+
{"find": "customers", "filter": {"email": "$1"}, "projection": {"name": 1}}
|
|
204
|
+
{"count": "orders", "filter": {"customer": "$1:objectId"}}
|
|
205
|
+
{"insert": "customers", "documents": [{"name": "$1", "email": "$2"}]}
|
|
206
|
+
{"update": "customers", "filter": {"email": "$1"}, "set": {"tier": "$2"}}
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
- A `count` returns one row, `{"n": <how many>}`: check `/0/n` equals `1`
|
|
210
|
+
with type `number`.
|
|
211
|
+
- A placeholder names the BSON type it is sent as: `"$1:objectId"`,
|
|
212
|
+
`"$1:uuid"`, `"$1:number"`, `"$1:long"`, `"$1:decimal"`, `"$1:bool"`,
|
|
213
|
+
`"$1:date"`, `"$1:binary"` (bytes as `0x` hex). A bind given as null
|
|
214
|
+
finds a field held as null (or missing); `{"$exists": false}` finds only
|
|
215
|
+
a document without the field.
|
|
216
|
+
- Filters use `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`,
|
|
217
|
+
`$exists`, `$all`, `$elemMatch`, `$not`, `$and`, `$or` and `$nor`;
|
|
218
|
+
`$where`, `$expr`, `$regex` and aggregation are refused. A field equal
|
|
219
|
+
to a bind, or `$in` binds, ties a check, on each side of an `$or`.
|
|
220
|
+
Compare embedded fields with dotted paths (`"addr.city": "$1"`), never a
|
|
221
|
+
whole document or list as a value, plainly or under `$eq`, `$ne`, `$in`,
|
|
222
|
+
`$nin` or `$all`. Placeholders start at `$1`.
|
|
223
|
+
- A "not in the database" check is red when its filter finds a document
|
|
224
|
+
with a bind sent as another type (an ObjectId's hex as text), when no
|
|
225
|
+
document has a field the filter finds documents by (compared with a
|
|
226
|
+
bind, in a range, under `$elemMatch`, or `$exists` true) or a field a
|
|
227
|
+
check says must not be there, when
|
|
228
|
+
`$elemMatch` reads a field held as one value, or when the collection is
|
|
229
|
+
not there, so type each bind as its field holds it, both ends of a range
|
|
230
|
+
too.
|
|
231
|
+
- A `find` with a `projection` cannot check that a field is missing, at
|
|
232
|
+
any depth, a projection names fields only (no `lines.$`), and `every` or
|
|
233
|
+
a count above 0 under `limit` is refused.
|
|
234
|
+
- An `update` never sets `_id`; its `set` may name dotted paths
|
|
235
|
+
(`"addr.city": "$2"`, a list's item by its number, `"items.0.qty"`),
|
|
236
|
+
never `$`, `$[]` or `$[name]`, and an inserted document writes a nested
|
|
237
|
+
field as a nested object.
|
|
238
|
+
- An ObjectId reads as its hex, a UUID as its dashed text, a date as UTC
|
|
239
|
+
ISO text, an int64 or a decimal as text. `extract` keeps a new `_id` with `/_id`.
|
|
240
|
+
- Cleanup deletes inserted documents and puts back only the paths an
|
|
241
|
+
update set, type for type. A MongoDB check is never repaired: a collection has no schema.
|
|
242
|
+
|
|
243
|
+
## Oracle and SQLite
|
|
244
|
+
|
|
245
|
+
- Oracle: names are in capitals unless created in quotes, so check a
|
|
246
|
+
row's columns as `/0/NAME`; a name created in quotes cannot be used.
|
|
247
|
+
Write plain SQL: `FETCH FIRST` and `ROWNUM` are a `LIMIT`, and `CONNECT
|
|
248
|
+
BY`, `(+)`, `AS OF`, `q'...'` strings, `MERGE` and PL/SQL are refused. A
|
|
249
|
+
`DATE` holds a time (`2026-11-07 00:00:00`), a `NUMBER` reads as a
|
|
250
|
+
number, decimals too (only one past 15 digits reads as text), and an
|
|
251
|
+
empty string is `NULL`. A check never calls `NEXTVAL`. An `INSERT` writes one row when Oracle
|
|
252
|
+
numbers its key.
|
|
253
|
+
- SQLite: `db.url` is `sqlite:path/to/app.db`, a file that must be there,
|
|
254
|
+
and `db.allow` lists its name or path. A boolean is `1` or `0`; a date
|
|
255
|
+
or JSON is the text the app wrote. A bind compared with an expression
|
|
256
|
+
(`json_extract(data, '$.id') = $1`) must be the type the expression
|
|
257
|
+
gives, or cast both to one type: a "not in the database" check that
|
|
258
|
+
only a type keeps empty is refused. `INSERT OR REPLACE`, `UPDATE OR ...`
|
|
259
|
+
and `REPLACE` are refused.
|
|
260
|
+
- MariaDB runs as MySQL and CockroachDB as PostgreSQL, by the same rules.
|
|
261
|
+
|
|
262
|
+
## DynamoDB
|
|
263
|
+
|
|
264
|
+
One JSON command per step, every value a placeholder:
|
|
265
|
+
|
|
266
|
+
```json
|
|
267
|
+
{"get": "Orders", "key": {"customer": "$1", "orderId": "$2"}}
|
|
268
|
+
{"query": "Orders", "key": {"customer": "$1", "orderId": {"$beginsWith": "$2"}}, "filter": {"status": "$3"}}
|
|
269
|
+
{"put": "Orders", "item": {"customer": "$1", "orderId": "$2", "total": "$3:number"}}
|
|
270
|
+
{"update": "Orders", "key": {"customer": "$1", "orderId": "$2"}, "set": {"status": "$3"}}
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
- `db.url` is `dynamodb://<region>`, or `dynamodb://<host>:<port>` for
|
|
274
|
+
DynamoDB Local; AWS credentials come from the environment, never the
|
|
275
|
+
URL. `db.allow` lists the region or the local host.
|
|
276
|
+
- A `get` names the whole key; a `query` names its partition key by
|
|
277
|
+
equality, with a sort key condition (`$eq`, `$gt`, `$gte`, `$lt`,
|
|
278
|
+
`$lte`, `$between`, `$beginsWith`), an `index`, a `filter` (those and
|
|
279
|
+
`$ne`, `$in`, `$exists`, on top-level attributes) or a `limit`. No
|
|
280
|
+
scan, no delete.
|
|
281
|
+
- A key attribute is sent as the type the table declares. Any other
|
|
282
|
+
attribute that is not text names its type: `"$1:number"`, `"$1:bool"`,
|
|
283
|
+
`"$1:binary"` (bytes as `0x` hex). A bind given as null finds an
|
|
284
|
+
attribute held as NULL. A "not in the database" check is red when an
|
|
285
|
+
item is found with a bind sent as another type (both ends of a range
|
|
286
|
+
too), or when it compares a set or a list with a value, and never reads
|
|
287
|
+
a global secondary index.
|
|
288
|
+
- A `put` never replaces an item; an `update` changes one item that is
|
|
289
|
+
there, never a key attribute. Cleanup deletes the put item and puts set
|
|
290
|
+
attributes back. A DynamoDB check is never repaired.
|
|
291
|
+
|
|
292
|
+
## Firestore
|
|
293
|
+
|
|
294
|
+
One JSON command per step, every value a placeholder:
|
|
295
|
+
|
|
296
|
+
```json
|
|
297
|
+
{"get": "users/$1"}
|
|
298
|
+
{"query": "users", "where": [["email", "==", "$1"], ["age", ">=", "$2:number"]]}
|
|
299
|
+
{"create": "users/$1", "data": {"name": "$2"}}
|
|
300
|
+
{"add": "users", "data": {"name": "$1"}}
|
|
301
|
+
{"update": "users/$1", "set": {"tier": "$2", "address.city": "$3"}}
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
- `db.url` is `firestore://<project-id>`, or
|
|
305
|
+
`firestore://<host>:<port>/<project-id>` for the emulator; credentials
|
|
306
|
+
come from the environment. `db.allow` lists the project id or the
|
|
307
|
+
emulator's host.
|
|
308
|
+
- A collection is written in the path; a document's id may be a
|
|
309
|
+
placeholder. `where` filters are joined by AND and use `==`, `!=`, `<`,
|
|
310
|
+
`<=`, `>`, `>=`, `in`, `not-in`, `array-contains`,
|
|
311
|
+
`array-contains-any`. No delete, `orderBy` or collection group.
|
|
312
|
+
- A placeholder names its type when the field is not text: `"$1:number"`,
|
|
313
|
+
`"$1:double"`, `"$1:bool"`, `"$1:timestamp"`, `"$1:reference"`,
|
|
314
|
+
`"$1:binary"` (bytes as `0x` hex). A "not in the database" check is red
|
|
315
|
+
when a document is found with a bind sent as another type (both ends of
|
|
316
|
+
a range too), when it compares a list by `==` or `in` (write
|
|
317
|
+
`array-contains` or `array-contains-any`) or one value by
|
|
318
|
+
`array-contains`, and refused on a collection that holds no document.
|
|
319
|
+
- A row holds the document's fields with `__id__` and `__path__`. A
|
|
320
|
+
`create` never writes over a document; cleanup deletes created
|
|
321
|
+
documents and puts set fields back. A Firestore check is never
|
|
322
|
+
repaired.
|
|
323
|
+
|
|
161
324
|
## Not yet
|
|
162
325
|
|
|
163
|
-
|
|
326
|
+
`saffron eject` writes these scenarios as `test.fixme`; the
|
|
164
327
|
editor plugins show database steps only as the runner reports them.
|
|
@@ -7,13 +7,26 @@ everything below.
|
|
|
7
7
|
## Steps
|
|
8
8
|
|
|
9
9
|
```gherkin
|
|
10
|
-
|
|
10
|
+
# action (healable)
|
|
11
|
+
Given I am on the login page
|
|
11
12
|
When I enter "standard_user" in the username field
|
|
12
13
|
And I click the "Log in" button
|
|
13
|
-
|
|
14
|
+
# assertion (never healed)
|
|
15
|
+
Then I should see the products page
|
|
14
16
|
But I should not see an error banner
|
|
15
17
|
```
|
|
16
18
|
|
|
19
|
+
Spell the headers and step keywords exactly: `Feature:`, `Scenario:`,
|
|
20
|
+
`Scenario Outline:`, `Examples:`, `Background:`, `StepSet:`, and each step
|
|
21
|
+
keyword followed by one space. Gherkin reads a line it does not know as
|
|
22
|
+
the description above it, so `scenario:` in lower case, `Scenario :`,
|
|
23
|
+
`Scenario Name` with no colon, a tab after `Given`, or `stepset` would
|
|
24
|
+
drop the scenario or the step from the run without a word, even with a
|
|
25
|
+
single step or `*` steps under the header. Saffron refuses such a file
|
|
26
|
+
instead, naming the line. Prose stays prose: a description line may start
|
|
27
|
+
with "When" or "and", and `* item` bullets may follow a line like
|
|
28
|
+
`Example usage:`.
|
|
29
|
+
|
|
17
30
|
`And`/`But` inherit action-vs-assertion from the preceding step. The
|
|
18
31
|
trailing block of assertions is always strict; under the opt-in
|
|
19
32
|
`adaptable-mid` policy only mid-scenario checkpoints may be adapted.
|
|
@@ -197,7 +210,8 @@ And the "Book" link should point to "/booking"
|
|
|
197
210
|
```gherkin
|
|
198
211
|
When I place the order and wait for the order API to return 201
|
|
199
212
|
And I wait until the job status API reports "READY"
|
|
200
|
-
|
|
213
|
+
# assertion, never healed
|
|
214
|
+
Then the order request should have returned 201
|
|
201
215
|
```
|
|
202
216
|
|
|
203
217
|
Recorded as URL-pattern + method + status (+ body pattern) matchers;
|
|
@@ -231,12 +245,15 @@ recordings require format version 5, which older runners refuse.
|
|
|
231
245
|
## Page furniture
|
|
232
246
|
|
|
233
247
|
```gherkin
|
|
234
|
-
|
|
248
|
+
# alert/confirm/prompt
|
|
249
|
+
When I click "Clear workspace" and confirm the dialog
|
|
235
250
|
When I upload the file "examples/sample.txt" as the attachment
|
|
236
251
|
When I drag the task card onto the done column
|
|
237
|
-
|
|
252
|
+
# inside an iframe: just interact
|
|
253
|
+
When I enter "Great tool" in the feedback comment and send it
|
|
238
254
|
Then the feedback widget should show "Thanks for: Great tool"
|
|
239
|
-
|
|
255
|
+
# a click that downloads a file
|
|
256
|
+
When I add the stay to my Apple Calendar
|
|
240
257
|
Then the calendar file should contain "DTSTART;VALUE=DATE:{date+7:YYYYMMDD}"
|
|
241
258
|
```
|
|
242
259
|
|
|
@@ -249,8 +266,10 @@ as today's value.
|
|
|
249
266
|
## Tabs (plain prose, no keyword)
|
|
250
267
|
|
|
251
268
|
```gherkin
|
|
252
|
-
|
|
253
|
-
|
|
269
|
+
# opens a new tab: replay follows it
|
|
270
|
+
When I click the "Open preview" link
|
|
271
|
+
# asserted on the new tab
|
|
272
|
+
Then the preview should show "Draft 3"
|
|
254
273
|
When I switch back to the first tab
|
|
255
274
|
And I close the preview tab
|
|
256
275
|
When I open a new tab at the admin page
|
|
@@ -268,9 +287,11 @@ click) is not followed.
|
|
|
268
287
|
|
|
269
288
|
```gherkin
|
|
270
289
|
Given As "seller" I am on the orders page
|
|
271
|
-
|
|
290
|
+
# the buyer's own browser
|
|
291
|
+
When As "buyer" I order the blue lamp
|
|
272
292
|
Then As "seller" I see an order for the blue lamp
|
|
273
|
-
|
|
293
|
+
# still the seller
|
|
294
|
+
And the order is marked new
|
|
274
295
|
```
|
|
275
296
|
|
|
276
297
|
A step starting `As "name"` runs in that person's browser session: a
|
|
@@ -307,7 +328,7 @@ Scenario: Checkout happy path
|
|
|
307
328
|
| Form | Syntax | Notes |
|
|
308
329
|
|---|---|---|
|
|
309
330
|
| Define | `StepSet: <name>` | Colon, like `Scenario:`; steps indented below |
|
|
310
|
-
| Invoke | `StepSet <name>` | No colon;
|
|
331
|
+
| Invoke | `StepSet <name>` | No colon; a line of its own anywhere among steps (`When StepSet Login` is refused) |
|
|
311
332
|
|
|
312
333
|
- Expanded at parse time: inlined steps cache and seed like ordinary
|
|
313
334
|
steps; editing a set makes every caller honestly stale (re-recorded
|
|
@@ -332,7 +353,9 @@ Scenario: ...
|
|
|
332
353
|
(`--filter "@e2e and not @wip"`, `--filter "(@smoke or @sanity) and not
|
|
333
354
|
@flaky"`). A comma, `|`, a space or a repeated `--filter` between plain
|
|
334
355
|
tags means `or`; once `and`, `not` or a parenthesis appears, join every
|
|
335
|
-
tag with `and` or `or`. The `@` may be left out.
|
|
356
|
+
tag with `and` or `or`. The `@` may be left out. A comma always
|
|
357
|
+
separates (a tag holding one cannot be named), and `&&`, `!` and `~` are
|
|
358
|
+
refused: the operators are the words.
|
|
336
359
|
|
|
337
360
|
Tags that change how a scenario runs:
|
|
338
361
|
|