saffron-ai 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +419 -0
- package/dist-pkg/cli.js +845 -459
- package/package.json +4 -3
- package/skills/saffron/SKILL.md +32 -3
- package/skills/saffron/references/api.md +15 -0
- package/skills/saffron/references/config.md +1 -1
- package/templates/playwright/saffron-helpers.ts +58 -7
- package/textmate/saffron/syntaxes/saffron.tmLanguage.json +1 -1
package/package.json
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
|
7
7
|
"build": "tsc",
|
|
8
8
|
"dev": "tsc --watch",
|
|
9
9
|
"e2e": "node scripts/e2e.mjs && node scripts/e2e-eject.mjs",
|
|
10
|
+
"benchmark": "node scripts/benchmark.mjs",
|
|
10
11
|
"prepublishOnly": "npm run build && npm test",
|
|
11
12
|
"prepack": "npm run bundle",
|
|
12
|
-
"docs": "node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
|
|
13
|
+
"docs": "node scripts/build-compat.mjs && node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
|
|
13
14
|
"bundle": "esbuild src/cli/index.ts --bundle --platform=node --format=esm --packages=external --minify --legal-comments=none --outfile=dist-pkg/cli.js",
|
|
14
15
|
"plugin": "node scripts/build-claude-plugin.mjs"
|
|
15
16
|
},
|
|
@@ -65,7 +66,7 @@
|
|
|
65
66
|
"vscode-languageserver": "^10.1.0",
|
|
66
67
|
"vscode-languageserver-textdocument": "^1.0.12",
|
|
67
68
|
"yaml": "^2.9.1",
|
|
68
|
-
"zod": "^4.
|
|
69
|
+
"zod": "^4.6.7"
|
|
69
70
|
},
|
|
70
71
|
"homepage": "https://saffron-ai.io",
|
|
71
72
|
"repository": {
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -3,12 +3,15 @@ name: saffron
|
|
|
3
3
|
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
|
+
saffron-ai: "1.3.0"
|
|
6
7
|
author: saffron-ai
|
|
7
8
|
homepage: https://saffron-ai.io
|
|
8
9
|
---
|
|
9
10
|
|
|
10
11
|
# Saffron
|
|
11
12
|
|
|
13
|
+
This skill describes saffron-ai 1.3.0 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
|
|
14
|
+
|
|
12
15
|
Apply this workflow only when asked to create, translate, or edit Saffron
|
|
13
16
|
scenarios in `.saffron` or `.feature` files. A Saffron mention or an
|
|
14
17
|
existing test file in the project does not activate authoring work.
|
|
@@ -61,14 +64,18 @@ names the old value elsewhere (a `delete-bob` test id, a `#row-bob`
|
|
|
61
64
|
selector) is not reused for the new one: that step records afresh. `saffron init` registers the
|
|
62
65
|
`saffron` MCP server with Claude Code, Cursor and VS Code; its
|
|
63
66
|
`search_steps` / `list_step_sets` / `project_status` tools give the same
|
|
64
|
-
answers without a shell,
|
|
65
|
-
|
|
67
|
+
answers without a shell, `run_plan` says what a run would replay and
|
|
68
|
+
record (and which new steps to record and accept first) before it costs
|
|
69
|
+
anything, `api_contract` lists the operations of the pinned API
|
|
70
|
+
contract for `@api` scenarios, and `verify` replays what you changed at
|
|
71
|
+
zero AI and reports any check you weakened against the reviewed baseline.
|
|
66
72
|
|
|
67
73
|
**Know the project before touching it:**
|
|
68
74
|
|
|
69
75
|
```bash
|
|
70
76
|
npx saffron status # files and scenarios (● cached, ◐ proposal pending, ○ unrecorded), tags, proposals, last run, health
|
|
71
77
|
npx saffron status --json # the same as data (what the IDE panels and the project_status MCP tool read)
|
|
78
|
+
npx saffron run --plan # what a run would replay, record or refuse, and the agent sessions it needs, without running
|
|
72
79
|
```
|
|
73
80
|
|
|
74
81
|
Use it to pick tags that already exist, to see which scenarios are
|
|
@@ -187,6 +194,24 @@ means visible.
|
|
|
187
194
|
alone does not prove which way the flow went. Name the exact control a
|
|
188
195
|
variant uses (`the "Guarantee your booking by card" button`, not "pay");
|
|
189
196
|
two similar buttons are not interchangeable.
|
|
197
|
+
- **Put a step's values in a data table when there are more than two.**
|
|
198
|
+
A step reads as one sentence of intent; the values it enters or sends go
|
|
199
|
+
in a 2-column table on the lines below it (a key and its value), not in
|
|
200
|
+
one long line of quoted parameters. This holds for an API request's
|
|
201
|
+
headers, query parameters and body fields as much as for a form:
|
|
202
|
+
|
|
203
|
+
```gherkin
|
|
204
|
+
When I validate the no-show reservation with these request details
|
|
205
|
+
| reservation | {data:no-show.chargeable} |
|
|
206
|
+
| API key | {env:NOSHOW_API_KEY} |
|
|
207
|
+
| correlation ID | {unique:correlation} |
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Each cell records as its `<table:key>`, so editing a value replays at
|
|
211
|
+
zero AI; renaming or adding a key re-records the step. Use a header
|
|
212
|
+
table for a list of records, and a `"""` doc string for free text or a
|
|
213
|
+
whole JSON body. Keep one or two values inline when they read as part of
|
|
214
|
+
the sentence (`When I read customer "cust-42" through the API`).
|
|
190
215
|
- **Name the section a step acts in** when the page repeats labels (a
|
|
191
216
|
guest form and a booker form): `I replace "E-mail" in "Guest information"
|
|
192
217
|
with "..."`. The recording is then scoped to that section, so it acts on
|
|
@@ -260,7 +285,10 @@ Feature: Customers API
|
|
|
260
285
|
And the customer should match the contract
|
|
261
286
|
```
|
|
262
287
|
|
|
263
|
-
One request intent per `When` step
|
|
288
|
+
One request intent per `When` step. A value or two can sit in the step
|
|
289
|
+
text; more (an API key, a correlation id, query parameters, body fields)
|
|
290
|
+
go in a 2-column data table under the step, one row each, never one long
|
|
291
|
+
line. Records
|
|
264
292
|
that must be new each run take `{unique:name}`. Refer to an earlier
|
|
265
293
|
result by intent ("the customer"), never by an id you saw: the agent keeps
|
|
266
294
|
the fields later steps need. `Then` steps state a status, a field's value
|
|
@@ -352,6 +380,7 @@ setup, and other configuration needed by the requested test, read
|
|
|
352
380
|
| Write an id you saw in a response into a later API step | Refer to it by intent ("the customer I created"); the recording keeps it as `{captured:name}` |
|
|
353
381
|
| Create a fixed record (`"cust-1"`) in an API step | `{unique:name}`: every run, and the proof replay, creates its own |
|
|
354
382
|
| Check an API response's status in a `When` | Name the status in a `Then` ("should answer 404") |
|
|
383
|
+
| One long step line carrying an API key, ids and several quoted parameters | A short step and a 2-column data table under it, one value per row |
|
|
355
384
|
|
|
356
385
|
## References
|
|
357
386
|
|
|
@@ -56,6 +56,21 @@ Feature: Customers API
|
|
|
56
56
|
the step text: quoted strings, numbers, `<column>`s, `{data:...}`. The
|
|
57
57
|
recording keeps them as the step writes them, and a number stays a
|
|
58
58
|
number in the request body on every Examples row.
|
|
59
|
+
- **More than two values go in a data table** under the step, one row
|
|
60
|
+
each: headers (an API key, a correlation id), query parameters, body
|
|
61
|
+
fields. The step stays one readable sentence, and each cell records as
|
|
62
|
+
its `<table:key>` wherever the request sends it (its path, a header,
|
|
63
|
+
the body, a form field), so editing a cell replays at zero AI:
|
|
64
|
+
|
|
65
|
+
```gherkin
|
|
66
|
+
When I validate the no-show reservation with these request details
|
|
67
|
+
| reservation | {data:no-show.chargeable} |
|
|
68
|
+
| API key | {env:NOSHOW_API_KEY} |
|
|
69
|
+
| correlation ID | {unique:correlation} |
|
|
70
|
+
Then the response status should be 200
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
A whole JSON body the step sends as written goes in a `"""` doc string.
|
|
59
74
|
- **Records that must be new each run** take `{unique:name}`. A proof
|
|
60
75
|
replay is a second execution, and every later run is another: each
|
|
61
76
|
creates its own customer instead of colliding with the last one. Clean
|
|
@@ -86,7 +86,7 @@ your-project/
|
|
|
86
86
|
| `saffron steps [search] [--json] [--snippets]` | The step vocabulary with recorded/divergent/unrecorded badges |
|
|
87
87
|
| `saffron author <prose-file> [--provider p]` | Draft a `.saffron` file from plain-paragraph requirements using the project vocabulary |
|
|
88
88
|
| `saffron login [provider]` | Check that a provider can record and heal here and list what is missing; `saffron login cursor` also runs Cursor's one-time sign-in, `saffron login antigravity` also allows Saffron's tools in agy |
|
|
89
|
-
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `api_contract`) for AI assistants / language server for editors |
|
|
89
|
+
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `run_plan`, `api_contract`, `verify`) for AI assistants / language server for editors |
|
|
90
90
|
| `saffron api pin <source> [--out file] [--service name]` | Pin the API contract: fetch an OpenAPI 3 document from a Swagger UI page, a document URL or a file, and write it as JSON (default `api/openapi.json`, or `api/<name>.json` with `--service`). Add it to the config as `api.contract`, or as `api.services.<name>.contract` |
|
|
91
91
|
| `saffron init [--examples] [--agents list]` | Install this skill into the project's agent directories, register MCP, add an AGENTS.md block, scaffold config; `--examples` adds the Saucedemo example suite |
|
|
92
92
|
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs, 1 GB, fixed), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
|
|
@@ -66,12 +66,61 @@ function mask(text: string): string {
|
|
|
66
66
|
return text;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
/** Words a
|
|
70
|
-
const CREDENTIAL_WORDS = new Set(["token", "secret", "password", "passwd", "pwd", "apikey", "accesskey", "privatekey", "session", "credential", "credentials", "auth", "authorization", "jwt", "bearer", "cookie", "otp", "pin", "csrf", "xsrf", "signature", "nonce", "refresh"]);
|
|
69
|
+
/** Words a name ends in when what it holds is a credential: a login's token, a session id, a key, a code. */
|
|
70
|
+
const CREDENTIAL_WORDS = new Set(["token", "secret", "password", "passwd", "pwd", "passphrase", "apikey", "accesskey", "privatekey", "session", "credential", "credentials", "auth", "authorization", "jwt", "bearer", "cookie", "otp", "pin", "csrf", "xsrf", "signature", "nonce", "refresh"]);
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
/** Words after a credential's that say how it is held, never what it is. */
|
|
73
|
+
const HELD_AS = new Set(["code", "value", "pem", "encrypted", "json", "data", "string", "raw", "b64", "base64"]);
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Whether a name (a capture's, a field's, a header's) names a credential, by its last word, a version or a number
|
|
77
|
+
* after it aside (`accessToken`, `api_key`, `X-Session-Id`, `X-Hub-Signature-256`, `Set-Cookie`), never one it only
|
|
78
|
+
* mentions (`sessionTitle`, `pinColor`, `tokenExpiry`). The runner's rule (src/http/replay.ts), kept the same by a test.
|
|
79
|
+
*/
|
|
80
|
+
export function namesSecret(name: string): boolean {
|
|
81
|
+
const words = name.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
|
82
|
+
// A version, a number or a word that only says how it is held (`authCode`, `apiKeyValue`, `privateKeyPem`,
|
|
83
|
+
// `accessTokenEncrypted`, `sessionData`) goes before the word that names it (found in review).
|
|
84
|
+
while (words.length > 1 && (/^v?\d+$/.test(words.at(-1)!) || HELD_AS.has(words.at(-1)!))) words.pop();
|
|
85
|
+
const last = words.at(-1) ?? "";
|
|
86
|
+
const before = words.at(-2) ?? "";
|
|
87
|
+
if (last === "key") return ["api", "access", "private", "secret", "signing", "session", "encryption", "client", "subscription"].includes(before);
|
|
88
|
+
if (last === "id") return before === "session" || before === "key";
|
|
89
|
+
return CREDENTIAL_WORDS.has(last);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Whether a captured value is a credential, read from its name and the field it came from (`token`,
|
|
94
|
+
* `/data/access_token`, `sessionId`, `header:Set-Cookie`): a value the API under test handed out that must not reach
|
|
95
|
+
* a transcript, a report or another host. A value's shape says nothing; `id` and `name` are plain.
|
|
96
|
+
*/
|
|
97
|
+
export function isCredentialCapture(name: string, pointer: string): boolean {
|
|
98
|
+
// A pointer's last segment, unescaped (`/data/id~1token` names `id/token`).
|
|
99
|
+
return namesSecret(name) || namesSecret((pointer.split("/").at(-1) ?? "").replaceAll("~1", "/").replaceAll("~0", "~"));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* The credentials inside an extracted object or list (`extract "/data"` holding `{"token": ...}`), by their
|
|
104
|
+
* pointer under it: a capture of the whole is a credential too, and each of these is masked on its own. A value of
|
|
105
|
+
* 8 characters or more, or a code (`otp`, `pin`) of 4 to 8 digits, as text or a number: a short word under such a key
|
|
106
|
+
* (`"auth": "yes"`) is no secret, and masked it would garble every text it appears in.
|
|
107
|
+
*/
|
|
108
|
+
export function nestedCredentials(value: unknown, at = ""): [string, string][] {
|
|
109
|
+
if (!value || typeof value !== "object") return [];
|
|
110
|
+
return Object.entries(value).flatMap(([key, item]): [string, string][] => {
|
|
111
|
+
const here = `${at}/${key}`;
|
|
112
|
+
if (!namesSecret(key)) return nestedCredentials(item, here);
|
|
113
|
+
const words = key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
|
114
|
+
const code = words.some((w) => w === "otp" || w === "pin");
|
|
115
|
+
// A password is one at any length a password takes (a test user's "abc123").
|
|
116
|
+
const password = words.some((w) => ["password", "passwd", "pwd", "passphrase", "secret"].includes(w));
|
|
117
|
+
const secret = (v: unknown): string | undefined =>
|
|
118
|
+
typeof v === "string" && (v.length >= 8 || (password && v.length >= 3)) ? v
|
|
119
|
+
: code && (typeof v === "string" || typeof v === "number") && /^\d{4,8}$/.test(String(v)) ? String(v) : undefined;
|
|
120
|
+
const secrets = (Array.isArray(item) ? item.map((v, i) => [`${here}/${i}`, secret(v)] as const) : [[here, secret(item)] as const])
|
|
121
|
+
.filter((entry): entry is readonly [string, string] => entry[1] !== undefined);
|
|
122
|
+
return secrets.length > 0 ? secrets.map(([where, v]) => [where, v]) : nestedCredentials(item, here);
|
|
123
|
+
});
|
|
75
124
|
}
|
|
76
125
|
|
|
77
126
|
/** Whether a host name resolves to a link-local address, looked up for every request. */
|
|
@@ -840,7 +889,6 @@ export class Run {
|
|
|
840
889
|
}
|
|
841
890
|
// A response the step fails on still hands out its credentials, and the error quotes its body: they are known first.
|
|
842
891
|
for (const [name, pointer] of Object.entries(spec.extract ?? {})) {
|
|
843
|
-
if (!isCredentialCapture(name, pointer)) continue;
|
|
844
892
|
const header = /^header:(.+)$/i.exec(pointer);
|
|
845
893
|
let value: unknown;
|
|
846
894
|
try {
|
|
@@ -848,11 +896,14 @@ export class Run {
|
|
|
848
896
|
} catch {
|
|
849
897
|
value = undefined;
|
|
850
898
|
}
|
|
851
|
-
|
|
899
|
+
// An object holding one is a credential too, and its token is masked on its own.
|
|
900
|
+
const nested = nestedCredentials(value);
|
|
901
|
+
if (value !== undefined && (isCredentialCapture(name, pointer) || nested.length > 0)) {
|
|
852
902
|
const kept = typeof value === "string" ? value : JSON.stringify(value);
|
|
853
903
|
this.credentials.set(name, kept);
|
|
854
904
|
CREDENTIALS.set(name, kept);
|
|
855
905
|
}
|
|
906
|
+
for (const [path, secret] of nested) CREDENTIALS.set(`${name}${path}`, secret);
|
|
856
907
|
}
|
|
857
908
|
const problem = notYet(exchange);
|
|
858
909
|
if (problem !== undefined) throw new Error(mask(until === undefined ? `${method} ${url} ${problem}` : `${method} ${url} still ${problem} after ${spec.until!.timeoutMs ?? 30_000} ms of polling`));
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
}
|
|
53
53
|
},
|
|
54
54
|
"block-keyword": {
|
|
55
|
-
"match": "^\\s*(Feature|Background|Scenario Outline|Scenario Template|Scenario|Examples|Scenarios|Rule)(:)(.*)$",
|
|
55
|
+
"match": "^\\s*(Feature|Background|Fixture|Scenario Outline|Scenario Template|Scenario|Examples|Scenarios|Rule)(:)(.*)$",
|
|
56
56
|
"captures": {
|
|
57
57
|
"1": {
|
|
58
58
|
"name": "keyword.control.gherkin.saffron"
|