saffron-ai 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,15 +1,16 @@
1
1
  {
2
2
  "name": "saffron-ai",
3
- "version": "1.2.1",
3
+ "version": "1.3.0",
4
4
  "description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
5
5
  "scripts": {
6
6
  "test": "vitest run",
7
7
  "build": "tsc",
8
8
  "dev": "tsc --watch",
9
9
  "e2e": "node scripts/e2e.mjs && node scripts/e2e-eject.mjs",
10
+ "benchmark": "node scripts/benchmark.mjs",
10
11
  "prepublishOnly": "npm run build && npm test",
11
12
  "prepack": "npm run bundle",
12
- "docs": "node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
13
+ "docs": "node scripts/build-compat.mjs && node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
13
14
  "bundle": "esbuild src/cli/index.ts --bundle --platform=node --format=esm --packages=external --minify --legal-comments=none --outfile=dist-pkg/cli.js",
14
15
  "plugin": "node scripts/build-claude-plugin.mjs"
15
16
  },
@@ -65,7 +66,7 @@
65
66
  "vscode-languageserver": "^10.1.0",
66
67
  "vscode-languageserver-textdocument": "^1.0.12",
67
68
  "yaml": "^2.9.1",
68
- "zod": "^4.4.3"
69
+ "zod": "^4.6.7"
69
70
  },
70
71
  "homepage": "https://saffron-ai.io",
71
72
  "repository": {
@@ -3,12 +3,15 @@ name: saffron
3
3
  description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
4
4
  license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
5
5
  metadata:
6
+ saffron-ai: "1.3.0"
6
7
  author: saffron-ai
7
8
  homepage: https://saffron-ai.io
8
9
  ---
9
10
 
10
11
  # Saffron
11
12
 
13
+ This skill describes saffron-ai 1.3.0 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
14
+
12
15
  Apply this workflow only when asked to create, translate, or edit Saffron
13
16
  scenarios in `.saffron` or `.feature` files. A Saffron mention or an
14
17
  existing test file in the project does not activate authoring work.
@@ -61,14 +64,18 @@ names the old value elsewhere (a `delete-bob` test id, a `#row-bob`
61
64
  selector) is not reused for the new one: that step records afresh. `saffron init` registers the
62
65
  `saffron` MCP server with Claude Code, Cursor and VS Code; its
63
66
  `search_steps` / `list_step_sets` / `project_status` tools give the same
64
- answers without a shell, and `api_contract` lists the operations of the
65
- pinned API contract for `@api` scenarios.
67
+ answers without a shell, `run_plan` says what a run would replay and
68
+ record (and which new steps to record and accept first) before it costs
69
+ anything, `api_contract` lists the operations of the pinned API
70
+ contract for `@api` scenarios, and `verify` replays what you changed at
71
+ zero AI and reports any check you weakened against the reviewed baseline.
66
72
 
67
73
  **Know the project before touching it:**
68
74
 
69
75
  ```bash
70
76
  npx saffron status # files and scenarios (● cached, ◐ proposal pending, ○ unrecorded), tags, proposals, last run, health
71
77
  npx saffron status --json # the same as data (what the IDE panels and the project_status MCP tool read)
78
+ npx saffron run --plan # what a run would replay, record or refuse, and the agent sessions it needs, without running
72
79
  ```
73
80
 
74
81
  Use it to pick tags that already exist, to see which scenarios are
@@ -187,6 +194,24 @@ means visible.
187
194
  alone does not prove which way the flow went. Name the exact control a
188
195
  variant uses (`the "Guarantee your booking by card" button`, not "pay");
189
196
  two similar buttons are not interchangeable.
197
+ - **Put a step's values in a data table when there are more than two.**
198
+ A step reads as one sentence of intent; the values it enters or sends go
199
+ in a 2-column table on the lines below it (a key and its value), not in
200
+ one long line of quoted parameters. This holds for an API request's
201
+ headers, query parameters and body fields as much as for a form:
202
+
203
+ ```gherkin
204
+ When I validate the no-show reservation with these request details
205
+ | reservation | {data:no-show.chargeable} |
206
+ | API key | {env:NOSHOW_API_KEY} |
207
+ | correlation ID | {unique:correlation} |
208
+ ```
209
+
210
+ Each cell records as its `<table:key>`, so editing a value replays at
211
+ zero AI; renaming or adding a key re-records the step. Use a header
212
+ table for a list of records, and a `"""` doc string for free text or a
213
+ whole JSON body. Keep one or two values inline when they read as part of
214
+ the sentence (`When I read customer "cust-42" through the API`).
190
215
  - **Name the section a step acts in** when the page repeats labels (a
191
216
  guest form and a booker form): `I replace "E-mail" in "Guest information"
192
217
  with "..."`. The recording is then scoped to that section, so it acts on
@@ -260,7 +285,10 @@ Feature: Customers API
260
285
  And the customer should match the contract
261
286
  ```
262
287
 
263
- One request intent per `When` step, its values in the step text. Records
288
+ One request intent per `When` step. A value or two can sit in the step
289
+ text; more (an API key, a correlation id, query parameters, body fields)
290
+ go in a 2-column data table under the step, one row each, never one long
291
+ line. Records
264
292
  that must be new each run take `{unique:name}`. Refer to an earlier
265
293
  result by intent ("the customer"), never by an id you saw: the agent keeps
266
294
  the fields later steps need. `Then` steps state a status, a field's value
@@ -352,6 +380,7 @@ setup, and other configuration needed by the requested test, read
352
380
  | Write an id you saw in a response into a later API step | Refer to it by intent ("the customer I created"); the recording keeps it as `{captured:name}` |
353
381
  | Create a fixed record (`"cust-1"`) in an API step | `{unique:name}`: every run, and the proof replay, creates its own |
354
382
  | Check an API response's status in a `When` | Name the status in a `Then` ("should answer 404") |
383
+ | One long step line carrying an API key, ids and several quoted parameters | A short step and a 2-column data table under it, one value per row |
355
384
 
356
385
  ## References
357
386
 
@@ -56,6 +56,21 @@ Feature: Customers API
56
56
  the step text: quoted strings, numbers, `<column>`s, `{data:...}`. The
57
57
  recording keeps them as the step writes them, and a number stays a
58
58
  number in the request body on every Examples row.
59
+ - **More than two values go in a data table** under the step, one row
60
+ each: headers (an API key, a correlation id), query parameters, body
61
+ fields. The step stays one readable sentence, and each cell records as
62
+ its `<table:key>` wherever the request sends it (its path, a header,
63
+ the body, a form field), so editing a cell replays at zero AI:
64
+
65
+ ```gherkin
66
+ When I validate the no-show reservation with these request details
67
+ | reservation | {data:no-show.chargeable} |
68
+ | API key | {env:NOSHOW_API_KEY} |
69
+ | correlation ID | {unique:correlation} |
70
+ Then the response status should be 200
71
+ ```
72
+
73
+ A whole JSON body the step sends as written goes in a `"""` doc string.
59
74
  - **Records that must be new each run** take `{unique:name}`. A proof
60
75
  replay is a second execution, and every later run is another: each
61
76
  creates its own customer instead of colliding with the last one. Clean
@@ -86,7 +86,7 @@ your-project/
86
86
  | `saffron steps [search] [--json] [--snippets]` | The step vocabulary with recorded/divergent/unrecorded badges |
87
87
  | `saffron author <prose-file> [--provider p]` | Draft a `.saffron` file from plain-paragraph requirements using the project vocabulary |
88
88
  | `saffron login [provider]` | Check that a provider can record and heal here and list what is missing; `saffron login cursor` also runs Cursor's one-time sign-in, `saffron login antigravity` also allows Saffron's tools in agy |
89
- | `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `api_contract`) for AI assistants / language server for editors |
89
+ | `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `run_plan`, `api_contract`, `verify`) for AI assistants / language server for editors |
90
90
  | `saffron api pin <source> [--out file] [--service name]` | Pin the API contract: fetch an OpenAPI 3 document from a Swagger UI page, a document URL or a file, and write it as JSON (default `api/openapi.json`, or `api/<name>.json` with `--service`). Add it to the config as `api.contract`, or as `api.services.<name>.contract` |
91
91
  | `saffron init [--examples] [--agents list]` | Install this skill into the project's agent directories, register MCP, add an AGENTS.md block, scaffold config; `--examples` adds the Saucedemo example suite |
92
92
  | `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs, 1 GB, fixed), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
@@ -66,12 +66,61 @@ function mask(text: string): string {
66
66
  return text;
67
67
  }
68
68
 
69
- /** Words a capture's name or its field carries when the value is a credential, as the runner reads them. */
70
- const CREDENTIAL_WORDS = new Set(["token", "secret", "password", "passwd", "pwd", "apikey", "accesskey", "privatekey", "session", "credential", "credentials", "auth", "authorization", "jwt", "bearer", "cookie", "otp", "pin", "csrf", "xsrf", "signature", "nonce", "refresh"]);
69
+ /** Words a name ends in when what it holds is a credential: a login's token, a session id, a key, a code. */
70
+ const CREDENTIAL_WORDS = new Set(["token", "secret", "password", "passwd", "pwd", "passphrase", "apikey", "accesskey", "privatekey", "session", "credential", "credentials", "auth", "authorization", "jwt", "bearer", "cookie", "otp", "pin", "csrf", "xsrf", "signature", "nonce", "refresh"]);
71
71
 
72
- function isCredentialCapture(name: string, pointer: string): boolean {
73
- const words = `${name} ${pointer.split("/").at(-1) ?? ""}`.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
74
- return words.some((w, i) => CREDENTIAL_WORDS.has(w) || (w === "key" && ["api", "access", "private", "secret"].includes(words[i - 1] ?? "")));
72
+ /** Words after a credential's that say how it is held, never what it is. */
73
+ const HELD_AS = new Set(["code", "value", "pem", "encrypted", "json", "data", "string", "raw", "b64", "base64"]);
74
+
75
+ /**
76
+ * Whether a name (a capture's, a field's, a header's) names a credential, by its last word, a version or a number
77
+ * after it aside (`accessToken`, `api_key`, `X-Session-Id`, `X-Hub-Signature-256`, `Set-Cookie`), never one it only
78
+ * mentions (`sessionTitle`, `pinColor`, `tokenExpiry`). The runner's rule (src/http/replay.ts), kept the same by a test.
79
+ */
80
+ export function namesSecret(name: string): boolean {
81
+ const words = name.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
82
+ // A version, a number or a word that only says how it is held (`authCode`, `apiKeyValue`, `privateKeyPem`,
83
+ // `accessTokenEncrypted`, `sessionData`) goes before the word that names it (found in review).
84
+ while (words.length > 1 && (/^v?\d+$/.test(words.at(-1)!) || HELD_AS.has(words.at(-1)!))) words.pop();
85
+ const last = words.at(-1) ?? "";
86
+ const before = words.at(-2) ?? "";
87
+ if (last === "key") return ["api", "access", "private", "secret", "signing", "session", "encryption", "client", "subscription"].includes(before);
88
+ if (last === "id") return before === "session" || before === "key";
89
+ return CREDENTIAL_WORDS.has(last);
90
+ }
91
+
92
+ /**
93
+ * Whether a captured value is a credential, read from its name and the field it came from (`token`,
94
+ * `/data/access_token`, `sessionId`, `header:Set-Cookie`): a value the API under test handed out that must not reach
95
+ * a transcript, a report or another host. A value's shape says nothing; `id` and `name` are plain.
96
+ */
97
+ export function isCredentialCapture(name: string, pointer: string): boolean {
98
+ // A pointer's last segment, unescaped (`/data/id~1token` names `id/token`).
99
+ return namesSecret(name) || namesSecret((pointer.split("/").at(-1) ?? "").replaceAll("~1", "/").replaceAll("~0", "~"));
100
+ }
101
+
102
+ /**
103
+ * The credentials inside an extracted object or list (`extract "/data"` holding `{"token": ...}`), by their
104
+ * pointer under it: a capture of the whole is a credential too, and each of these is masked on its own. A value of
105
+ * 8 characters or more, or a code (`otp`, `pin`) of 4 to 8 digits, as text or a number: a short word under such a key
106
+ * (`"auth": "yes"`) is no secret, and masked it would garble every text it appears in.
107
+ */
108
+ export function nestedCredentials(value: unknown, at = ""): [string, string][] {
109
+ if (!value || typeof value !== "object") return [];
110
+ return Object.entries(value).flatMap(([key, item]): [string, string][] => {
111
+ const here = `${at}/${key}`;
112
+ if (!namesSecret(key)) return nestedCredentials(item, here);
113
+ const words = key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
114
+ const code = words.some((w) => w === "otp" || w === "pin");
115
+ // A password is one at any length a password takes (a test user's "abc123").
116
+ const password = words.some((w) => ["password", "passwd", "pwd", "passphrase", "secret"].includes(w));
117
+ const secret = (v: unknown): string | undefined =>
118
+ typeof v === "string" && (v.length >= 8 || (password && v.length >= 3)) ? v
119
+ : code && (typeof v === "string" || typeof v === "number") && /^\d{4,8}$/.test(String(v)) ? String(v) : undefined;
120
+ const secrets = (Array.isArray(item) ? item.map((v, i) => [`${here}/${i}`, secret(v)] as const) : [[here, secret(item)] as const])
121
+ .filter((entry): entry is readonly [string, string] => entry[1] !== undefined);
122
+ return secrets.length > 0 ? secrets.map(([where, v]) => [where, v]) : nestedCredentials(item, here);
123
+ });
75
124
  }
76
125
 
77
126
  /** Whether a host name resolves to a link-local address, looked up for every request. */
@@ -840,7 +889,6 @@ export class Run {
840
889
  }
841
890
  // A response the step fails on still hands out its credentials, and the error quotes its body: they are known first.
842
891
  for (const [name, pointer] of Object.entries(spec.extract ?? {})) {
843
- if (!isCredentialCapture(name, pointer)) continue;
844
892
  const header = /^header:(.+)$/i.exec(pointer);
845
893
  let value: unknown;
846
894
  try {
@@ -848,11 +896,14 @@ export class Run {
848
896
  } catch {
849
897
  value = undefined;
850
898
  }
851
- if (value !== undefined) {
899
+ // An object holding one is a credential too, and its token is masked on its own.
900
+ const nested = nestedCredentials(value);
901
+ if (value !== undefined && (isCredentialCapture(name, pointer) || nested.length > 0)) {
852
902
  const kept = typeof value === "string" ? value : JSON.stringify(value);
853
903
  this.credentials.set(name, kept);
854
904
  CREDENTIALS.set(name, kept);
855
905
  }
906
+ for (const [path, secret] of nested) CREDENTIALS.set(`${name}${path}`, secret);
856
907
  }
857
908
  const problem = notYet(exchange);
858
909
  if (problem !== undefined) throw new Error(mask(until === undefined ? `${method} ${url} ${problem}` : `${method} ${url} still ${problem} after ${spec.until!.timeoutMs ?? 30_000} ms of polling`));
@@ -52,7 +52,7 @@
52
52
  }
53
53
  },
54
54
  "block-keyword": {
55
- "match": "^\\s*(Feature|Background|Scenario Outline|Scenario Template|Scenario|Examples|Scenarios|Rule)(:)(.*)$",
55
+ "match": "^\\s*(Feature|Background|Fixture|Scenario Outline|Scenario Template|Scenario|Examples|Scenarios|Rule)(:)(.*)$",
56
56
  "captures": {
57
57
  "1": {
58
58
  "name": "keyword.control.gherkin.saffron"