saffron-ai 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +377 -1
- package/README.md +4 -4
- package/dist-pkg/cli.js +833 -462
- package/package.json +25 -9
- package/skills/saffron/SKILL.md +9 -2
- package/skills/saffron/references/api.md +6 -4
- package/skills/saffron/references/config.md +7 -6
- package/templates/playwright/saffron-helpers.ts +8 -1
package/package.json
CHANGED
|
@@ -1,28 +1,44 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.2.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.2.2",
|
|
4
|
+
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
|
7
7
|
"build": "tsc",
|
|
8
8
|
"dev": "tsc --watch",
|
|
9
9
|
"e2e": "node scripts/e2e.mjs && node scripts/e2e-eject.mjs",
|
|
10
|
+
"benchmark": "node scripts/benchmark.mjs",
|
|
10
11
|
"prepublishOnly": "npm run build && npm test",
|
|
11
12
|
"prepack": "npm run bundle",
|
|
12
|
-
"docs": "node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
|
|
13
|
+
"docs": "node scripts/build-compat.mjs && node scripts/build-docs.mjs && node scripts/build-site-docs.mjs && node scripts/build-claude-plugin.mjs && node scripts/build-vscode-keywords.mjs",
|
|
13
14
|
"bundle": "esbuild src/cli/index.ts --bundle --platform=node --format=esm --packages=external --minify --legal-comments=none --outfile=dist-pkg/cli.js",
|
|
14
15
|
"plugin": "node scripts/build-claude-plugin.mjs"
|
|
15
16
|
},
|
|
16
17
|
"keywords": [
|
|
18
|
+
"e2e",
|
|
19
|
+
"e2e-testing",
|
|
20
|
+
"end-to-end-testing",
|
|
17
21
|
"testing",
|
|
18
|
-
"
|
|
19
|
-
"bdd",
|
|
22
|
+
"test-automation",
|
|
20
23
|
"playwright",
|
|
21
|
-
"ai",
|
|
24
|
+
"ai-testing",
|
|
22
25
|
"self-healing",
|
|
23
|
-
"
|
|
26
|
+
"self-healing-tests",
|
|
27
|
+
"natural-language",
|
|
28
|
+
"plain-language",
|
|
29
|
+
"bdd",
|
|
30
|
+
"gherkin",
|
|
31
|
+
"cucumber",
|
|
32
|
+
"feature-files",
|
|
33
|
+
"no-step-definitions",
|
|
34
|
+
"browser-testing",
|
|
35
|
+
"regression-testing",
|
|
36
|
+
"claude",
|
|
37
|
+
"codex",
|
|
38
|
+
"cursor",
|
|
39
|
+
"mcp"
|
|
24
40
|
],
|
|
25
|
-
"author": "",
|
|
41
|
+
"author": "Chathuranga Jayasinghe",
|
|
26
42
|
"license": "SEE LICENSE IN LICENSE",
|
|
27
43
|
"type": "module",
|
|
28
44
|
"bin": {
|
|
@@ -50,7 +66,7 @@
|
|
|
50
66
|
"vscode-languageserver": "^10.1.0",
|
|
51
67
|
"vscode-languageserver-textdocument": "^1.0.12",
|
|
52
68
|
"yaml": "^2.9.1",
|
|
53
|
-
"zod": "^4.
|
|
69
|
+
"zod": "^4.6.7"
|
|
54
70
|
},
|
|
55
71
|
"homepage": "https://saffron-ai.io",
|
|
56
72
|
"repository": {
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -3,12 +3,15 @@ name: saffron
|
|
|
3
3
|
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
|
+
saffron-ai: "1.2.2"
|
|
6
7
|
author: saffron-ai
|
|
7
8
|
homepage: https://saffron-ai.io
|
|
8
9
|
---
|
|
9
10
|
|
|
10
11
|
# Saffron
|
|
11
12
|
|
|
13
|
+
This skill describes saffron-ai 1.2.2 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
|
|
14
|
+
|
|
12
15
|
Apply this workflow only when asked to create, translate, or edit Saffron
|
|
13
16
|
scenarios in `.saffron` or `.feature` files. A Saffron mention or an
|
|
14
17
|
existing test file in the project does not activate authoring work.
|
|
@@ -61,14 +64,18 @@ names the old value elsewhere (a `delete-bob` test id, a `#row-bob`
|
|
|
61
64
|
selector) is not reused for the new one: that step records afresh. `saffron init` registers the
|
|
62
65
|
`saffron` MCP server with Claude Code, Cursor and VS Code; its
|
|
63
66
|
`search_steps` / `list_step_sets` / `project_status` tools give the same
|
|
64
|
-
answers without a shell,
|
|
65
|
-
|
|
67
|
+
answers without a shell, `run_plan` says what a run would replay and
|
|
68
|
+
record (and which new steps to record and accept first) before it costs
|
|
69
|
+
anything, `api_contract` lists the operations of the pinned API
|
|
70
|
+
contract for `@api` scenarios, and `verify` replays what you changed at
|
|
71
|
+
zero AI and reports any check you weakened against the reviewed baseline.
|
|
66
72
|
|
|
67
73
|
**Know the project before touching it:**
|
|
68
74
|
|
|
69
75
|
```bash
|
|
70
76
|
npx saffron status # files and scenarios (● cached, ◐ proposal pending, ○ unrecorded), tags, proposals, last run, health
|
|
71
77
|
npx saffron status --json # the same as data (what the IDE panels and the project_status MCP tool read)
|
|
78
|
+
npx saffron run --plan # what a run would replay, record or refuse, and the agent sessions it needs, without running
|
|
72
79
|
```
|
|
73
80
|
|
|
74
81
|
Use it to pick tags that already exist, to see which scenarios are
|
|
@@ -118,10 +118,12 @@ npx saffron trace "A missing customer answers 404" # the requests and response
|
|
|
118
118
|
- A write (`POST`, `PUT`, `PATCH`, `DELETE`) is never retried on replay:
|
|
119
119
|
a timeout fails the step instead of creating a second record. Reads are
|
|
120
120
|
retried as usual.
|
|
121
|
-
- A recording with a schema check
|
|
122
|
-
changes, or when `api.contract` names another file:
|
|
123
|
-
|
|
124
|
-
|
|
121
|
+
- A recording with a schema check is checked again when the pinned
|
|
122
|
+
contract changes, or when `api.contract` names another file: a zero-AI
|
|
123
|
+
replay of the same recording, filed for review (yellow) when it passes,
|
|
124
|
+
red when the API breaks the new contract, and no agent session either way.
|
|
125
|
+
It never passes green over a contract it was not checked against.
|
|
126
|
+
`saffron run --rerecord` records it again.
|
|
125
127
|
- `api.transcript` sets what the request transcript keeps: `"on-failure"`
|
|
126
128
|
(default), `"on"` (green scenarios too) or `"off"` (statuses and timings
|
|
127
129
|
only). The report lists each scenario's requests per step, and
|
|
@@ -69,6 +69,7 @@ your-project/
|
|
|
69
69
|
| `provider` | Whose agent records and heals, on its login on this machine: `claude` (default), `codex` (ChatGPT plan), `antigravity` (Google AI Pro/Ultra via the `agy` CLI, experimental), `cursor`. Also `--provider` |
|
|
70
70
|
| `model` | Model in the provider's own names (unset: the provider's default). A `--provider` run on another provider ignores it |
|
|
71
71
|
| `healModel` | Cheaper model for heal sessions only |
|
|
72
|
+
| `antigravityApiKey` | `true`: Antigravity signs in with `GEMINI_API_KEY` even beside an agy login (default: a login wins) |
|
|
72
73
|
| `ci` | Where `saffron runs` and `import --run` find CI runs: `{"provider": "github" \| "azure", "organization": ..., "project": ...}`. Unset, an Azure Repos remote means Azure Pipelines (through `az`), anything else GitHub Actions (through `gh`). `organization` and `project` are for a GitHub repository built by Azure Pipelines |
|
|
73
74
|
| `api` | The API under test: `{"baseURL": ..., "contract": "api/openapi.json", "transcript": "on-failure"}`. `baseURL` is where API steps' relative paths go (unset: the run's `baseURL`); `contract` is the pinned OpenAPI 3 document schema checks validate against (`saffron api pin` writes it), and a change to it, or to which file it names, makes the recordings checked against it stale; `transcript` keeps each scenario's requests and responses for the report and `saffron trace`: `"on-failure"` (default), `"on"` or `"off"` (statuses and timings only); `services` names more APIs, each `{"baseURL": ..., "contract": ...}`, which a request names instead of a host (`--api-url name=<url>` points one elsewhere for a run). See [API tests](api.md) |
|
|
74
75
|
|
|
@@ -85,7 +86,7 @@ your-project/
|
|
|
85
86
|
| `saffron steps [search] [--json] [--snippets]` | The step vocabulary with recorded/divergent/unrecorded badges |
|
|
86
87
|
| `saffron author <prose-file> [--provider p]` | Draft a `.saffron` file from plain-paragraph requirements using the project vocabulary |
|
|
87
88
|
| `saffron login [provider]` | Check that a provider can record and heal here and list what is missing; `saffron login cursor` also runs Cursor's one-time sign-in, `saffron login antigravity` also allows Saffron's tools in agy |
|
|
88
|
-
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `api_contract`) for AI assistants / language server for editors |
|
|
89
|
+
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `run_plan`, `api_contract`, `verify`) for AI assistants / language server for editors |
|
|
89
90
|
| `saffron api pin <source> [--out file] [--service name]` | Pin the API contract: fetch an OpenAPI 3 document from a Swagger UI page, a document URL or a file, and write it as JSON (default `api/openapi.json`, or `api/<name>.json` with `--service`). Add it to the config as `api.contract`, or as `api.services.<name>.contract` |
|
|
90
91
|
| `saffron init [--examples] [--agents list]` | Install this skill into the project's agent directories, register MCP, add an AGENTS.md block, scaffold config; `--examples` adds the Saucedemo example suite |
|
|
91
92
|
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs, 1 GB, fixed), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
|
|
@@ -100,11 +101,11 @@ one line (`SAFFRON_DEBUG=1` shows the stack).
|
|
|
100
101
|
|
|
101
102
|
## AI access
|
|
102
103
|
|
|
103
|
-
Recording and healing need the configured provider's login on the machine
|
|
104
|
-
Claude (default): `ANTHROPIC_API_KEY`, or a Claude Code login. Codex: a
|
|
105
|
-
ChatGPT-plan `npx @openai/codex login` plus `npm i -D "@openai/codex-sdk@>=0.158.0"`. Antigravity:
|
|
106
|
-
Google's `agy` CLI, signed in once
|
|
107
|
-
(allows Saffron's tools in agy's settings). Cursor: `npm i -D @cursor/sdk`
|
|
104
|
+
Recording and healing need the configured provider's login on the machine,
|
|
105
|
+
or its API key. Claude (default): `ANTHROPIC_API_KEY`, or a Claude Code login. Codex: a
|
|
106
|
+
ChatGPT-plan `npx @openai/codex login`, or `CODEX_API_KEY` (or `OPENAI_API_KEY`), plus `npm i -D "@openai/codex-sdk@>=0.158.0"`. Antigravity:
|
|
107
|
+
Google's `agy` CLI, signed in once or with `GEMINI_API_KEY`, then `npx saffron login antigravity`
|
|
108
|
+
(allows Saffron's tools in agy's settings). A Codex or Antigravity login wins over a key. Cursor: `npm i -D @cursor/sdk`
|
|
108
109
|
and `npx saffron login cursor`. `npx saffron login` lists what is missing.
|
|
109
110
|
Replay-only runs (`--no-agent`) need none: that is the normal CI mode once
|
|
110
111
|
caches are committed. With no `model` configured the provider's default
|
|
@@ -83,7 +83,14 @@ async function resolvesLinkLocal(hostname: string): Promise<boolean> {
|
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
/** A number or boolean a request body sends where the recording holds text with tokens (a row's "<seats>"). */
|
|
86
|
-
export function typed(text: string, type: "number" | "boolean"):
|
|
86
|
+
export function typed(text: string, type: "number" | "boolean" | "json"): unknown {
|
|
87
|
+
if (type === "json") {
|
|
88
|
+
try {
|
|
89
|
+
return JSON.parse(text);
|
|
90
|
+
} catch {
|
|
91
|
+
throw new Error("a body part must be JSON, but its value resolves to text that is not: give the column a whole JSON value");
|
|
92
|
+
}
|
|
93
|
+
}
|
|
87
94
|
if (type === "boolean") {
|
|
88
95
|
if (text === "true" || text === "false") return text === "true";
|
|
89
96
|
throw new Error(`a body field must be true or false, but its value resolves to ${JSON.stringify(mask(text))}`);
|