saffron-ai 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +527 -0
- package/dist-pkg/cli.js +390 -370
- package/package.json +1 -1
- package/skills/saffron/SKILL.md +13 -2
- package/skills/saffron/references/config.md +3 -2
- package/skills/saffron/references/syntax.md +5 -0
- package/templates/playwright/saffron-helpers.ts +203 -16
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -3,14 +3,14 @@ name: saffron
|
|
|
3
3
|
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
|
-
saffron-ai: "1.3.
|
|
6
|
+
saffron-ai: "1.3.1"
|
|
7
7
|
author: saffron-ai
|
|
8
8
|
homepage: https://saffron-ai.io
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
# Saffron
|
|
12
12
|
|
|
13
|
-
This skill describes saffron-ai 1.3.
|
|
13
|
+
This skill describes saffron-ai 1.3.1 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
|
|
14
14
|
|
|
15
15
|
Apply this workflow only when asked to create, translate, or edit Saffron
|
|
16
16
|
scenarios in `.saffron` or `.feature` files. A Saffron mention or an
|
|
@@ -366,6 +366,17 @@ or weakening assertions. For CI artifacts, browser matrices, provider
|
|
|
366
366
|
setup, and other configuration needed by the requested test, read
|
|
367
367
|
[Configuration & CI](references/config.md).
|
|
368
368
|
|
|
369
|
+
## When Saffron cannot express a value: a small supporting script
|
|
370
|
+
|
|
371
|
+
Use Saffron itself first: `{date...}` tokens (ranges and locales included), captures and comparisons, `{data:...}` and `{unique:...}`, the checks in the syntax reference, and `setup`/`teardown`. A recording that freezes a value or checks a pattern any value would pass is a recording to reword or re-record, not a missing capability. Reach for a script only when the runner cannot express a requirement at replay, and ask the user first when it would add a dependency, a hook, or a file outside the project's existing scripts folder.
|
|
372
|
+
|
|
373
|
+
- Keep it small: one file in the project's existing scripts folder, in its runtime and module style, using the standard library or dependencies already installed. Clicks and checks stay in the scenario; the script only prepares data.
|
|
374
|
+
- Expected values come from the test's inputs or an independent source (a contract, a fixture), never from the output being checked. Generate volatile values at run time, and state locale, time zone, format and day-boundary assumptions in the script.
|
|
375
|
+
- Run it as a `setup` command in `saffron.config.json`, keeping the hooks already there. It writes a git-ignored file under `data/`, which scenarios read through named `{data:...}` tokens: Saffron reads data after setup, so every replay sees fresh values. Commit the script and its configuration, never what it generates, and never write secrets into it or its output. On failure it exits non-zero with a message: setup failing stops the run before any scenario. `--no-hooks` skips it: scenarios then read its last output, or stop naming the missing file; say so.
|
|
376
|
+
- Never weaken a check to fit, and never hand-edit caches or proposals.
|
|
377
|
+
- Check it in proportion: its logic at the boundaries it handles (month end, year end, the time zone), the recording's checks and `{data:...}` tokens reviewed, only verified proposals accepted, and the scenario passing with `--no-agent`.
|
|
378
|
+
- Report which files you wrote, which the script generates and which recordings Saffron made, why the script is needed, and which Saffron capability could replace it.
|
|
379
|
+
|
|
369
380
|
## Anti-patterns that cost money or hide bugs
|
|
370
381
|
|
|
371
382
|
| Don't | Do |
|
|
@@ -59,6 +59,7 @@ your-project/
|
|
|
59
59
|
| `snapshotMode` | `none` (default, ~60% fewer AI calls) or `full` for highly dynamic pages |
|
|
60
60
|
| `setup` / `teardown` | shell command or list, run once before / after the run in the project root (seed and clean test data). Failed setup: exit 2, nothing runs. Teardown always runs. They receive `SAFFRON_BASE_URL` and `SAFFRON_ENV`. `--no-hooks` skips them; `hookTimeoutMs` (default 300000) bounds each |
|
|
61
61
|
| `trace` | `off` (default), `on` or `retain-on-failure`: keep an execution trace per scenario; `saffron trace` opens it as a replay with the page inspectable at every step |
|
|
62
|
+
| `video` | `off` (default), `on` or `retain-on-failure`: keep a video of each scenario's run, `video.webm` beside its screenshots; also `--video` |
|
|
62
63
|
| `browser` | `chromium` / `chrome` / `msedge` / `firefox` / `webkit`: replay runs anywhere; recording and healing need one of the first three. `chrome` and `msedge` must be installed on the machine. A list runs the suite in each into one report (browser matrix; Chromium-family first) |
|
|
63
64
|
| `device` / `viewport` | A Playwright device profile (`"iPhone 15"`) and/or a `"WIDTHxHEIGHT"` size for every scenario; a desktop recording replays under it or fails at the step that differs |
|
|
64
65
|
| `timezone` | IANA zone (`"Europe/Berlin"`) the browser runs in and `{date...}` tokens count days in (default: the machine's) |
|
|
@@ -70,7 +71,7 @@ your-project/
|
|
|
70
71
|
| `model` | Model in the provider's own names (unset: the provider's default). A `--provider` run on another provider ignores it |
|
|
71
72
|
| `healModel` | Cheaper model for heal sessions only |
|
|
72
73
|
| `antigravityApiKey` | `true`: Antigravity signs in with `GEMINI_API_KEY` even beside an agy login (default: a login wins) |
|
|
73
|
-
| `ci` | Where `saffron runs` and `import --run` find CI runs: `{"provider": "github" \| "azure", "organization": ..., "project": ...}`. Unset, an Azure Repos remote means Azure Pipelines (through `az`), anything else GitHub Actions (through `gh`). `organization` and `project` are for a GitHub repository built by Azure Pipelines |
|
|
74
|
+
| `ci` | Where `saffron runs` and `import --run` find CI runs: `{"provider": "github" \| "azure", "organization": ..., "project": ...}`. Unset, an Azure Repos remote means Azure Pipelines (through `az`), anything else GitHub Actions (through `gh`). `organization` and `project` are for a GitHub repository built by Azure Pipelines. `cacheRuns` and `cacheMB` cap the runs viewed into `.saffron/ci/` (default 10 and 1024) |
|
|
74
75
|
| `api` | The API under test: `{"baseURL": ..., "contract": "api/openapi.json", "transcript": "on-failure"}`. `baseURL` is where API steps' relative paths go (unset: the run's `baseURL`); `contract` is the pinned OpenAPI 3 document schema checks validate against (`saffron api pin` writes it), and a change to it, or to which file it names, makes the recordings checked against it stale; `transcript` keeps each scenario's requests and responses for the report and `saffron trace`: `"on-failure"` (default), `"on"` or `"off"` (statuses and timings only); `services` names more APIs, each `{"baseURL": ..., "contract": ...}`, which a request names instead of a host (`--api-url name=<url>` points one elsewhere for a run). See [API tests](api.md) |
|
|
75
76
|
|
|
76
77
|
## CLI
|
|
@@ -89,7 +90,7 @@ your-project/
|
|
|
89
90
|
| `saffron mcp` / `saffron lsp` | MCP tools (`search_steps`, `list_step_sets`, `project_status`, `run_plan`, `api_contract`, `verify`) for AI assistants / language server for editors |
|
|
90
91
|
| `saffron api pin <source> [--out file] [--service name]` | Pin the API contract: fetch an OpenAPI 3 document from a Swagger UI page, a document URL or a file, and write it as JSON (default `api/openapi.json`, or `api/<name>.json` with `--service`). Add it to the config as `api.contract`, or as `api.services.<name>.contract` |
|
|
91
92
|
| `saffron init [--examples] [--agents list]` | Install this skill into the project's agent directories, register MCP, add an AGENTS.md block, scaffold config; `--examples` adds the Saucedemo example suite |
|
|
92
|
-
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs
|
|
93
|
+
| `saffron report [--merge [paths]] \| --run <id> [--provider p] [--allow-fork] [--max-download MB] [--json \| --panel]` | Open the latest HTML report; `--merge` combines a complete set of shard reports into `latest.json`, `latest.html` and one history line. `--run` views a CI run's results without importing: downloaded once into `.saffron/ci/` (newest 10 runs and 1 GB unless `ci.cacheRuns` and `ci.cacheMB` say otherwise), checked like an import, proposals judged importable or not with the reason, nothing filed. No bundle exits 0 with `reasonKind` `running`, `expired` or `none`; over `--max-download` exits 2 with `kind: "too-large"` and `sizeBytes`. `saffron import --run` reuses the download. `--panel` prints the run's results as the one HTML page the editors' CI results panel shows |
|
|
93
94
|
| `saffron dashboard [--stdout] [--no-open]` | The suite's quality from the run history (failing and for how long, flaky, repeatedly healed, never run, slowest, trends, per tag); writes and opens `.saffron/reports/dashboard.html`, `--stdout` prints it |
|
|
94
95
|
| `saffron export [-o file] [--branch name [--base branch]] [--report-only]` | Pack the pending proposals, the run's report, screenshots and traces into one bundle (default `.saffron/reports/saffron-bundle.zip`). `--report-only` writes a summary instead (report and screenshots only, `saffron-summary.zip`): upload it as a `saffron-summary*` artifact on every run so any run's results can be viewed; a run that left only summaries has nothing to import. In CI write it with `--out <file outside .saffron>` and upload that file as an artifact (`upload-artifact` skips hidden folders). `--branch` also pushes the proposals as a new branch and opens a draft pull request (GitHub through `gh`, Azure DevOps through `az`); exit 1 only when the branch is pushed but the pull request could not be opened |
|
|
95
96
|
| `saffron runs [--branch b] [--limit n] [--provider github\|azure] [--json]` | The branch's CI runs, with how many of each run's proposals are pending here; exit 2 when `gh` or `az` is missing or signed out |
|
|
@@ -18,6 +18,11 @@ But I should not see an error banner
|
|
|
18
18
|
trailing block of assertions is always strict; under the opt-in
|
|
19
19
|
`adaptable-mid` policy only mid-scenario checkpoints may be adapted.
|
|
20
20
|
|
|
21
|
+
A check on the page's address reads what the URL contains (`Then the URL
|
|
22
|
+
contains "/orders"`). There is no check that a URL does not contain
|
|
23
|
+
something, or is exactly something: name a part the URL must contain, or
|
|
24
|
+
check what the page shows.
|
|
25
|
+
|
|
21
26
|
## Scenario Outlines
|
|
22
27
|
|
|
23
28
|
One cache serves every row: `<param>` placeholders stay in the cache and
|
|
@@ -72,6 +72,9 @@ const CREDENTIAL_WORDS = new Set(["token", "secret", "password", "passwd", "pwd"
|
|
|
72
72
|
/** Words after a credential's that say how it is held, never what it is. */
|
|
73
73
|
const HELD_AS = new Set(["code", "value", "pem", "encrypted", "json", "data", "string", "raw", "b64", "base64"]);
|
|
74
74
|
|
|
75
|
+
/** Credential words a name may end in written as one word: only ones no ordinary word ends in. */
|
|
76
|
+
const JOINED_SUFFIXES = ["token", "secret", "password", "passwd", "passphrase", "apikey", "accesskey", "privatekey", "credential", "credentials", "cookie", "sessionid", "sessid"];
|
|
77
|
+
|
|
75
78
|
/**
|
|
76
79
|
* Whether a name (a capture's, a field's, a header's) names a credential, by its last word, a version or a number
|
|
77
80
|
* after it aside (`accessToken`, `api_key`, `X-Session-Id`, `X-Hub-Signature-256`, `Set-Cookie`), never one it only
|
|
@@ -86,7 +89,8 @@ export function namesSecret(name: string): boolean {
|
|
|
86
89
|
const before = words.at(-2) ?? "";
|
|
87
90
|
if (last === "key") return ["api", "access", "private", "secret", "signing", "session", "encryption", "client", "subscription"].includes(before);
|
|
88
91
|
if (last === "id") return before === "session" || before === "key";
|
|
89
|
-
|
|
92
|
+
// Written as one word (`authtoken`, `JSESSIONID`), and `sid` (`connect.sid`) only whole.
|
|
93
|
+
return CREDENTIAL_WORDS.has(last) || (last === "sid" && (words.length === 1 || before === "connect" || before === "session")) || JOINED_SUFFIXES.some((w) => last.endsWith(w));
|
|
90
94
|
}
|
|
91
95
|
|
|
92
96
|
/**
|
|
@@ -113,7 +117,7 @@ export function nestedCredentials(value: unknown, at = ""): [string, string][] {
|
|
|
113
117
|
const words = key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
|
114
118
|
const code = words.some((w) => w === "otp" || w === "pin");
|
|
115
119
|
// A password is one at any length a password takes (a test user's "abc123").
|
|
116
|
-
const password = words.some((w) => ["password", "passwd", "pwd", "passphrase", "secret"].
|
|
120
|
+
const password = words.some((w) => ["password", "passwd", "pwd", "passphrase", "secret"].some((p) => w === p || w.endsWith(p)));
|
|
117
121
|
const secret = (v: unknown): string | undefined =>
|
|
118
122
|
typeof v === "string" && (v.length >= 8 || (password && v.length >= 3)) ? v
|
|
119
123
|
: code && (typeof v === "string" || typeof v === "number") && /^\d{4,8}$/.test(String(v)) ? String(v) : undefined;
|
|
@@ -190,6 +194,14 @@ export function pattern(source: string, flags = ""): RegExp {
|
|
|
190
194
|
return new RegExp(source, flags);
|
|
191
195
|
}
|
|
192
196
|
|
|
197
|
+
/** A value as one piece of a URL, as replay encodes it in a request's path (a copy of src/http/replay.ts urlValue). */
|
|
198
|
+
/*URL_VALUE*/
|
|
199
|
+
export function urlValue(value: string, query: boolean): string {
|
|
200
|
+
const unsafe = query ? /%(?![0-9A-Fa-f]{2})|[^A-Za-z0-9\-._~!$'()*,;:@/%?]/gu : /%(?![0-9A-Fa-f]{2})|[^A-Za-z0-9\-._~!$&'()*+,;=:@/%]/gu;
|
|
201
|
+
return value.replace(unsafe, (c) => encodeURIComponent(c));
|
|
202
|
+
}
|
|
203
|
+
/*END*/
|
|
204
|
+
|
|
193
205
|
// --- dates ----------------------------------------------------------------
|
|
194
206
|
|
|
195
207
|
const DATE_TOKEN_RE = /\{date(?:([+-]\d+)(?:([^\d{}:]+?)([+-]\d+))?)?(?::([^}]+))?\}/g;
|
|
@@ -412,7 +424,8 @@ function isLinkLocal(hostname: string): boolean {
|
|
|
412
424
|
const host = hostname.replace(/^\[|\]$/g, "").toLowerCase();
|
|
413
425
|
if (/^169\.254\.\d{1,3}\.\d{1,3}$/.test(host)) return true;
|
|
414
426
|
if (/^fe[89ab][0-9a-f]?:/.test(host) || host === "fd00:ec2::254") return true;
|
|
415
|
-
|
|
427
|
+
// IPv4-compatible (::a9fe:a9fe) and NAT64 (64:ff9b::a9fe:a9fe) reach it too, as replay reads them.
|
|
428
|
+
const mapped = /^(?:::(?:ffff:)?|64:ff9b::)(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):[0-9a-f]{1,4})$/.exec(host);
|
|
416
429
|
if (mapped) return mapped[1] ? /^169\.254\./.test(mapped[1]) : parseInt(mapped[2], 16) === 0xa9fe;
|
|
417
430
|
return false;
|
|
418
431
|
}
|
|
@@ -724,9 +737,11 @@ export class Run {
|
|
|
724
737
|
.toBe(true);
|
|
725
738
|
}
|
|
726
739
|
|
|
727
|
-
captured
|
|
740
|
+
/** A captured value; `checked`: in a text a check compares, where a blank one would hold for more than it means, as replay refuses it. */
|
|
741
|
+
captured(name: string, checked = false): string {
|
|
728
742
|
const value = this.vars.get(name);
|
|
729
743
|
if (value === undefined) throw new Error(`no captured value named "${name}"`);
|
|
744
|
+
if (checked && value.trim() === "") throw new Error(`{captured:${name}} is blank, so the check would hold for more than it means`);
|
|
730
745
|
return value;
|
|
731
746
|
}
|
|
732
747
|
|
|
@@ -738,21 +753,32 @@ export class Run {
|
|
|
738
753
|
*/
|
|
739
754
|
async expectDiffers(compareTo: string, target?: Locator, left?: string, same = false): Promise<void> {
|
|
740
755
|
const right = this.captured(compareTo);
|
|
756
|
+
// Amounts compare as amounts, as replay's valuesDiffer does: "$10" is "$10.00", two currencies are a problem.
|
|
741
757
|
if (!target) {
|
|
742
|
-
|
|
758
|
+
const verdict = valuesDiffer(this.captured(left ?? ""), right);
|
|
759
|
+
if ("problem" in verdict) throw new Error(`the differs check: ${verdict.problem}`);
|
|
760
|
+
expect(verdict.differ, `"${left}" to differ from "${compareTo}"`).toBe(true);
|
|
743
761
|
return;
|
|
744
762
|
}
|
|
745
763
|
let element: ElementHandle | undefined = same ? this.captures.get(compareTo) : undefined;
|
|
764
|
+
/** The latest look's reason the two cannot be compared: said only if no look passes. */
|
|
765
|
+
let problem: string | undefined;
|
|
746
766
|
await expect
|
|
747
767
|
.poll(async () => {
|
|
748
768
|
element ??= (await target.first().elementHandle({ timeout: 1000 }).catch(() => null)) ?? undefined;
|
|
749
769
|
if (!element) return false;
|
|
750
770
|
const text = await drawn(element);
|
|
751
|
-
if (text === ""
|
|
771
|
+
if (text === "") return false;
|
|
772
|
+
const verdict = valuesDiffer(text, right);
|
|
773
|
+
problem = "problem" in verdict ? verdict.problem : undefined;
|
|
774
|
+
if (!("differ" in verdict) || !verdict.differ) return false;
|
|
752
775
|
// The element must still be one the target names.
|
|
753
776
|
return target.evaluateAll((els, el) => els.includes(el as Element), element).catch(() => false);
|
|
754
777
|
}, { message: `target text to differ from "${compareTo}" (${right})`, timeout: SETTINGS.timeoutMs })
|
|
755
|
-
.toBe(true)
|
|
778
|
+
.toBe(true)
|
|
779
|
+
.catch((err) => {
|
|
780
|
+
throw problem ? new Error(`the differs check: ${problem}`) : err;
|
|
781
|
+
});
|
|
756
782
|
}
|
|
757
783
|
|
|
758
784
|
/** A response matching the recorded request has been seen (a wait) or was answered since the latest action (a check). */
|
|
@@ -805,6 +831,8 @@ export class Run {
|
|
|
805
831
|
async send(spec: RequestSpec): Promise<void> {
|
|
806
832
|
const method = spec.method.toUpperCase();
|
|
807
833
|
const where = spec.service === undefined ? "api.baseURL" : `api.services.${spec.service}.baseURL`;
|
|
834
|
+
// A value that comes out blank ({captured:code} as "") is refused, as replay refuses it, never read as any 2xx.
|
|
835
|
+
if (spec.status !== undefined && spec.status.trim() === "") throw new Error(`${method} ${mask(spec.path)} expects a status that is blank, so it would hold for any 2xx`);
|
|
808
836
|
if (spec.until !== undefined && !(spec.idempotent ?? IDEMPOTENT_METHODS.has(method))) {
|
|
809
837
|
throw new Error(`${method} ${spec.path} polls (until), and a ${method} may not be sent twice: poll with a GET that reads the state, or mark the request idempotent`);
|
|
810
838
|
}
|
|
@@ -960,7 +988,7 @@ export class Run {
|
|
|
960
988
|
} else if (expected.pattern !== undefined) {
|
|
961
989
|
let re: RegExp;
|
|
962
990
|
try {
|
|
963
|
-
re =
|
|
991
|
+
re = patternRegex(expected.pattern);
|
|
964
992
|
} catch {
|
|
965
993
|
throw new Error(`expectHeader: /${expected.pattern}/ is not a regular expression`);
|
|
966
994
|
}
|
|
@@ -1054,6 +1082,35 @@ async function lines(element: ElementHandle): Promise<string[]> {
|
|
|
1054
1082
|
return (await read(element)).pieces;
|
|
1055
1083
|
}
|
|
1056
1084
|
|
|
1085
|
+
// A copy of the runner's choice of the match a check reads (src/replay/index.ts), kept the same by a test.
|
|
1086
|
+
/*PICK_DISPLAYED*/
|
|
1087
|
+
/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
|
|
1088
|
+
function pickDisplayed(els: any[], [limit, src]: [number, string]): number {
|
|
1089
|
+
const drawn = (0, eval)("(" + src + ")");
|
|
1090
|
+
const gone = (e: any): boolean => {
|
|
1091
|
+
for (let at = e; at; at = at.parentElement ?? at.getRootNode().host) {
|
|
1092
|
+
if (at.hidden || at.ownerDocument.defaultView.getComputedStyle(at).display === "none") return true;
|
|
1093
|
+
}
|
|
1094
|
+
return false;
|
|
1095
|
+
};
|
|
1096
|
+
const text = (e: any) => String(e.textContent ?? "").replace(/\s+/g, " ").trim();
|
|
1097
|
+
// An element the browser says is visible draws its text: the full walk only for one it does not (visibility:hidden, a closed <details>).
|
|
1098
|
+
const visible = (e: any) => e.checkVisibility?.({ visibilityProperty: true, contentVisibilityAuto: false }) === true;
|
|
1099
|
+
const undrawn = (e: any) => gone(e) || (text(e) !== "" && !visible(e) && !/\S/.test(drawn(e).text));
|
|
1100
|
+
if (els.length === 0 || !undrawn(els[0])) return 0;
|
|
1101
|
+
for (let i = 1; i < Math.min(els.length, limit); i++) {
|
|
1102
|
+
if (!undrawn(els[i])) return text(els[i]) === text(els[0]) ? i : 0;
|
|
1103
|
+
}
|
|
1104
|
+
return 0;
|
|
1105
|
+
}
|
|
1106
|
+
/*END*/
|
|
1107
|
+
|
|
1108
|
+
/** The match replay reads for a target: its first, or a displayed copy with the same text after a hidden first one (a mobile menu's). */
|
|
1109
|
+
async function theMatch(target: Locator): Promise<Locator> {
|
|
1110
|
+
const at = await target.evaluateAll(pickDisplayed, [20, DRAWN_TEXT] as [number, string]).catch(() => 0);
|
|
1111
|
+
return at > 0 ? target.nth(at) : target.first();
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1057
1114
|
const poll = (check: () => Promise<boolean>, message: string) =>
|
|
1058
1115
|
expect.poll(check, { message, timeout: SETTINGS.timeoutMs }).toBe(true);
|
|
1059
1116
|
|
|
@@ -1080,15 +1137,30 @@ export async function expectAttribute(target: Locator, name: string, source: str
|
|
|
1080
1137
|
const re = pattern(source);
|
|
1081
1138
|
const absentHolds = re.test("") && re.test("false") && !re.test("true");
|
|
1082
1139
|
await poll(async () => {
|
|
1083
|
-
const found = await target
|
|
1140
|
+
const found = await (await theMatch(target)).getAttribute(name, { timeout: 1000 }).catch(() => undefined);
|
|
1084
1141
|
if (found === undefined) return false;
|
|
1085
1142
|
return found === null ? absentHolds : re.test(found);
|
|
1086
1143
|
}, `attribute ${name} to match /${source}/`);
|
|
1087
1144
|
}
|
|
1088
1145
|
|
|
1146
|
+
/** The target's computed CSS `property` is `expected` as the browser computes it, or matches it. */
|
|
1147
|
+
export async function expectStyle(target: Locator, property: string, expected: string | RegExp): Promise<void> {
|
|
1148
|
+
said(expected, "the expected style");
|
|
1149
|
+
const wanted = typeof expected === "string" ? normalize(expected) : expected;
|
|
1150
|
+
await poll(async () => {
|
|
1151
|
+
const found = await (await theMatch(target)).evaluate((el, name) => (globalThis as unknown as { getComputedStyle(e: unknown): { getPropertyValue(n: string): string } }).getComputedStyle(el).getPropertyValue(name), property, { timeout: 1000 }).catch(() => undefined);
|
|
1152
|
+
if (found === undefined) return false;
|
|
1153
|
+
return typeof wanted === "string" ? normalize(found) === wanted : wanted.test(normalize(found));
|
|
1154
|
+
}, `${property} to be ${expected}`);
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1089
1157
|
/** Every match of every locator the recording names is hidden. */
|
|
1090
1158
|
export async function expectNotVisible(...targets: Locator[]): Promise<void> {
|
|
1091
|
-
|
|
1159
|
+
// All hidden at the same look, as replay checks them: one gone, then another, while the first came back, is not a pass.
|
|
1160
|
+
await poll(async () => {
|
|
1161
|
+
for (const target of targets) if ((await target.filter({ visible: true }).count()) > 0) return false;
|
|
1162
|
+
return true;
|
|
1163
|
+
}, `every match of every recorded locator to be hidden (${targets.length} locator(s))`);
|
|
1092
1164
|
}
|
|
1093
1165
|
|
|
1094
1166
|
/** Each value is shown in the container, as a whole line of its text. */
|
|
@@ -1109,14 +1181,121 @@ async function shownLines(target: Locator): Promise<string[]> {
|
|
|
1109
1181
|
}
|
|
1110
1182
|
|
|
1111
1183
|
/** The active page's URL contains `expected`, raw, decoded, or with + read as a space. */
|
|
1112
|
-
|
|
1184
|
+
/** The URL contains `expected`; with "excludes", it does not; with "path", its path is exactly `expected`. */
|
|
1185
|
+
export async function expectUrl(page: Page, expected: string | RegExp, match?: "excludes" | "path"): Promise<void> {
|
|
1113
1186
|
said(expected, "the expected URL");
|
|
1114
1187
|
await poll(async () => {
|
|
1115
1188
|
const url = page.url();
|
|
1116
|
-
|
|
1117
|
-
|
|
1189
|
+
if (match === "path") {
|
|
1190
|
+
const path = new URL(url).pathname;
|
|
1191
|
+
return [path, decode(path)].some((p) => (typeof expected === "string" ? p === expected : expected.test(p)));
|
|
1192
|
+
}
|
|
1193
|
+
const holds = [url, decode(url), decode(url.replace(/\+/g, "%20"))].some((u) => contains(u, expected));
|
|
1194
|
+
return match === "excludes" ? !holds : holds;
|
|
1195
|
+
}, match === "excludes" ? `URL not containing ${expected} on the active page` : match === "path" ? `URL path ${expected} on the active page` : `URL containing ${expected} on the active page`);
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
// --- amounts ---------------------------------------------------------------
|
|
1199
|
+
|
|
1200
|
+
// A copy of the runner's amount reading (src/util/amount.ts), so a differs check
|
|
1201
|
+
// compares "$10" and "$10.00" as replay does: as one amount.
|
|
1202
|
+
/*AMOUNT*/
|
|
1203
|
+
const currencies = new Set(Intl.supportedValuesOf("currency"));
|
|
1204
|
+
/** Currency notation without a symbol of its own in Unicode: Rs 500, 5,00 zł, 5.00 Kč, Ft 500. */
|
|
1205
|
+
const LOCAL_CURRENCIES = new Set(["kr", "rs", "zł", "kč", "ft", "lei", "lv", "лв", "din", "ден", "дин"]);
|
|
1206
|
+
// An ISO code in capitals; in lower case only beside decimals ("(usd 5.00)"):
|
|
1207
|
+
// "(2 pen)" and "(3 all)" are counts, not Peruvian soles or Albanian lek.
|
|
1208
|
+
const isCurrency = (s: string) => /\p{Sc}/u.test(s) || currencies.has(s) || LOCAL_CURRENCIES.has(s.toLowerCase().replace(/\.$/, ""));
|
|
1209
|
+
|
|
1210
|
+
/** Digits grouped by a space, a no-break space or an apostrophe, as one number ("1 234", "1'234"). */
|
|
1211
|
+
const joinGroups = (text: string) => text.replace(/(\d)[\s\u00a0\u202f'\u2019](?=\d{3}(?!\d))/g, "$1");
|
|
1212
|
+
|
|
1213
|
+
export function parseAmount(text: string): number | undefined {
|
|
1214
|
+
// Digits grouped by a space, a no-break space or an apostrophe are one number.
|
|
1215
|
+
const joined = joinGroups(text);
|
|
1216
|
+
const runs = [...joined.matchAll(/[-\u2212\u2013]?\d(?:[\d.,]*\d)?/g)];
|
|
1217
|
+
if (runs.length !== 1) return undefined;
|
|
1218
|
+
const [found] = runs;
|
|
1219
|
+
const before = joined.slice(0, found.index);
|
|
1220
|
+
const after = joined.slice(found.index + found[0].length);
|
|
1221
|
+
// A decimal with no whole part ("$.50", "€,99"): the separator before it is
|
|
1222
|
+
// the decimal point, and the sign is judged by what comes before that ("-.50", "($.50)").
|
|
1223
|
+
const bare = /[.,]$/.test(before) && !/\d[.,]$/.test(before) && /^\d+$/.test(found[0]);
|
|
1224
|
+
const lead = bare ? before.slice(0, -1) : before;
|
|
1225
|
+
const opening = /\(\s*([^\d\s.,()]{0,4})\s*$/.exec(lead);
|
|
1226
|
+
const closing = /^\s*([^\d\s.,()]{0,4})\s*\)/.exec(after);
|
|
1227
|
+
const labels = [opening?.[1], closing?.[1]].filter((s): s is string => !!s);
|
|
1228
|
+
// An unknown label keeps a decimal amount accounting only when it holds no letter: a unit (Weight (1.5 kg)) or a word (Guests (2.5 pax)) does not.
|
|
1229
|
+
const separated = /[.,]/.test(found[0]);
|
|
1230
|
+
const accounting = opening && closing && (labels.length === 0 ? separated : labels.every((l) => isCurrency(l) || (separated && currencies.has(l.toUpperCase()))) || (separated && !labels.some((l) => /\p{L}/u.test(l))));
|
|
1231
|
+
// A dash glued to a letter or digit before it is a hyphen ("ID-123"),
|
|
1232
|
+
// never a sign; after a currency it is the sign ("CHF-5.00", as de-CH writes -5).
|
|
1233
|
+
const glued = /([\p{L}\p{N}]+)$/u.exec(before)?.[1];
|
|
1234
|
+
const hyphen = glued !== undefined && !(/^\p{L}+$/u.test(glued) && (isCurrency(glued) || currencies.has(glued.toUpperCase())));
|
|
1235
|
+
const negative =
|
|
1236
|
+
(/^[-\u2212\u2013]/.test(found[0]) && !hyphen) ||
|
|
1237
|
+
// A minus spaced off at the very start, before the currency or the number ("- €5.00", "- 5,00 €").
|
|
1238
|
+
/^\s*[-\u2212\u2013]\s*(?:[^\d\s.,()\-\u2212\u2013]{1,4}\s*)?$/u.test(lead) ||
|
|
1239
|
+
(bare && /[-\u2212\u2013]$/.test(lead)) ||
|
|
1240
|
+
// Attached to the currency, not spaced off like the dash in "Breakfast - $5.00".
|
|
1241
|
+
/(?:^|[\s(:])[-\u2212\u2013][^\d\s.,()\-\u2212\u2013]{1,4}\s?$/.test(lead) ||
|
|
1242
|
+
/^[-\u2212\u2013](?!\d)/.test(after) ||
|
|
1243
|
+
// Parentheses denote a negative amount only with currency notation or an unlabelled decimal.
|
|
1244
|
+
accounting;
|
|
1245
|
+
|
|
1246
|
+
const digits = bare ? `0${before.slice(-1)}${found[0]}` : found[0].replace(/^[-\u2212\u2013]/, "");
|
|
1247
|
+
const last = Math.max(digits.lastIndexOf("."), digits.lastIndexOf(","));
|
|
1248
|
+
const head = last < 0 ? digits : digits.slice(0, last);
|
|
1249
|
+
const tail = last < 0 ? "" : digits.slice(last + 1);
|
|
1250
|
+
const decimal = last >= 0 && (tail.length !== 3 || /^0?$/.test(head) || (head.includes(digits[last] === "." ? "," : ".")));
|
|
1251
|
+
const whole = decimal ? head : digits;
|
|
1252
|
+
// Grouping: one separator, every group after the first three digits long, not the decimal one.
|
|
1253
|
+
const groups = whole.split(/[.,]/);
|
|
1254
|
+
const grouping = new Set(whole.match(/[.,]/g) ?? []);
|
|
1255
|
+
if (grouping.size > 1 || (decimal && grouping.has(digits[last])) || groups[0] === "" || groups.slice(1).some((g) => g.length !== 3)) return undefined;
|
|
1256
|
+
const amount = Number(`${negative ? "-" : ""}${groups.join("")}${decimal ? `.${tail}` : ""}`);
|
|
1257
|
+
return Number.isFinite(amount) ? amount : undefined;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
/** What an amount's text says besides its number ("NOK", "kr", "Book now for NOK"), lower case, sign and parentheses left out. */
|
|
1261
|
+
const amountUnit = (text: string): string =>
|
|
1262
|
+
joinGroups(text).replace(/[-\u2212\u2013]?\d(?:[\d.,]*\d)?/, " ").replace(/[()\-\u2212\u2013]/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
|
|
1263
|
+
|
|
1264
|
+
/**
|
|
1265
|
+
* Whether two displayed values differ, for an expectDiffers check. Two
|
|
1266
|
+
* amounts compare as numbers ("NOK 1 295" and "NOK 1,295.00" are equal), and
|
|
1267
|
+
* only when they say the same besides the number: "SEK 1,295" and "NOK 1,347"
|
|
1268
|
+
* differ by currency alone, and "Book now for NOK 1,295" against "NOK 1,295"
|
|
1269
|
+
* by its label, so either is a problem with the recording, not a pass.
|
|
1270
|
+
* Anything else compares as text.
|
|
1271
|
+
*/
|
|
1272
|
+
export function valuesDiffer(left: string, right: string): { differ: boolean } | { problem: string } {
|
|
1273
|
+
// A bare number in parentheses is a negative one in a ledger, as "-5" is: "(5)" and "-5" are equal.
|
|
1274
|
+
const signed = (text: string, n: number | undefined) => (n !== undefined && /^\s*\(\s*[\d.,\s\u00a0]+\)\s*$/.test(text) ? -Math.abs(n) : n);
|
|
1275
|
+
const [a, b] = [signed(left, parseAmount(left)), signed(right, parseAmount(right))];
|
|
1276
|
+
if (a === undefined || b === undefined) return { differ: left !== right };
|
|
1277
|
+
const [ua, ub] = [amountUnit(left), amountUnit(right)];
|
|
1278
|
+
if (ua !== ub) {
|
|
1279
|
+
const currency = (unit: string) => unit !== "" && (isCurrency(unit) || isCurrency(unit.toUpperCase()));
|
|
1280
|
+
const words = (unit: string) => unit.split(/\s+/);
|
|
1281
|
+
// Two currencies, or a price against a label holding that price's currency ("Book now for NOK 1,295"):
|
|
1282
|
+
// a change no amount explains. Any other two (references, counts: "ABC-123", "1 night") compare as text.
|
|
1283
|
+
if (currency(ua) && currency(ub)) {
|
|
1284
|
+
return { problem: `"${left}" and "${right}" are amounts in different currencies, so they differ whatever the amount does. Capture both in one currency.` };
|
|
1285
|
+
}
|
|
1286
|
+
if ((currency(ua) && words(ub).includes(ua)) || (currency(ub) && words(ua).includes(ub))) {
|
|
1287
|
+
return { problem: `"${left}" and "${right}" are not the same kind of value (one says more than the amount), so they differ whatever the amount does. Capture both from the same kind of element: the amount alone.` };
|
|
1288
|
+
}
|
|
1289
|
+
if (ua === "" || ub === "") return { differ: !sameAmount(a, b) };
|
|
1290
|
+
return { differ: left !== right };
|
|
1291
|
+
}
|
|
1292
|
+
return { differ: !sameAmount(a, b) };
|
|
1118
1293
|
}
|
|
1119
1294
|
|
|
1295
|
+
/** Equal to the cent, so 0.1 + 0.2 is 0.3. */
|
|
1296
|
+
export const sameAmount = (a: number, b: number): boolean => Math.abs(a - b) < 0.005;
|
|
1297
|
+
/*END*/
|
|
1298
|
+
|
|
1120
1299
|
// --- JSON field checks ----------------------------------------------------
|
|
1121
1300
|
|
|
1122
1301
|
/** A JSON field check, as a recording holds one (ResponseJsonCheck in saffron-ai). */
|
|
@@ -1283,7 +1462,11 @@ function checkValue(json: unknown, checks: ResponseJsonCheck[], prefix = ""): st
|
|
|
1283
1462
|
const other = jsonExpected(check, check.differs);
|
|
1284
1463
|
if (value === undefined) return `${field}: expected a value other than ${describe(other)}, got missing`;
|
|
1285
1464
|
// Untyped, the text is compared as written: a number 5 is the same as "5" (a captured id is kept as text).
|
|
1286
|
-
|
|
1465
|
+
// A list, an object or null is compared as its JSON text: [{"sku":"a"}] is the same as itself written out.
|
|
1466
|
+
// Keys in any order and any spacing: {"b":2,"a":1} is the same as {"a": 1, "b": 2}.
|
|
1467
|
+
const canonical = (v: unknown) => JSON.stringify(v, (_key, x: unknown) => (x && typeof x === "object" && !Array.isArray(x) ? Object.fromEntries(Object.entries(x).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) : x));
|
|
1468
|
+
const written = (text: unknown) => { try { return canonical(JSON.parse(String(text))); } catch { return String(text); } };
|
|
1469
|
+
if (value === other || (check.type === undefined && (value !== null && typeof value === "object" ? canonical(value) === written(other) : String(value) === other))) return `${field}: expected a value other than ${describe(other)}, got the same`;
|
|
1287
1470
|
} else if (check.pattern !== undefined) {
|
|
1288
1471
|
const re = jsonPattern(check);
|
|
1289
1472
|
if (typeof value !== "string") return `${field}: expected text matching /${check.pattern}/, got ${got}`;
|
|
@@ -1311,10 +1494,14 @@ export function checkResponseJson(body: string | undefined, checks: ResponseJson
|
|
|
1311
1494
|
}
|
|
1312
1495
|
|
|
1313
1496
|
/** Unicode boundaries need unicode mode; preserve legacy flags on patterns without them. */
|
|
1497
|
+
// `\x27` as the secret masker spells it; `\u{...}` past one byte.
|
|
1498
|
+
const hexEscape = (code: number) => (code < 0x100 ? `\\x${code.toString(16).padStart(2, "0")}` : `\\u{${code.toString(16)}}`);
|
|
1499
|
+
|
|
1314
1500
|
export function patternRegex(source: string, flags = ""): RegExp {
|
|
1315
1501
|
if (/[uv]/.test(flags) || source.includes(String.raw`[\p{L}\p{N}\p{M}]`)) {
|
|
1316
|
-
//
|
|
1317
|
-
|
|
1502
|
+
// Unicode mode allows only syntax characters and `/` escaped: Playwright's
|
|
1503
|
+
// escaped quotes and chain separators, and an agent's harmless `\#` or `\:`, become code point escapes.
|
|
1504
|
+
source = source.replace(/\\[\s\S]|[\s\S]/gu, part => /^\\[^\p{L}\p{N}^$\\.*+?()[\]{}|/]$/u.test(part) ? hexEscape(part.codePointAt(1)!) : part);
|
|
1318
1505
|
if (!/[uv]/.test(flags)) flags += "u";
|
|
1319
1506
|
}
|
|
1320
1507
|
return new RegExp(source, flags);
|