saffron-ai 1.3.4 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "saffron-ai",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.5",
|
|
4
4
|
"description": "End-to-end tests written in plain language: an AI agent records each scenario once, replays run at zero tokens, and broken replays heal into proposals you review",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "vitest run",
|
package/skills/saffron/SKILL.md
CHANGED
|
@@ -3,14 +3,14 @@ name: saffron
|
|
|
3
3
|
description: Create, translate, or edit Saffron test scenarios in .saffron or .feature files, browser UI tests and HTTP API tests (@api) alike, including validating their recordings and replay. Use when asked to author or fix these test files in a Saffron project. Do not activate for ordinary application development, generic Cucumber projects, or requests only to run existing tests, inspect reports, review proposals, or configure Saffron.
|
|
4
4
|
license: Saffron Free Use License v1.0 (see the saffron-ai package LICENSE)
|
|
5
5
|
metadata:
|
|
6
|
-
saffron-ai: "1.3.
|
|
6
|
+
saffron-ai: "1.3.5"
|
|
7
7
|
author: saffron-ai
|
|
8
8
|
homepage: https://saffron-ai.io
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
# Saffron
|
|
12
12
|
|
|
13
|
-
This skill describes saffron-ai 1.3.
|
|
13
|
+
This skill describes saffron-ai 1.3.5 (see the compatibility page: https://saffron-ai.io/docs/compatibility).
|
|
14
14
|
|
|
15
15
|
Apply this workflow only when asked to create, translate, or edit Saffron
|
|
16
16
|
scenarios in `.saffron` or `.feature` files. A Saffron mention or an
|
|
@@ -399,6 +399,7 @@ Use Saffron itself first: `{date...}` tokens (ranges and locales included), capt
|
|
|
399
399
|
outlines, secrets, dates, network waits, page furniture, StepSets.
|
|
400
400
|
- [Configuration & CI](references/config.md): `saffron.config.json`,
|
|
401
401
|
flags, `--strict`, cross-browser and browser matrix, devices, workers
|
|
402
|
-
and agent workers
|
|
402
|
+
and agent workers (`@serial` keeps scenarios that share data apart),
|
|
403
|
+
heal model, secrets setup, the API settings.
|
|
403
404
|
- [API tests](references/api.md): `@api` scenarios, the pinned contract,
|
|
404
405
|
writing and reviewing request steps, API calls from a browser scenario.
|
|
@@ -64,7 +64,7 @@ your-project/
|
|
|
64
64
|
| `device` / `viewport` | A Playwright device profile (`"iPhone 15"`) and/or a `"WIDTHxHEIGHT"` size for every scenario; a desktop recording replays under it or fails at the step that differs |
|
|
65
65
|
| `timezone` | IANA zone (`"Europe/Berlin"`) the browser runs in and `{date...}` tokens count days in (default: the machine's) |
|
|
66
66
|
| `dateLocale` | Locale of month and weekday names in `{date...}` tokens that name none (default `en-US`) |
|
|
67
|
-
| `workers` | Parallel replay workers |
|
|
67
|
+
| `workers` | Parallel replay workers; scenarios tagged `@serial` or `@serial:<name>` still run one at a time per name |
|
|
68
68
|
| `agentWorkers` | Agent sessions (record, heal) that may run at once, each in its own browser (default 1; each is a paid session) |
|
|
69
69
|
| `shardBy` | How `--shard` splits the suite: `"scenario"` (default) or `"file"` (each feature file on one machine) |
|
|
70
70
|
| `provider` | Whose agent records and heals, on its login on this machine: `claude` (default), `codex` (ChatGPT plan), `antigravity` (Google AI Pro/Ultra via the `agy` CLI, experimental), `cursor`. Also `--provider` |
|
|
@@ -322,6 +322,22 @@ Scenario: ...
|
|
|
322
322
|
tags means `or`; once `and`, `not` or a parenthesis appears, join every
|
|
323
323
|
tag with `and` or `or`. The `@` may be left out.
|
|
324
324
|
|
|
325
|
+
Tags that change how a scenario runs:
|
|
326
|
+
|
|
327
|
+
| Tag | Effect |
|
|
328
|
+
|-----|--------|
|
|
329
|
+
| `@api` | An HTTP API test, with no browser ([API tests](api.md)) |
|
|
330
|
+
| `@side-effects` | It changes the app's data; `run_plan` counts how often it runs |
|
|
331
|
+
| `@serial` | Never runs at the same time as another `@serial` scenario |
|
|
332
|
+
| `@serial:<name>` | Never runs at the same time as another scenario with that name (`@serial:rooms`); any case, and commas give several (`@serial:rooms,users`) |
|
|
333
|
+
|
|
334
|
+
`@serial` holds across workers, the record and heal phases, a heal's proof
|
|
335
|
+
replays of an Outline's other rows, and `--shard` (one name's scenarios go to
|
|
336
|
+
one machine). `saffron eject` writes `workers: 1` into the Playwright config it
|
|
337
|
+
creates for such a suite. Tag the scenarios that share data the app keeps
|
|
338
|
+
(one account, one booking slot) instead of running the whole suite with
|
|
339
|
+
`--workers 1`.
|
|
340
|
+
|
|
325
341
|
## A complete file
|
|
326
342
|
|
|
327
343
|
How the pieces fit in one project. The shared sign-in lives in a
|
|
@@ -12,7 +12,7 @@ import { createHash, randomBytes } from "node:crypto";
|
|
|
12
12
|
import dns from "node:dns";
|
|
13
13
|
import fs from "node:fs";
|
|
14
14
|
import path from "node:path";
|
|
15
|
-
import { expect, test, type APIResponse, type Browser, type BrowserContext, type Dialog, type Download, type ElementHandle, type Locator, type Page, type Request, type Response } from "@playwright/test";
|
|
15
|
+
import { expect, test, type APIResponse, type Browser, type BrowserContext, type Dialog, type Download, type ElementHandle, type Frame, type Locator, type Page, type Request, type Response } from "@playwright/test";
|
|
16
16
|
|
|
17
17
|
const SETTINGS: {
|
|
18
18
|
/** Locale and time zone {date...} tokens are written in. */
|
|
@@ -212,7 +212,7 @@ export function uploadFile(file: string): string {
|
|
|
212
212
|
const LETTER = String.raw`[\p{L}\p{N}\p{M}]`;
|
|
213
213
|
// A script written without spaces (Chinese, Japanese, Thai...): an edge of it is not bounded, as in replay, but for
|
|
214
214
|
// a negating prefix before it ("成功" is not in "不成功").
|
|
215
|
-
const SPACELESS = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]/u;
|
|
215
|
+
const SPACELESS = /(?!\p{N})[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]/u;
|
|
216
216
|
const NEGATING = "无法|無法|不能|没有|沒有|未能|ไม่ได้|不|未|非|無|无|没|沒|ไม่";
|
|
217
217
|
|
|
218
218
|
/**
|
|
@@ -221,20 +221,26 @@ const NEGATING = "无法|無法|不能|没有|沒有|未能|ไม่ได้|
|
|
|
221
221
|
* the step wrote, a zero decimal part may follow ("$10" is in "$10.00", not
|
|
222
222
|
* "$10.50"); after a row or data value (`rows` "end"), none.
|
|
223
223
|
*/
|
|
224
|
-
export function bounded(text: string, edges: "start" | "end" | "both", flags = "u", rows: "" | "end" = ""): RegExp {
|
|
224
|
+
export function bounded(text: string, edges: "start" | "end" | "both", flags = "u", rows: "" | "end" = "", grouped = true): RegExp {
|
|
225
225
|
text = said(normalize(text), "the expected text");
|
|
226
226
|
const first = [...text][0];
|
|
227
227
|
// An amount ("$5.00") is never the negative one after a minus ("-$5.00"), as replay reads it.
|
|
228
228
|
const amount = /^\p{Sc}\s?\p{N}/u.test(text);
|
|
229
229
|
const start = edges !== "end" && (amount || (new RegExp(`^${LETTER}`, "u").test(text) && !(SPACELESS.test(first) && new RegExp(`^(?:${NEGATING})`, "u").test(text))));
|
|
230
230
|
const end = edges !== "start" && new RegExp(`${LETTER}$`, "u").test(text) && !SPACELESS.test([...text].at(-1)!);
|
|
231
|
-
|
|
232
|
-
|
|
231
|
+
// Thousands grouped by a space or an apostrophe ("1 250 kr") are one number, and a currency symbol after a letter ("R$") another currency, as replay reads them.
|
|
232
|
+
// Not for "not shown" (grouped false): there a number that might run on into the next ("15 128") counts as shown.
|
|
233
|
+
const group = "[\\x20\\u00a0\\u202f\\u2007\\u2009\\u200a\\u0027\\u2019]";
|
|
234
|
+
// For "not shown", only a decimal point joins two numbers: "2" counts as shown in "1,2,3", as replay reads it.
|
|
235
|
+
const point = grouped ? "[.,\\u066b\\u066c]" : "[.\\u066b]";
|
|
236
|
+
const groupAfter = grouped && /(?:^|[^\p{L}\p{N}\p{M}])\p{N}{1,3}$/u.test(text) ? `|${group}\\p{N}{3}(?!\\p{N})` : "";
|
|
237
|
+
const before = SPACELESS.test(first) ? `(?<!${NEGATING})` : /^\p{N}/u.test(text) ? `(?<!${LETTER}|\\p{N}${point}|(?:^|[^\\p{L}\\p{N}])[-\\u2212\\u2013]${grouped && /^\p{N}{3}(?!\p{N})/u.test(text) ? `|(?:^|[^\\p{L}\\p{N}\\p{M}])\\p{N}{1,3}${group}` : ""})` : amount ? `(?<!${grouped ? "\\p{L}|" : ""}(?:^|[^\\p{L}\\p{N}])[-\\u2212\\u2013])` : `(?<!${LETTER})`;
|
|
238
|
+
const after = !/\p{N}$/u.test(text) ? `(?!${LETTER})` : rows === "end" ? `(?!${LETTER}|${point}\\p{N}${groupAfter})` : `(?!${LETTER}|${point}(?!0{1,2}(?!\\p{N}|${point}\\p{N}))\\p{N}${groupAfter})`;
|
|
233
239
|
// Quotes and `>` as hex, so Playwright reads no string or chain in the regex; any whitespace as replay's label match allows.
|
|
234
240
|
const body = escapeRegExp(text).replace(/ /g, "\\s+").replace(/["'`>]/g, (c) => `\\x${c.charCodeAt(0).toString(16)}`);
|
|
235
241
|
// After a number's point or comma ("$10."), no more digits, as replay reads it.
|
|
236
|
-
const
|
|
237
|
-
return new RegExp(`${start ? before : ""}${body}${end ? after :
|
|
242
|
+
const pointEnd = /\p{N}[.,\u066b\u066c]$/u.test(text) ? "(?!\\p{N})" : "";
|
|
243
|
+
return new RegExp(`${start ? before : ""}${body}${end ? after : pointEnd}`, flags);
|
|
238
244
|
}
|
|
239
245
|
|
|
240
246
|
const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
@@ -415,6 +421,10 @@ const decode = (url: string) => {
|
|
|
415
421
|
}
|
|
416
422
|
};
|
|
417
423
|
|
|
424
|
+
/** Raw, a match ending on a letter or digit written percent-encoded is part of a longer word ("Jos" in "Jos%C3%A9"), as replay reads it. */
|
|
425
|
+
const rawUrlMatch = (re: RegExp, url: string) =>
|
|
426
|
+
[...url.matchAll(new RegExp(re.source, re.flags.replace("g", "") + "g"))].some((m) => !/^%(?!3[Ff]|26|3[Dd]|23|2[Ff])[0-9A-Fa-f]{2}/.test(url.slice(m.index! + m[0].length)));
|
|
427
|
+
|
|
418
428
|
function statusMatches(observed: number, expected?: string): boolean {
|
|
419
429
|
if (!expected) return observed >= 200 && observed < 300;
|
|
420
430
|
const cls = /^([1-5])xx$/i.exec(expected.trim());
|
|
@@ -768,6 +778,34 @@ export class Run {
|
|
|
768
778
|
await (await chooser).setFiles(files.map(uploadFile));
|
|
769
779
|
}
|
|
770
780
|
|
|
781
|
+
/**
|
|
782
|
+
* Nothing the recording names is shown, in the page and (`ownFrames`) the app's own frames that are shown, once
|
|
783
|
+
* open requests are answered and the hidden state has held for half a second, as replay checks a "not shown":
|
|
784
|
+
* an error drawn 300 ms after the click is shown (found in the 1.3.5 review).
|
|
785
|
+
*/
|
|
786
|
+
async expectNotShown(build: (root: Page | Frame) => Locator[], ownFrames = true): Promise<void> {
|
|
787
|
+
await this.lane.net.quiet(Math.min(SETTINGS.timeoutMs / 2, 3000));
|
|
788
|
+
const settle = Math.min(500, SETTINGS.timeoutMs / 4);
|
|
789
|
+
let since: number | undefined;
|
|
790
|
+
await expect
|
|
791
|
+
.poll(
|
|
792
|
+
async () => {
|
|
793
|
+
for (const root of [this.page, ...(ownFrames ? await shownOwnFrames(this.page) : [])]) {
|
|
794
|
+
for (const target of build(root)) {
|
|
795
|
+
if ((await target.filter({ visible: true }).count()) > 0) {
|
|
796
|
+
since = undefined;
|
|
797
|
+
return false;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
since ??= Date.now();
|
|
802
|
+
return Date.now() - since >= settle;
|
|
803
|
+
},
|
|
804
|
+
{ message: "every match of every recorded locator to be hidden, and to stay hidden", timeout: SETTINGS.timeoutMs + settle },
|
|
805
|
+
)
|
|
806
|
+
.toBe(true);
|
|
807
|
+
}
|
|
808
|
+
|
|
771
809
|
/** The text `target` shows, once open requests are answered and the text has held for half a second. */
|
|
772
810
|
async capture(name: string, target: Locator): Promise<void> {
|
|
773
811
|
await this.lane.net.quiet(Math.min(SETTINGS.timeoutMs / 2, 3000));
|
|
@@ -830,7 +868,7 @@ export class Run {
|
|
|
830
868
|
problem = "problem" in verdict ? verdict.problem : undefined;
|
|
831
869
|
if (!("differ" in verdict) || !verdict.differ) return false;
|
|
832
870
|
// The element must still be one the target names.
|
|
833
|
-
return target.evaluateAll((els, el) => els.includes(el as
|
|
871
|
+
return target.evaluateAll((els, el) => els.includes(el as HTMLElement), element).catch(() => false);
|
|
834
872
|
}, { message: `target text to differ from "${compareTo}" (${right})`, timeout: SETTINGS.timeoutMs })
|
|
835
873
|
.toBe(true)
|
|
836
874
|
.catch((err) => {
|
|
@@ -854,7 +892,7 @@ export class Run {
|
|
|
854
892
|
() =>
|
|
855
893
|
(kind === "check" ? net.sinceAction() : net.sincePreviousStep()).some(
|
|
856
894
|
(e) =>
|
|
857
|
-
(url
|
|
895
|
+
(rawUrlMatch(url, e.url) || url.test(decode(e.url))) &&
|
|
858
896
|
(!match.method || e.method.toUpperCase() === match.method.toUpperCase()) &&
|
|
859
897
|
statusMatches(e.status, match.status) &&
|
|
860
898
|
(!match.json || masked(() => checkResponseJson(e.body, match.json!)) === undefined) &&
|
|
@@ -1017,7 +1055,12 @@ export class Run {
|
|
|
1017
1055
|
}
|
|
1018
1056
|
const value = at(parsed, pointer);
|
|
1019
1057
|
if (value === undefined) throw new Error(mask(`${method} ${url}: the response has no ${pointer || "/"} to extract as "${name}"${excerpt(exchange.body)}`));
|
|
1020
|
-
|
|
1058
|
+
// A list or object keeps its numbers as the API wrote them, numbers still, as replay keeps them (found in the 1.3.5 review).
|
|
1059
|
+
const raw = (JSON as unknown as { rawJSON(source: string): unknown }).rawJSON;
|
|
1060
|
+
const kept = value !== null && typeof value === "object" && !(value instanceof ExactNumber)
|
|
1061
|
+
? JSON.stringify(at(JSON.parse(exchange.body ?? "", (_key, v: unknown, context?: { source?: string }) => (typeof v === "number" && context?.source !== undefined ? raw(context.source) : v)), pointer))
|
|
1062
|
+
: JSON.stringify(value);
|
|
1063
|
+
this.vars.set(name, typeof value === "string" ? value : kept);
|
|
1021
1064
|
}
|
|
1022
1065
|
}
|
|
1023
1066
|
|
|
@@ -1238,6 +1281,20 @@ export async function expectNotVisible(...targets: Locator[]): Promise<void> {
|
|
|
1238
1281
|
}, `every match of every recorded locator to be hidden (${targets.length} locator(s))`);
|
|
1239
1282
|
}
|
|
1240
1283
|
|
|
1284
|
+
async function shownOwnFrames(page: Page): Promise<Frame[]> {
|
|
1285
|
+
const origin = (url: string) => { try { return new URL(url).origin; } catch { return undefined; } };
|
|
1286
|
+
const own = origin(page.mainFrame().url());
|
|
1287
|
+
const frames: Frame[] = [];
|
|
1288
|
+
for (const frame of page.frames()) {
|
|
1289
|
+
if (frame === page.mainFrame() || !(/^about:(?:srcdoc|blank)$|^$/.test(frame.url()) || origin(frame.url()) === own)) continue;
|
|
1290
|
+
const element = await frame.frameElement().catch(() => undefined);
|
|
1291
|
+
const box = element && (await element.isVisible().catch(() => false)) ? await element.boundingBox().catch(() => null) : null;
|
|
1292
|
+
await element?.dispose().catch(() => {});
|
|
1293
|
+
if (box && box.width > 1 && box.height > 1 && box.x + box.width > 0 && box.y + box.height > 0) frames.push(frame);
|
|
1294
|
+
}
|
|
1295
|
+
return frames;
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1241
1298
|
/** Each value is shown in the container, as a whole line of its text. */
|
|
1242
1299
|
export async function expectEach(container: Locator, values: string[]): Promise<void> {
|
|
1243
1300
|
let missing = values;
|
|
@@ -1703,9 +1760,19 @@ function checkValue(json: unknown, checks: ResponseJsonCheck[], prefix = ""): st
|
|
|
1703
1760
|
// Keys in any order and any spacing: {"b":2,"a":1} is the same as {"a": 1, "b": 2}.
|
|
1704
1761
|
const canonical = (v: unknown) => JSON.stringify(v, (_key, x: unknown) => (x && typeof x === "object" && !Array.isArray(x) ? Object.fromEntries(Object.entries(x).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) : x));
|
|
1705
1762
|
const written = (text: unknown) => { try { return canonical(parseChecked(String(text))); } catch { return String(text); } };
|
|
1706
|
-
|
|
1763
|
+
// A number is the same however the step writes it: 10.5 is "10.50" and 100 is "1e2" (found in the 1.3.5 review).
|
|
1764
|
+
const sameNumber = (v: unknown) => {
|
|
1765
|
+
const [a, b] = [typeof v === "number" || v instanceof ExactNumber ? decimal(String(v)) : undefined, typeof other === "string" ? decimal(other) : undefined];
|
|
1766
|
+
return a !== undefined && b !== undefined && compareDecimal(a, b) === 0;
|
|
1767
|
+
};
|
|
1768
|
+
// Checked as a string, a number, true or null is its text, as equals reads it: "differs" on the same id fails (found in the 1.3.5 review).
|
|
1769
|
+
if (value === other || (check.type === "string" && (value === null || typeof value !== "object" || value instanceof ExactNumber) && String(value) === other) || (check.type === undefined && (value !== null && typeof value === "object" && !(value instanceof ExactNumber) ? canonical(value) === written(other) : String(value) === other || sameNumber(value)))) return `${field}: expected a value other than ${describe(other)}, got the same`;
|
|
1707
1770
|
} else if (check.pattern !== undefined) {
|
|
1708
1771
|
const re = jsonPattern(check);
|
|
1772
|
+
// "A non-empty field" holds on a number too: an id is often one (found in the 1.3.5 review). Any other
|
|
1773
|
+
// pattern reads text only, so "1" never passes on the number 10.
|
|
1774
|
+
const nonEmptyNumber = check.pattern === NON_EMPTY && (typeof value === "number" || value instanceof ExactNumber);
|
|
1775
|
+
if (nonEmptyNumber) continue;
|
|
1709
1776
|
if (typeof value !== "string") return `${field}: expected text matching /${check.pattern}/, got ${got}`;
|
|
1710
1777
|
if (!re.test(value)) return `${field}: expected text matching /${check.pattern}/, got ${JSON.stringify(value)}`;
|
|
1711
1778
|
} else if (check.min !== undefined || check.max !== undefined) {
|
|
@@ -1736,14 +1803,20 @@ function checkValue(json: unknown, checks: ResponseJsonCheck[], prefix = ""): st
|
|
|
1736
1803
|
return undefined;
|
|
1737
1804
|
}
|
|
1738
1805
|
|
|
1806
|
+
/** The pattern "should have a non-empty field" compiles to: text with a character other than a space, or any number. */
|
|
1807
|
+
export const NON_EMPTY = "\\S";
|
|
1808
|
+
|
|
1739
1809
|
/**
|
|
1740
1810
|
* JSON with an integer beyond JavaScript's precision (a snowflake or int64
|
|
1741
|
-
* id) kept as the digits the API sent, never rounded into another id, as a
|
|
1811
|
+
* id), or a decimal it cannot hold, kept as the digits the API sent, never rounded into another id, as a
|
|
1742
1812
|
* capture keeps it: a differs check then sees the same id (found in review).
|
|
1743
1813
|
*/
|
|
1814
|
+
|
|
1744
1815
|
export function parseExact(text: string): unknown {
|
|
1816
|
+
// As a check reads it (parseChecked): a decimal JavaScript would round keeps its digits too, so a differs check
|
|
1817
|
+
// never passes on an unchanged balance, nor a later request send another number (found in the 1.3.5 review).
|
|
1745
1818
|
return JSON.parse(text, (_key, value: unknown, context?: { source?: string }) =>
|
|
1746
|
-
typeof value === "number" &&
|
|
1819
|
+
typeof value === "number" && context?.source !== undefined && (Math.abs(value) > Number.MAX_SAFE_INTEGER || !heldAsWritten(context.source, value)) ? context.source : value);
|
|
1747
1820
|
}
|
|
1748
1821
|
|
|
1749
1822
|
/** A missing property is different from a property whose value is null. */
|