playwright-mutation-gate 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -4
- package/dist/behavior.d.ts +34 -0
- package/dist/behavior.js +167 -0
- package/dist/cli.js +10 -2
- package/dist/extract.d.ts +21 -0
- package/dist/extract.js +103 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +3 -1
- package/dist/mutate.d.ts +101 -0
- package/dist/mutate.js +156 -31
- package/dist/report.d.ts +29 -1
- package/dist/report.js +93 -3
- package/dist/run.d.ts +54 -1
- package/dist/run.js +149 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -109,6 +109,36 @@ npx playwright-mutation-gate tests/dashboard.spec.ts --test-id authedTest
|
|
|
109
109
|
A runtime `authedTest.skip(condition)` inside a test is respected: if the test
|
|
110
110
|
skips, the mutation reports `cannot-mutate` rather than a false `survived`.
|
|
111
111
|
|
|
112
|
+
## Acknowledged skips
|
|
113
|
+
|
|
114
|
+
A skipped test reports `cannot-mutate` and fails the gate: its assertion never
|
|
115
|
+
runs, so there is nothing to verify. That is the right default, but it leaves no
|
|
116
|
+
room for a skip the team has *consciously* accepted for a while. The two escapes
|
|
117
|
+
are both bad: keep the gate red until someone fixes the test (so people stop
|
|
118
|
+
running it), or work around the skip so it quietly lives for months with no
|
|
119
|
+
owner and no deadline.
|
|
120
|
+
|
|
121
|
+
A `// @gate-skip` marker turns that skip into a loan with a due date. Put it on
|
|
122
|
+
its own line right above the test, or trailing the declaration:
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
// @gate-skip owner=vlad expires=2026-08-15 reason="checkout redesign in flight"
|
|
126
|
+
test.skip('checkout shows confirmation', async ({ page }) => {
|
|
127
|
+
// @primary-assert
|
|
128
|
+
await expect(page.getByText('Order confirmed')).toBeVisible();
|
|
129
|
+
});
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
- Before `expires`, the verdict stays `cannot-mutate` but is reported as
|
|
133
|
+
**acknowledged** (owner, expiry and reason in the table and `--json`), and the
|
|
134
|
+
run can exit 0.
|
|
135
|
+
- After `expires`, the gate **fails closed** again. Extending the deadline means
|
|
136
|
+
editing the marker, a deliberate commit, not silent drift.
|
|
137
|
+
- `owner` and a real `expires=YYYY-MM-DD` are required. A marker missing either,
|
|
138
|
+
with an unparseable date, or not sitting on a test is **malformed** and fails
|
|
139
|
+
the gate, same as a malformed `@primary-assert`.
|
|
140
|
+
- Unmarked skips keep the default: `cannot-mutate`, gate fails.
|
|
141
|
+
|
|
112
142
|
## CI recipe
|
|
113
143
|
|
|
114
144
|
```yaml
|
|
@@ -152,6 +182,116 @@ do not want the full sweep on every push. Two patterns that work well:
|
|
|
152
182
|
catching it a day later is fine. `--playwright-args` is available if you
|
|
153
183
|
need to shard or point at a specific config.
|
|
154
184
|
|
|
185
|
+
## Behavior mutation mode (`--behavior`)
|
|
186
|
+
|
|
187
|
+
Assertion inversion answers "can this test fail". It cannot answer "does this
|
|
188
|
+
test fail when the thing it names breaks". An assertion on a value that renders
|
|
189
|
+
on every page is killed by inversion and clears the gate, even if the feature it
|
|
190
|
+
claims to check is broken.
|
|
191
|
+
|
|
192
|
+
`--behavior` adds a second, opt-in pass that closes part of that gap. Instead of
|
|
193
|
+
inverting the assertion, it keeps it intact and corrupts the *value the
|
|
194
|
+
assertion names* in the response, using Playwright's own `page.route()` to
|
|
195
|
+
rewrite the body and headers in transit. Then it re-runs and expects red.
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
npx playwright-mutation-gate "tests/**/*.spec.ts" --behavior
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
- **killed** -- the test failed once its named value was broken. (A red here can
|
|
202
|
+
also be flow breakage rather than the assertion itself; the report says so.)
|
|
203
|
+
- **weak** -- the test stayed green while the value it names was broken. The
|
|
204
|
+
assertion does not actually depend on the behavior it claims to check. This
|
|
205
|
+
fails the gate.
|
|
206
|
+
- **cannot-locate** -- the named value never appeared in a rewritable response,
|
|
207
|
+
so nothing was corrupted. Reported honestly and, unlike `cannot-mutate`, this
|
|
208
|
+
does **not** fail the gate: a value that is not in an interceptable response
|
|
209
|
+
(a visibility check, a client-computed string, an API-request-fixture call) is
|
|
210
|
+
expected, not a defect.
|
|
211
|
+
|
|
212
|
+
### What the gate does for you
|
|
213
|
+
|
|
214
|
+
You write an ordinary test and mark its main assertion. The gate reads the value
|
|
215
|
+
out of that assertion, writes a throwaway copy of the spec with a `page.route()`
|
|
216
|
+
preamble injected at the top of the test, runs it, and deletes the copy. Your
|
|
217
|
+
spec and the app under test are never touched. The interception happens in the
|
|
218
|
+
browser at the network layer, so it works against any URL the test already drives
|
|
219
|
+
(local, staging, a deployed site) with no access to the app's source.
|
|
220
|
+
|
|
221
|
+
The preamble it generates per assertion is just this:
|
|
222
|
+
|
|
223
|
+
```ts
|
|
224
|
+
await page.route('**/*', async (route) => {
|
|
225
|
+
const resp = await route.fetch();
|
|
226
|
+
let body = await resp.text();
|
|
227
|
+
if (body.includes('Thanks, Ada!')) {
|
|
228
|
+
body = body.split('Thanks, Ada!').join('PMG_MUTATED_VALUE'); // break the named value in transit
|
|
229
|
+
}
|
|
230
|
+
await route.fulfill({ response: resp, body });
|
|
231
|
+
});
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
### A worked example
|
|
235
|
+
|
|
236
|
+
A test that reads fine on review but checks the wrong layer:
|
|
237
|
+
|
|
238
|
+
```ts
|
|
239
|
+
// The confirmation greeting is server-rendered, then re-rendered on the client
|
|
240
|
+
// from a data-name attribute. This assertion checks the client render.
|
|
241
|
+
await expect(page.getByTestId('receipt-greeting')).toHaveText('Thanks, Ada!'); // @primary-assert
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Inversion kills it (flip the text and the test goes red), so it clears the
|
|
245
|
+
standard gate. Behavior mutation corrupts `Thanks, Ada!` in the response; the
|
|
246
|
+
client rebuilds it from `data-name`, the assertion stays green:
|
|
247
|
+
|
|
248
|
+
```text
|
|
249
|
+
tests/receipt.spec.ts
|
|
250
|
+
behavior:
|
|
251
|
+
weak Greets the customer by name on the confirmation page :22
|
|
252
|
+
the test stayed green while the value it names was broken
|
|
253
|
+
|
|
254
|
+
behavior: 0 killed, 1 weak, 0 cannot-locate
|
|
255
|
+
Gate FAILED: hollow, unverified or broken assertions above.
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
A runnable version ships in the
|
|
259
|
+
[ai-qa-pipeline](https://github.com/VladyslavDmitriiev/ai-qa-pipeline) demo repo.
|
|
260
|
+
Clone it and run:
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
npx playwright-mutation-gate tests/behavior-demo/receipt.spec.ts --behavior
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
The two modes compose and neither replaces the other. Inversion is cheap and
|
|
267
|
+
catches assertions that cannot fail at all (dead selectors, missing `await`,
|
|
268
|
+
silent skips). Behavior mutation is expensive and catches weak assertions.
|
|
269
|
+
Running `--behavior` performs both passes, so **the run cost roughly doubles**;
|
|
270
|
+
keep it for a nightly sweep or the specs a PR touches rather than every push.
|
|
271
|
+
|
|
272
|
+
**v1 scope: exact string values.** Behavior mutation locates a plain string
|
|
273
|
+
literal argument (`toContain`, `toContainText`, `toHaveText`, `toHaveValue`,
|
|
274
|
+
`toBe('...')`) and corrupts every occurrence in HTML/JSON bodies and header
|
|
275
|
+
values. Numbers, status codes, regex/URL matchers, `toHaveCount`,
|
|
276
|
+
`toBeVisible`, negated matchers, interpolated values and request-only fixtures
|
|
277
|
+
all report `cannot-locate`. Values that are template-rendered or reconstructed
|
|
278
|
+
on the client (common on SPAs) also report `cannot-locate`, since the literal
|
|
279
|
+
never reaches an interceptable response. As with inversion, the gate reports
|
|
280
|
+
what it could not do rather than guessing.
|
|
281
|
+
|
|
282
|
+
**Which stacks it fits.** Behavior mutation needs the asserted string to appear
|
|
283
|
+
verbatim in an HTTP response it can intercept. It fits server-rendered HTML
|
|
284
|
+
(Rails, Django, Laravel, Express templates, Next.js and Nuxt SSR) and any app
|
|
285
|
+
whose API returns the exact string the UI shows. It goes mostly blind on
|
|
286
|
+
client-rendered SPAs that build the text in the browser: if a React or Vue app
|
|
287
|
+
fetches `{"totalCents": 3400}` and renders `$34.00`, or assembles the string
|
|
288
|
+
through i18n or concatenation, the literal never crosses the wire and every such
|
|
289
|
+
assertion comes back `cannot-locate`. The same is true of formatted dates,
|
|
290
|
+
pluralized counts, and anything derived client-side. A mismatched stack costs you
|
|
291
|
+
nothing, since `cannot-locate` does not fail the gate, but it will not gain you
|
|
292
|
+
much either. Inversion still works everywhere; only the behavior pass is
|
|
293
|
+
stack-sensitive.
|
|
294
|
+
|
|
155
295
|
## CLI reference
|
|
156
296
|
|
|
157
297
|
```
|
|
@@ -160,6 +300,8 @@ npx playwright-mutation-gate [options] <specs...>
|
|
|
160
300
|
Options:
|
|
161
301
|
--json machine-readable JSON report
|
|
162
302
|
--no-require-sentinel do not fail on tests without a marker
|
|
303
|
+
--behavior also corrupt each asserted value in the response
|
|
304
|
+
and expect red (weak assertions fail the gate)
|
|
163
305
|
--playwright-args <args> extra arguments for playwright test
|
|
164
306
|
--test-id <name> treat <name> as a test builder (repeatable)
|
|
165
307
|
-V, --version output the version number
|
|
@@ -192,9 +334,12 @@ if (!summary.gatePassed) {
|
|
|
192
334
|
`killed` / `survived` / `cannot-mutate` / `error`, plus unmarked/stray/malformed
|
|
193
335
|
markers). `summarize(specs, opts)` folds one or more results into a `Summary`
|
|
194
336
|
with the `gatePassed` verdict. `jsonReport` and `renderTable` produce the CLI's
|
|
195
|
-
two output formats.
|
|
196
|
-
the
|
|
197
|
-
|
|
337
|
+
two output formats. Pass `{ behavior: true }` to also run the behavior pass;
|
|
338
|
+
the result then carries a `behavior` array of `BehaviorResult`s (`killed` /
|
|
339
|
+
`weak` / `cannot-locate` / `error`). Advanced: `planSpecMutations`,
|
|
340
|
+
`planBehaviorMutations`, `invertAssertLine` and `extractAssertValue` expose the
|
|
341
|
+
planners, the single-line inverter and the value extractor directly. Full types
|
|
342
|
+
ship with the package.
|
|
198
343
|
|
|
199
344
|
## Limitations
|
|
200
345
|
|
|
@@ -206,6 +351,9 @@ guarantees is narrower and mechanical: every marked assertion actually runs and
|
|
|
206
351
|
is load-bearing, so the test cannot stay green regardless of what the app does.
|
|
207
352
|
Judging whether the assertion is the *right* one still takes review, or
|
|
208
353
|
application-code mutation testing (Stryker-style), which is a different tool.
|
|
354
|
+
[Behavior mutation mode](#behavior-mutation-mode---behavior) (`--behavior`)
|
|
355
|
+
closes part of this gap for string-valued assertions by corrupting the named
|
|
356
|
+
value in the response and expecting red.
|
|
209
357
|
|
|
210
358
|
**A kill is slightly generous by construction.** `killed` means the mutated
|
|
211
359
|
run failed, not that the flipped assertion is what failed. Playwright actions
|
|
@@ -240,7 +388,9 @@ test was skipped at runtime (conditional test.skip), so the inverted
|
|
|
240
388
|
assertion never ran"). In
|
|
241
389
|
practice this is a feature: a `test.skip(true, ...)` from months ago looks
|
|
242
390
|
identical to a healthy passing spec in normal CI output, and the gate is
|
|
243
|
-
often what surfaces it.
|
|
391
|
+
often what surfaces it. When a skip is a conscious, temporary decision, an
|
|
392
|
+
[`// @gate-skip`](#acknowledged-skips) marker records its owner and expiry so
|
|
393
|
+
the gate accepts it until the deadline instead of staying red.
|
|
244
394
|
|
|
245
395
|
## Why
|
|
246
396
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type PlanOptions } from './mutate.js';
|
|
2
|
+
/**
|
|
3
|
+
* A behavior mutation for one marked test: either an injected copy that
|
|
4
|
+
* corrupts the asserted value in transit, or a cannot-locate reason. Unlike
|
|
5
|
+
* inversion (which rewrites the assertion), behavior mutation keeps the
|
|
6
|
+
* assertion intact and injects a `page.route()` preamble that rewrites the
|
|
7
|
+
* value the assertion names in the response, then expects the test to go red.
|
|
8
|
+
*/
|
|
9
|
+
export type BehaviorMutation = {
|
|
10
|
+
testTitle: string;
|
|
11
|
+
testLine: number;
|
|
12
|
+
markerLine: number;
|
|
13
|
+
/** 1-based line of the assertion whose value is corrupted. */
|
|
14
|
+
assertLine: number;
|
|
15
|
+
/** The string value corrupted in the response. */
|
|
16
|
+
value: string;
|
|
17
|
+
/** Full spec source with the route-interception preamble injected. */
|
|
18
|
+
mutatedSource: string;
|
|
19
|
+
} | {
|
|
20
|
+
testTitle: string;
|
|
21
|
+
testLine: number;
|
|
22
|
+
markerLine: number;
|
|
23
|
+
assertLine: number | null;
|
|
24
|
+
cannotLocate: string;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Plan behavior mutations for a spec: for each marked test, extract the value
|
|
28
|
+
* its `@primary-assert` names, confirm the test can be routed (a `page`
|
|
29
|
+
* fixture is destructured), and inject a response-corrupting preamble. Every
|
|
30
|
+
* test the gate cannot mutate this way is reported with a reason, never
|
|
31
|
+
* skipped silently. Marker-level findings (unmarked, stray, malformed) are the
|
|
32
|
+
* inversion pass's job and are not repeated here.
|
|
33
|
+
*/
|
|
34
|
+
export declare function planBehaviorMutations(source: string, opts?: PlanOptions): BehaviorMutation[];
|
package/dist/behavior.js
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { extractAssertValue } from './extract.js';
|
|
2
|
+
import { CODE } from './lex.js';
|
|
3
|
+
import { analyzeSpec, UNGATEABLE_MODES } from './mutate.js';
|
|
4
|
+
/**
|
|
5
|
+
* The token the injected handler substitutes for the asserted value. Distinct
|
|
6
|
+
* enough that it will not collide with real content and will fail any equality
|
|
7
|
+
* or substring assertion on the original value.
|
|
8
|
+
*/
|
|
9
|
+
const REPLACE = 'PMG_MUTATED_VALUE';
|
|
10
|
+
/**
|
|
11
|
+
* Plan behavior mutations for a spec: for each marked test, extract the value
|
|
12
|
+
* its `@primary-assert` names, confirm the test can be routed (a `page`
|
|
13
|
+
* fixture is destructured), and inject a response-corrupting preamble. Every
|
|
14
|
+
* test the gate cannot mutate this way is reported with a reason, never
|
|
15
|
+
* skipped silently. Marker-level findings (unmarked, stray, malformed) are the
|
|
16
|
+
* inversion pass's job and are not repeated here.
|
|
17
|
+
*/
|
|
18
|
+
export function planBehaviorMutations(source, opts = {}) {
|
|
19
|
+
const analysis = analyzeSpec(source, opts);
|
|
20
|
+
const { lines, kind } = analysis;
|
|
21
|
+
const mutations = [];
|
|
22
|
+
for (const t of analysis.markedTests) {
|
|
23
|
+
const base = { testTitle: t.title, testLine: t.testLine, markerLine: t.markerLine };
|
|
24
|
+
if (UNGATEABLE_MODES.has(t.mode)) {
|
|
25
|
+
const why = t.mode === 'fail'
|
|
26
|
+
? 'test.fail() expects failure, so a behavior verdict would be meaningless'
|
|
27
|
+
: `test.${t.mode}() never runs, so behavior cannot be observed`;
|
|
28
|
+
mutations.push({ ...base, assertLine: t.assertLine, cannotLocate: why });
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
if (t.markerCount > 1) {
|
|
32
|
+
mutations.push({
|
|
33
|
+
...base,
|
|
34
|
+
assertLine: t.assertLine,
|
|
35
|
+
cannotLocate: `found ${t.markerCount} @primary-assert markers in one test, expected exactly one`,
|
|
36
|
+
});
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (t.assertLine === null) {
|
|
40
|
+
mutations.push({ ...base, assertLine: null, cannotLocate: t.resolveError ?? 'assertion line not found' });
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
const extracted = extractAssertValue(lines[t.assertLine - 1]);
|
|
44
|
+
if ('cannotLocate' in extracted) {
|
|
45
|
+
mutations.push({ ...base, assertLine: t.assertLine, cannotLocate: extracted.cannotLocate });
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const callback = findCallback(source, kind, t.open, t.close);
|
|
49
|
+
if ('error' in callback) {
|
|
50
|
+
mutations.push({ ...base, assertLine: t.assertLine, cannotLocate: callback.error });
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (!callback.fixtures.includes('page')) {
|
|
54
|
+
mutations.push({
|
|
55
|
+
...base,
|
|
56
|
+
assertLine: t.assertLine,
|
|
57
|
+
cannotLocate: 'no page fixture destructured; route interception is unsupported for this test in v1',
|
|
58
|
+
});
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const preamble = buildPreamble(extracted.value);
|
|
62
|
+
const mutatedSource = source.slice(0, callback.bodyOpen + 1) + '\n' + preamble + source.slice(callback.bodyOpen + 1);
|
|
63
|
+
mutations.push({ ...base, assertLine: t.assertLine, value: extracted.value, mutatedSource });
|
|
64
|
+
}
|
|
65
|
+
return mutations;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Locate the test callback inside a test() call: its destructured fixtures and
|
|
69
|
+
* the `{` that opens its body. Supports the arrow-function form Playwright
|
|
70
|
+
* tests use (`async ({ page }) => { ... }`); other forms are reported so the
|
|
71
|
+
* caller can fall back to cannot-locate rather than inject blindly.
|
|
72
|
+
*/
|
|
73
|
+
function findCallback(src, kind, open, close) {
|
|
74
|
+
// The callback's `=>` is the first arrow in code context within the call.
|
|
75
|
+
let arrow = -1;
|
|
76
|
+
for (let i = open; i < close; i++) {
|
|
77
|
+
if (kind[i] === CODE && src[i] === '=' && src[i + 1] === '>') {
|
|
78
|
+
arrow = i;
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (arrow === -1)
|
|
83
|
+
return { error: 'could not locate the test callback (arrow function expected)' };
|
|
84
|
+
// The `)` closing the params sits just before the arrow.
|
|
85
|
+
let j = arrow - 1;
|
|
86
|
+
while (j > open && /\s/.test(src[j]))
|
|
87
|
+
j--;
|
|
88
|
+
if (src[j] !== ')')
|
|
89
|
+
return { error: 'unexpected test callback parameter form' };
|
|
90
|
+
let depth = 0;
|
|
91
|
+
let paramOpen = -1;
|
|
92
|
+
for (let k = j; k > open; k--) {
|
|
93
|
+
if (kind[k] !== CODE)
|
|
94
|
+
continue;
|
|
95
|
+
if (src[k] === ')')
|
|
96
|
+
depth++;
|
|
97
|
+
else if (src[k] === '(' && --depth === 0) {
|
|
98
|
+
paramOpen = k;
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (paramOpen === -1)
|
|
103
|
+
return { error: 'could not parse test callback parameters' };
|
|
104
|
+
const fixtures = parseDestructuredFixtures(src.slice(paramOpen + 1, j));
|
|
105
|
+
// The body opens at the first `{` in code context after the arrow.
|
|
106
|
+
let bodyOpen = -1;
|
|
107
|
+
for (let i = arrow + 2; i < close; i++) {
|
|
108
|
+
if (kind[i] === CODE && src[i] === '{') {
|
|
109
|
+
bodyOpen = i;
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (bodyOpen === -1)
|
|
114
|
+
return { error: 'test callback has no block body' };
|
|
115
|
+
return { fixtures, bodyOpen };
|
|
116
|
+
}
|
|
117
|
+
/** Read fixture names from a destructured params string like `{ page, request }`. */
|
|
118
|
+
function parseDestructuredFixtures(params) {
|
|
119
|
+
const trimmed = params.trim();
|
|
120
|
+
if (!trimmed.startsWith('{'))
|
|
121
|
+
return []; // `(arg)` form: fixtures not detectable
|
|
122
|
+
const end = trimmed.lastIndexOf('}');
|
|
123
|
+
if (end <= 0)
|
|
124
|
+
return [];
|
|
125
|
+
return trimmed
|
|
126
|
+
.slice(1, end)
|
|
127
|
+
.split(',')
|
|
128
|
+
.map((part) => part.split(':')[0].trim()) // `page: p` -> the source fixture name `page`
|
|
129
|
+
.filter(Boolean);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Build the route-interception preamble injected at the top of the test body.
|
|
133
|
+
* It fetches each response, replaces every occurrence of the asserted value in
|
|
134
|
+
* the headers and body with a sentinel, and records via PMG_HIT_FILE whether
|
|
135
|
+
* any occurrence was found. `await import('node:fs')` keeps the preamble
|
|
136
|
+
* self-contained, with no top-level import that could clash with the spec.
|
|
137
|
+
*/
|
|
138
|
+
function buildPreamble(value) {
|
|
139
|
+
const needle = JSON.stringify(value);
|
|
140
|
+
const replace = JSON.stringify(REPLACE);
|
|
141
|
+
return [
|
|
142
|
+
` await page.route('**/*', async (route) => {`,
|
|
143
|
+
` const resp = await route.fetch();`,
|
|
144
|
+
` const headers = resp.headers();`,
|
|
145
|
+
` const NEEDLE = ${needle};`,
|
|
146
|
+
` const REPLACE = ${replace};`,
|
|
147
|
+
` let hit = false;`,
|
|
148
|
+
` for (const k of Object.keys(headers)) {`,
|
|
149
|
+
` if (typeof headers[k] === 'string' && headers[k].includes(NEEDLE)) {`,
|
|
150
|
+
` headers[k] = headers[k].split(NEEDLE).join(REPLACE);`,
|
|
151
|
+
` hit = true;`,
|
|
152
|
+
` }`,
|
|
153
|
+
` }`,
|
|
154
|
+
` let body = null;`,
|
|
155
|
+
` try { body = await resp.text(); } catch {}`,
|
|
156
|
+
` if (body !== null && body.includes(NEEDLE)) {`,
|
|
157
|
+
` body = body.split(NEEDLE).join(REPLACE);`,
|
|
158
|
+
` hit = true;`,
|
|
159
|
+
` }`,
|
|
160
|
+
` if (hit && process.env.PMG_HIT_FILE) {`,
|
|
161
|
+
` const fs = await import('node:fs');`,
|
|
162
|
+
` fs.appendFileSync(process.env.PMG_HIT_FILE, '1');`,
|
|
163
|
+
` }`,
|
|
164
|
+
` await route.fulfill({ response: resp, headers, ...(body !== null ? { body } : {}) });`,
|
|
165
|
+
` });`,
|
|
166
|
+
].join('\n');
|
|
167
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -14,6 +14,9 @@ const program = new Command()
|
|
|
14
14
|
.argument('<specs...>', 'Playwright spec files to gate')
|
|
15
15
|
.option('--json', 'print a machine-readable JSON report instead of the table', false)
|
|
16
16
|
.option('--no-require-sentinel', 'do not fail the gate when a test() has no @primary-assert marker')
|
|
17
|
+
.option('--behavior', 'also run behavior mutation: corrupt each @primary-assert value in the response\n' +
|
|
18
|
+
'via page.route and expect red. A test that stays green is weak and fails the\n' +
|
|
19
|
+
'gate. Opt-in: this roughly doubles the run cost.', false)
|
|
17
20
|
.option('--playwright-args <args>', 'extra arguments passed to `npx playwright test`, split on spaces\n' +
|
|
18
21
|
'(values containing spaces are not supported)')
|
|
19
22
|
.option('--test-id <name>', 'treat <name> as a test builder in addition to `test` (repeatable);\n' +
|
|
@@ -23,7 +26,12 @@ const program = new Command()
|
|
|
23
26
|
.addHelpText('after', '\nMarking:\n' +
|
|
24
27
|
' Put `// @primary-assert` at the end of the primary assertion line,\n' +
|
|
25
28
|
' or on its own line right above it. One marker per test().\n' +
|
|
26
|
-
' await expect(page.getByText(\'Saved\')).toBeVisible(); // @primary-assert'
|
|
29
|
+
' await expect(page.getByText(\'Saved\')).toBeVisible(); // @primary-assert\n' +
|
|
30
|
+
'\nModes:\n' +
|
|
31
|
+
' Default: assertion inversion (proves the test can fail at all).\n' +
|
|
32
|
+
' --behavior: also corrupt the asserted value in the response and expect\n' +
|
|
33
|
+
' red (proves the test fails when the thing it names breaks). The modes\n' +
|
|
34
|
+
' compose; weak assertions fail the gate, cannot-locate is informational.')
|
|
27
35
|
.action(async (specPaths, opts) => {
|
|
28
36
|
const missing = specPaths.filter((p) => !existsSync(p));
|
|
29
37
|
if (missing.length > 0) {
|
|
@@ -35,7 +43,7 @@ const program = new Command()
|
|
|
35
43
|
for (const spec of specPaths) {
|
|
36
44
|
process.stderr.write(`gating ${spec}\n`);
|
|
37
45
|
try {
|
|
38
|
-
results.push(await runSpec(spec, { playwrightArgs, testIds: opts.testId }));
|
|
46
|
+
results.push(await runSpec(spec, { playwrightArgs, testIds: opts.testId, behavior: opts.behavior }));
|
|
39
47
|
}
|
|
40
48
|
catch (err) {
|
|
41
49
|
console.error(`failed to gate ${spec}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The value a `@primary-assert` assertion names, extracted for behavior
|
|
3
|
+
* mutation. `string` is a plain string literal the gate can locate in a
|
|
4
|
+
* response body/header and corrupt in transit. Everything the gate cannot
|
|
5
|
+
* honestly turn into a response mutation is `cannotLocate` with a reason,
|
|
6
|
+
* never guessed.
|
|
7
|
+
*/
|
|
8
|
+
export type AssertValue = {
|
|
9
|
+
kind: 'string';
|
|
10
|
+
value: string;
|
|
11
|
+
} | {
|
|
12
|
+
cannotLocate: string;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Extract the string value a single assertion line names, for behavior
|
|
16
|
+
* mutation. Purely string-based, reusing the same chain walk as invert.ts:
|
|
17
|
+
* `[await] expect[.soft|.poll](...) [.resolves|.rejects] .toXxx(<string>)`.
|
|
18
|
+
* A negated matcher (`.not`) has inverted semantics that behavior mutation
|
|
19
|
+
* does not model in v1, so it is reported as cannot-locate rather than guessed.
|
|
20
|
+
*/
|
|
21
|
+
export declare function extractAssertValue(line: string): AssertValue;
|
package/dist/extract.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { CODE, findCloseMasked, lex } from './lex.js';
|
|
2
|
+
// Matchers whose first argument, when a plain string literal, is the observed
|
|
3
|
+
// value the assertion depends on. Corrupting that string in the response is a
|
|
4
|
+
// meaningful behavior mutation. Others (toBeVisible, toHaveCount, toHaveURL
|
|
5
|
+
// with a regex, numeric toBe) do not carry a locatable string value in v1.
|
|
6
|
+
const STRING_MATCHERS = new Set(['toContain', 'toContainText', 'toHaveText', 'toHaveValue', 'toBe']);
|
|
7
|
+
const isWord = (c) => c !== undefined && /[\w$]/.test(c);
|
|
8
|
+
const ESCAPES = { n: '\n', t: '\t', r: '\r', b: '\b', f: '\f', v: '\v', '0': '\0' };
|
|
9
|
+
/**
|
|
10
|
+
* Extract the string value a single assertion line names, for behavior
|
|
11
|
+
* mutation. Purely string-based, reusing the same chain walk as invert.ts:
|
|
12
|
+
* `[await] expect[.soft|.poll](...) [.resolves|.rejects] .toXxx(<string>)`.
|
|
13
|
+
* A negated matcher (`.not`) has inverted semantics that behavior mutation
|
|
14
|
+
* does not model in v1, so it is reported as cannot-locate rather than guessed.
|
|
15
|
+
*/
|
|
16
|
+
export function extractAssertValue(line) {
|
|
17
|
+
const kind = lex(line);
|
|
18
|
+
// First CODE index at or after j: skips whitespace, comments and other
|
|
19
|
+
// non-code chars, matching invert.ts's walk.
|
|
20
|
+
const skip = (j) => {
|
|
21
|
+
while (j < line.length && (kind[j] !== CODE || /\s/.test(line[j])))
|
|
22
|
+
j++;
|
|
23
|
+
return j;
|
|
24
|
+
};
|
|
25
|
+
const anchors = [...line.matchAll(/\bexpect\s*[.(]/g)].filter((m) => kind[m.index] === CODE);
|
|
26
|
+
if (anchors.length === 0)
|
|
27
|
+
return { cannotLocate: 'no expect() call on this line' };
|
|
28
|
+
if (anchors.length > 1)
|
|
29
|
+
return { cannotLocate: 'multiple expect calls on one line' };
|
|
30
|
+
let i = anchors[0].index + anchors[0][0].length - 1;
|
|
31
|
+
if (line[i] === '.') {
|
|
32
|
+
i = skip(i + 1);
|
|
33
|
+
const start = i;
|
|
34
|
+
while (isWord(line[i]))
|
|
35
|
+
i++;
|
|
36
|
+
const form = line.slice(start, i);
|
|
37
|
+
if (form !== 'soft' && form !== 'poll')
|
|
38
|
+
return { cannotLocate: `unsupported expect form "expect.${form}"` };
|
|
39
|
+
i = skip(i);
|
|
40
|
+
if (line[i] !== '(')
|
|
41
|
+
return { cannotLocate: `expect.${form} is not called on this line` };
|
|
42
|
+
}
|
|
43
|
+
const close = findCloseMasked(line, kind, i);
|
|
44
|
+
if (close === -1)
|
|
45
|
+
return { cannotLocate: 'expect(...) does not close on this line (multi-line expect)' };
|
|
46
|
+
i = skip(close + 1);
|
|
47
|
+
for (;;) {
|
|
48
|
+
if (i >= line.length || line[i] !== '.')
|
|
49
|
+
return { cannotLocate: 'no matcher call after expect(...) on this line' };
|
|
50
|
+
i = skip(i + 1);
|
|
51
|
+
const start = i;
|
|
52
|
+
while (isWord(line[i]))
|
|
53
|
+
i++;
|
|
54
|
+
const id = line.slice(start, i);
|
|
55
|
+
if (id === '')
|
|
56
|
+
return { cannotLocate: 'unexpected character in expect(...) chain' };
|
|
57
|
+
const lookahead = skip(i);
|
|
58
|
+
if (line[lookahead] === '(') {
|
|
59
|
+
if (!STRING_MATCHERS.has(id)) {
|
|
60
|
+
return { cannotLocate: `matcher .${id}() does not carry a locatable string value` };
|
|
61
|
+
}
|
|
62
|
+
return readFirstStringArg(line, lookahead);
|
|
63
|
+
}
|
|
64
|
+
if (id === 'not')
|
|
65
|
+
return { cannotLocate: 'negated matcher (.not); behavior mutation semantics undefined in v1' };
|
|
66
|
+
if (id === 'resolves' || id === 'rejects') {
|
|
67
|
+
i = skip(i);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
return { cannotLocate: `unsupported chain segment ".${id}"` };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Read a plain string literal as the first argument at the matcher's `(`. */
|
|
74
|
+
function readFirstStringArg(line, open) {
|
|
75
|
+
let i = open + 1;
|
|
76
|
+
while (i < line.length && /\s/.test(line[i]))
|
|
77
|
+
i++;
|
|
78
|
+
const quote = line[i];
|
|
79
|
+
if (quote !== "'" && quote !== '"' && quote !== '`') {
|
|
80
|
+
return { cannotLocate: 'asserted value is not a plain string literal (number, variable, or expression)' };
|
|
81
|
+
}
|
|
82
|
+
let value = '';
|
|
83
|
+
for (i++; i < line.length; i++) {
|
|
84
|
+
const c = line[i];
|
|
85
|
+
if (c === '\\' && i + 1 < line.length) {
|
|
86
|
+
const next = line[i + 1];
|
|
87
|
+
value += ESCAPES[next] ?? next;
|
|
88
|
+
i++;
|
|
89
|
+
}
|
|
90
|
+
else if (quote === '`' && c === '$' && line[i + 1] === '{') {
|
|
91
|
+
return { cannotLocate: 'asserted value is an interpolated template literal' };
|
|
92
|
+
}
|
|
93
|
+
else if (c === quote) {
|
|
94
|
+
if (value === '')
|
|
95
|
+
return { cannotLocate: 'asserted value is an empty string' };
|
|
96
|
+
return { kind: 'string', value };
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
value += c;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return { cannotLocate: 'unterminated string literal in assertion' };
|
|
103
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
export { runSpec, sweepStaleMutationFiles, DEFAULT_TIMEOUT_MS, type SpecResult, type MutationResult, type Verdict, type RunOptions, } from './run.js';
|
|
1
|
+
export { runSpec, sweepStaleMutationFiles, DEFAULT_TIMEOUT_MS, type SpecResult, type MutationResult, type Verdict, type RunOptions, type BehaviorResult, type BehaviorVerdict, } from './run.js';
|
|
2
2
|
export { summarize, jsonReport, renderTable, type Summary, type GateOptions, type RenderOptions, } from './report.js';
|
|
3
|
-
export { planSpecMutations, MARKER, type SpecPlan, type Mutation, type PlanOptions, } from './mutate.js';
|
|
3
|
+
export { planSpecMutations, analyzeSpec, parseGateSkip, MARKER, type SpecPlan, type SpecAnalysis, type MarkedTest, type Mutation, type PlanOptions, type GateSkip, } from './mutate.js';
|
|
4
4
|
export { invertAssertLine, type InvertResult } from './invert.js';
|
|
5
|
+
export { planBehaviorMutations, type BehaviorMutation } from './behavior.js';
|
|
6
|
+
export { extractAssertValue, type AssertValue } from './extract.js';
|
package/dist/index.js
CHANGED
|
@@ -11,5 +11,7 @@
|
|
|
11
11
|
export { runSpec, sweepStaleMutationFiles, DEFAULT_TIMEOUT_MS, } from './run.js';
|
|
12
12
|
export { summarize, jsonReport, renderTable, } from './report.js';
|
|
13
13
|
// Advanced: build or inspect the mutation plan directly.
|
|
14
|
-
export { planSpecMutations, MARKER, } from './mutate.js';
|
|
14
|
+
export { planSpecMutations, analyzeSpec, parseGateSkip, MARKER, } from './mutate.js';
|
|
15
15
|
export { invertAssertLine } from './invert.js';
|
|
16
|
+
export { planBehaviorMutations } from './behavior.js';
|
|
17
|
+
export { extractAssertValue } from './extract.js';
|