playwright-mutation-gate 0.2.1 → 0.3.0

package/dist/mutate.js

@@ -3,6 +3,44 @@ import { basename, dirname, join } from 'node:path';
 import { invertAssertLine } from './invert.js';
 import { CODE, LINE_COMMENT, findCloseMasked, lex } from './lex.js';
 export const MARKER = '// @primary-assert';
+const GATE_SKIP = '@gate-skip';
+const YMD_RE = /^(\d{4})-(\d{2})-(\d{2})$/;
+/** True when `s` is a real calendar date in strict YYYY-MM-DD form. */
+export function isValidYmd(s) {
+    const m = YMD_RE.exec(s);
+    if (!m)
+        return false;
+    const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])];
+    if (mo < 1 || mo > 12 || d < 1 || d > 31)
+        return false;
+    const date = new Date(Date.UTC(y, mo - 1, d));
+    return date.getUTCFullYear() === y && date.getUTCMonth() === mo - 1 && date.getUTCDate() === d;
+}
+const attr = (text, name) => {
+    const m = new RegExp(`(?:^|\\s)${name}=("[^"]*"|\\S+)`).exec(text);
+    if (!m)
+        return null;
+    const raw = m[1];
+    return raw.startsWith('"') ? raw.slice(1, -1) : raw;
+};
+/**
+ * Parse a `@gate-skip owner=.. expires=YYYY-MM-DD reason=".."` line comment.
+ * Returns the acknowledgement, or null when it is malformed (missing owner or
+ * expires, or an unparseable date) - the same fail-closed policy as a malformed
+ * @primary-assert marker.
+ */
+export function parseGateSkip(commentText) {
+    const body = commentText.replace(/^\/\/\s*/, '');
+    if (!(body === GATE_SKIP || body.startsWith(GATE_SKIP + ' ')))
+        return null;
+    const rest = body.slice(GATE_SKIP.length);
+    const owner = attr(rest, 'owner');
+    const expires = attr(rest, 'expires');
+    if (owner === null || owner === '' || expires === null || !isValidYmd(expires))
+        return null;
+    const reason = attr(rest, 'reason');
+    return reason !== null && reason !== '' ? { owner, expires, reason } : { owner, expires };
+}
 /** Read the test title: the first argument when it is a literal without interpolation. */
 function readTitle(src, open) {
     let i = open + 1;
@@ -105,13 +143,15 @@ function findTestBlocks(src, kind, ids) {
 // test.skip/fixme never run and test.fail inverts pass/fail semantics, so a
 // mutation verdict on them would be a lie; they are also exempt from sentinel
 // accounting. test.only runs and is treated like a plain test.
-const UNGATEABLE_MODES = new Set(['skip', 'fixme', 'fail']);
+export const UNGATEABLE_MODES = new Set(['skip', 'fixme', 'fail']);
 /**
- * Parse a spec source and plan its mutations: one inverted copy per marked
- * test. Anything the gate cannot honestly mutate is reported with a reason,
- * never skipped silently.
+ * Parse a spec source into the shared view both planners build on: which
+ * test() blocks carry a valid `@primary-assert` marker (with the assertion
+ * line already resolved), plus the spec-level marker findings. The parser
+ * (lexer, test-id collection, block finding, marker association, assertion-line
+ * resolution) lives here so inversion and behavior planning cannot diverge.
  */
-export function planSpecMutations(source, opts = {}) {
+export function analyzeSpec(source, opts = {}) {
     const kind = lex(source);
     const ids = collectTestIds(source, kind, opts.testIds ?? []);
     const blocks = findTestBlocks(source, kind, ids);
@@ -182,7 +222,38 @@ export function planSpecMutations(source, opts = {}) {
         else
             strayMarkerLines.push(n + 1);
     }
-    const mutations = [];
+    // A @gate-skip decorates the test whose declaration it sits on (trailing) or
+    // immediately above (its own line, only blank/comment lines between). It must
+    // parse (owner + a real expires date) and land on a test() block; anything
+    // else is malformed and fails closed, like a malformed @primary-assert.
+    const malformedGateSkipLines = [];
+    for (let n = 0; n < lines.length; n++) {
+        if (!lines[n].includes(GATE_SKIP))
+            continue;
+        const lc = lineComment(n);
+        const parsed = lc ? parseGateSkip(lc.text) : null;
+        if (!lc || parsed === null) {
+            malformedGateSkipLines.push(n + 1);
+            continue;
+        }
+        let declLine;
+        if (lc.codeBefore) {
+            declLine = n + 1;
+        }
+        else {
+            let next = n + 1; // 0-based index of the line after the marker
+            while (next < lines.length && !lineHasCode(next))
+                next++;
+            declLine = next + 1;
+        }
+        const block = blocks.find((b) => lineOf(b.anchor) === declLine);
+        if (!block || block.gateSkip !== undefined) {
+            malformedGateSkipLines.push(n + 1);
+            continue;
+        }
+        block.gateSkip = parsed;
+    }
+    const markedTests = [];
     const unmarkedTests = [];
     const onlyTestLines = [];
     for (const block of blocks) {
@@ -196,48 +267,102 @@ export function planSpecMutations(source, opts = {}) {
             continue;
         }
         const marker = block.markers[0];
-        const base = { testTitle: block.title, testLine, markerLine: marker.line };
-        if (UNGATEABLE_MODES.has(block.mode)) {
-            const why = block.mode === 'fail'
+        // Resolve the assertion line for the first marker: a trailing marker sits
+        // on the assertion itself; a whole-line marker points at the next line that
+        // contains code (blank and comment-only lines are skipped, not mutated).
+        let assertLine = null;
+        let resolveError = null;
+        let target; // 0-based index of the assertion line
+        if (marker.sameLine) {
+            target = marker.line - 1;
+            assertLine = target + 1;
+        }
+        else {
+            target = marker.line; // 0-based index of the line after the marker
+            while (target < lines.length && !lineHasCode(target))
+                target++;
+            if (target >= lines.length)
+                resolveError = 'no code line follows the marker';
+            else
+                assertLine = target + 1;
+        }
+        markedTests.push({
+            title: block.title,
+            testLine,
+            mode: block.mode,
+            markerCount: block.markers.length,
+            markerLine: marker.line,
+            open: block.open,
+            close: block.close,
+            assertLine,
+            resolveError,
+            ...(block.gateSkip !== undefined ? { gateSkip: block.gateSkip } : {}),
+        });
+    }
+    return {
+        source,
+        lines,
+        kind,
+        markedTests,
+        unmarkedTests,
+        strayMarkerLines,
+        onlyTestLines,
+        malformedMarkerLines,
+        malformedGateSkipLines,
+    };
+}
+/**
+ * Parse a spec source and plan its mutations: one inverted copy per marked
+ * test. Anything the gate cannot honestly mutate is reported with a reason,
+ * never skipped silently.
+ */
+export function planSpecMutations(source, opts = {}) {
+    const analysis = analyzeSpec(source, opts);
+    const { lines, markedTests } = analysis;
+    const mutations = [];
+    for (const t of markedTests) {
+        const base = { testTitle: t.title, testLine: t.testLine, markerLine: t.markerLine };
+        // A @gate-skip rides along on every cannot-mutate verdict for this test, so
+        // the gate can treat an acknowledged skip as accepted rather than failing.
+        const gs = t.gateSkip !== undefined ? { gateSkip: t.gateSkip } : {};
+        if (UNGATEABLE_MODES.has(t.mode)) {
+            const why = t.mode === 'fail'
                 ? 'test.fail() expects failure, so a mutation verdict would be meaningless'
-                : `test.${block.mode}() never runs, so a mutation cannot be observed`;
-            mutations.push({ ...base, assertLine: null, cannotMutate: why });
+                : `test.${t.mode}() never runs, so a mutation cannot be observed`;
+            mutations.push({ ...base, ...gs, assertLine: null, cannotMutate: why });
             continue;
         }
-        if (block.markers.length > 1) {
+        if (t.markerCount > 1) {
             mutations.push({
                 ...base,
+                ...gs,
                 assertLine: null,
-                cannotMutate: `found ${block.markers.length} @primary-assert markers in one test, expected exactly one`,
+                cannotMutate: `found ${t.markerCount} @primary-assert markers in one test, expected exactly one`,
             });
             continue;
         }
-        // A trailing marker sits on the assertion itself; a whole-line marker
-        // points at the next line that contains code (blank and comment-only lines
-        // are skipped, not mutated).
-        let target; // 0-based index of the assertion line
-        if (marker.sameLine) {
-            target = marker.line - 1;
-        }
-        else {
-            target = marker.line; // 0-based index of the line after the marker
-            while (target < lines.length && !lineHasCode(target))
-                target++;
-            if (target >= lines.length) {
-                mutations.push({ ...base, assertLine: null, cannotMutate: 'no code line follows the marker' });
-                continue;
-            }
+        if (t.assertLine === null) {
+            mutations.push({ ...base, ...gs, assertLine: null, cannotMutate: t.resolveError ?? 'assertion line not found' });
+            continue;
         }
+        const target = t.assertLine - 1;
         const inverted = invertAssertLine(lines[target]);
         if ('cannotMutate' in inverted) {
-            mutations.push({ ...base, assertLine: target + 1, cannotMutate: inverted.cannotMutate });
+            mutations.push({ ...base, ...gs, assertLine: t.assertLine, cannotMutate: inverted.cannotMutate });
             continue;
         }
         const mutatedLines = lines.slice();
         mutatedLines[target] = inverted.ok;
-        mutations.push({ ...base, assertLine: target + 1, mutatedSource: mutatedLines.join('\n') });
+        mutations.push({ ...base, assertLine: t.assertLine, mutatedSource: mutatedLines.join('\n') });
     }
-    return { mutations, unmarkedTests, strayMarkerLines, malformedMarkerLines, onlyTestLines };
+    return {
+        mutations,
+        unmarkedTests: analysis.unmarkedTests,
+        strayMarkerLines: analysis.strayMarkerLines,
+        malformedMarkerLines: analysis.malformedMarkerLines,
+        malformedGateSkipLines: analysis.malformedGateSkipLines,
+        onlyTestLines: analysis.onlyTestLines,
+    };
 }
 /**
  * Name for a mutated copy: the marker segment goes right after the first

package/dist/report.d.ts

@@ -2,6 +2,8 @@ import type { SpecResult } from './run.js';
 export interface GateOptions {
     /** When true (default), tests without a @primary-assert marker fail the gate. */
     requireSentinel: boolean;
+    /** "Today" for @gate-skip expiry checks; defaults to the current date. */
+    now?: Date;
 }
 export interface Summary {
     killed: number;
@@ -11,11 +13,37 @@ export interface Summary {
     unmarkedTests: number;
     strayMarkers: number;
     malformedMarkers: number;
+    /**
+     * @gate-skip tallies. `acknowledgedSkips`: cannot-mutate verdicts covered by
+     * a valid, unexpired acknowledgement - accepted, does NOT fail the gate.
+     * `expiredSkips`: the acknowledgement's expires date has passed, so the skip
+     * fails closed again. `malformedGateSkip`: a @gate-skip that does not parse or
+     * is misplaced - fails the gate, like a malformed @primary-assert.
+     */
+    acknowledgedSkips: number;
+    expiredSkips: number;
+    malformedGateSkip: number;
+    /**
+     * Behavior mutation tallies, present only when the behavior pass ran.
+     * `behaviorKilled`: the test caught the corrupted value. `weak`: the test
+     * stayed green while the value it names was broken (a hollow assertion) -
+     * this fails the gate. `behaviorCannotLocate`: the value could not be found
+     * in a rewritable response - reported honestly and, unlike inversion's
+     * cannot-mutate, does NOT fail the gate (a value not living in an
+     * interceptable response is expected, not a defect). `behaviorErrors`: the
+     * behavior run itself failed - fails the gate.
+     */
+    behaviorRan: boolean;
+    behaviorKilled: number;
+    weak: number;
+    behaviorCannotLocate: number;
+    behaviorErrors: number;
     /**
      * Green means proven: every marked assertion was killed under inversion and
      * nothing was left unverified. Any survived, cannot-mutate or error verdict
      * fails the gate, as do marker mistakes (stray or malformed markers) and,
-     * under require-sentinel, tests with no marker at all.
+     * under require-sentinel, tests with no marker at all. When the behavior pass
+     * ran, any weak or behavior-error verdict also fails the gate.
      */
     gatePassed: boolean;
 }

package/dist/report.js

@@ -1,3 +1,10 @@
+/** Format a date as YYYY-MM-DD in local time, so expiry compares lexically. */
+function ymd(date) {
+    const y = date.getFullYear();
+    const m = String(date.getMonth() + 1).padStart(2, '0');
+    const d = String(date.getDate()).padStart(2, '0');
+    return `${y}-${m}-${d}`;
+}
 export function summarize(specs, opts) {
     const s = {
         killed: 0,
@@ -7,22 +14,53 @@ export function summarize(specs, opts) {
         unmarkedTests: 0,
         strayMarkers: 0,
         malformedMarkers: 0,
+        acknowledgedSkips: 0,
+        expiredSkips: 0,
+        malformedGateSkip: 0,
+        behaviorRan: false,
+        behaviorKilled: 0,
+        weak: 0,
+        behaviorCannotLocate: 0,
+        behaviorErrors: 0,
         gatePassed: true,
     };
+    const today = ymd(opts.now ?? new Date());
     for (const spec of specs) {
         for (const m of spec.results) {
             if (m.verdict === 'killed')
                 s.killed++;
             else if (m.verdict === 'survived')
                 s.survived++;
-            else if (m.verdict === 'cannot-mutate')
-                s.cannotMutate++;
+            else if (m.verdict === 'cannot-mutate') {
+                // A cannot-mutate covered by a valid @gate-skip is accepted until it
+                // expires; expired ones fail closed again. Uncovered ones fail as before.
+                if (m.gateSkip === undefined)
+                    s.cannotMutate++;
+                else if (today > m.gateSkip.expires)
+                    s.expiredSkips++;
+                else
+                    s.acknowledgedSkips++;
+            }
             else
                 s.errors++;
         }
         s.unmarkedTests += spec.unmarkedTests.length;
         s.strayMarkers += spec.strayMarkerLines.length;
         s.malformedMarkers += spec.malformedMarkerLines.length;
+        s.malformedGateSkip += spec.malformedGateSkipLines.length;
+        if (spec.behavior !== undefined) {
+            s.behaviorRan = true;
+            for (const b of spec.behavior) {
+                if (b.verdict === 'killed')
+                    s.behaviorKilled++;
+                else if (b.verdict === 'weak')
+                    s.weak++;
+                else if (b.verdict === 'cannot-locate')
+                    s.behaviorCannotLocate++;
+                else
+                    s.behaviorErrors++;
+            }
+        }
     }
     s.gatePassed =
         s.survived === 0 &&
@@ -30,9 +68,19 @@ export function summarize(specs, opts) {
             s.errors === 0 &&
             s.strayMarkers === 0 &&
             s.malformedMarkers === 0 &&
-            (!opts.requireSentinel || s.unmarkedTests === 0);
+            s.expiredSkips === 0 &&
+            s.malformedGateSkip === 0 &&
+            (!opts.requireSentinel || s.unmarkedTests === 0) &&
+            s.weak === 0 &&
+            s.behaviorErrors === 0;
     return s;
 }
+const BEHAVIOR_VERDICT_COLOR = {
+    killed: '\x1b[32m',
+    weak: '\x1b[31m',
+    'cannot-locate': '\x1b[33m',
+    error: '\x1b[33m',
+};
 export function jsonReport(specs, opts) {
     return JSON.stringify({ specs, summary: summarize(specs, opts) }, null, 2);
 }
@@ -57,13 +105,29 @@ export function renderTable(specs, opts) {
     const titleWidth = Math.min(TITLE_WIDTH, Math.max(4, ...specs.flatMap((spec) => [
         ...spec.results.map((m) => clip(m.testTitle).length),
         ...spec.unmarkedTests.map((t) => clip(t.title).length),
+        ...(spec.behavior ?? []).map((b) => clip(b.testTitle).length),
     ])));
     const row = (label, color, title, line) => '  ' + paint(color, label.padEnd(labelWidth)) + '  ' + clip(title).padEnd(titleWidth) + '  ' + line;
+    const today = ymd(opts.now ?? new Date());
     const lines = [];
     for (const spec of specs) {
         lines.push(spec.specPath);
         for (const m of spec.results) {
             const at = m.assertLine === null ? ':?' : `:${m.assertLine}`;
+            // An acknowledged (unexpired) @gate-skip reads as accepted, not failing;
+            // an expired one reads as a failure that fell back closed.
+            if (m.verdict === 'cannot-mutate' && m.gateSkip !== undefined) {
+                const expired = today > m.gateSkip.expires;
+                const label = expired ? 'expired-skip' : 'acknowledged';
+                const color = expired ? '\x1b[31m' : '\x1b[32m';
+                lines.push(row(label, color, m.testTitle, at));
+                const gs = m.gateSkip;
+                const detail = `owner=${gs.owner} expires=${gs.expires}` +
+                    (gs.reason !== undefined ? ` reason="${gs.reason}"` : '') +
+                    (expired ? ' (EXPIRED)' : '');
+                lines.push('  ' + ' '.repeat(labelWidth) + '  ' + detail);
+                continue;
+            }
             lines.push(row(m.verdict, VERDICT_COLOR[m.verdict], m.testTitle, at));
             if (m.reason !== undefined) {
                 lines.push('  ' + ' '.repeat(labelWidth) + '  ' + m.reason);
@@ -78,6 +142,19 @@ export function renderTable(specs, opts) {
         for (const line of spec.malformedMarkerLines) {
             lines.push(`  malformed @primary-assert marker at line ${line}`);
         }
+        for (const line of spec.malformedGateSkipLines) {
+            lines.push(`  malformed @gate-skip marker at line ${line} (needs owner= and a valid expires=YYYY-MM-DD on a test)`);
+        }
+        if (spec.behavior !== undefined && spec.behavior.length > 0) {
+            lines.push('  behavior:');
+            for (const b of spec.behavior) {
+                const at = b.assertLine === null ? ':?' : `:${b.assertLine}`;
+                lines.push(row(b.verdict, BEHAVIOR_VERDICT_COLOR[b.verdict], b.testTitle, at));
+                if (b.reason !== undefined) {
+                    lines.push('  ' + ' '.repeat(labelWidth) + '  ' + b.reason);
+                }
+            }
+        }
         lines.push('');
     }
     const s = summarize(specs, opts);
@@ -91,7 +168,20 @@ export function renderTable(specs, opts) {
         counts.push(`${s.unmarkedTests} unmarked ${s.unmarkedTests === 1 ? 'test' : 'tests'}` +
             (opts.requireSentinel ? '' : ' (not enforced)'));
     }
+    if (s.acknowledgedSkips > 0)
+        counts.push(`${s.acknowledgedSkips} acknowledged`);
+    if (s.expiredSkips > 0)
+        counts.push(`${s.expiredSkips} expired-skip`);
     lines.push(counts.join(', '));
+    if (s.behaviorRan) {
+        lines.push('behavior: ' +
+            [
+                `${s.behaviorKilled} killed`,
+                `${s.weak} weak`,
+                `${s.behaviorCannotLocate} cannot-locate`,
+                `${s.behaviorErrors} errors`,
+            ].join(', '));
+    }
     lines.push(s.gatePassed
         ? paint('\x1b[1;32m', 'Gate passed: every marked assertion can fail.')
         : paint('\x1b[1;31m', 'Gate FAILED: hollow, unverified or broken assertions above.'));

package/dist/run.d.ts

@@ -1,4 +1,5 @@
-import { type Mutation, type SpecPlan } from './mutate.js';
+import { type BehaviorMutation } from './behavior.js';
+import { type GateSkip, type Mutation, type SpecPlan } from './mutate.js';
 export type Verdict = 'killed' | 'survived' | 'cannot-mutate' | 'error';
 export interface MutationResult {
     testTitle: string;
@@ -8,6 +9,27 @@ export interface MutationResult {
     verdict: Verdict;
     /** Present for cannot-mutate and error verdicts. */
     reason?: string;
+    /** A valid @gate-skip acknowledgement on this test, for cannot-mutate verdicts. */
+    gateSkip?: GateSkip;
+}
+/**
+ * Behavior mutation verdicts. `killed`: the test caught the corrupted value
+ * (or the flow broke around it). `weak`: the test stayed green while the value
+ * it names was broken - a hollow assertion. `cannot-locate`: the value never
+ * appeared in a rewritable response, reported honestly and NOT a gate failure.
+ * `error`: the run itself failed (spawn, timeout, no report).
+ */
+export type BehaviorVerdict = 'killed' | 'weak' | 'cannot-locate' | 'error';
+export interface BehaviorResult {
+    testTitle: string;
+    testLine: number;
+    /** 1-based line of the assertion whose value was corrupted; null when none. */
+    assertLine: number | null;
+    verdict: BehaviorVerdict;
+    /** The corrupted value, when one was located. */
+    value?: string;
+    /** Explanation for weak / cannot-locate / error, and the flow-breakage caveat on killed. */
+    reason?: string;
 }
 export interface SpecResult {
     specPath: string;
@@ -15,6 +37,9 @@ export interface SpecResult {
     unmarkedTests: SpecPlan['unmarkedTests'];
     strayMarkerLines: number[];
     malformedMarkerLines: number[];
+    malformedGateSkipLines: number[];
+    /** Behavior mutation results, present only when the behavior pass ran. */
+    behavior?: BehaviorResult[];
 }
 export interface RunOptions {
     /** Directory Playwright is spawned from; defaults to process.cwd(). */
@@ -27,6 +52,8 @@ export interface RunOptions {
     playwrightCommand?: string[];
     /** Extra identifiers to treat as test builders (custom fixtures). */
     testIds?: string[];
+    /** When true, also run the behavior mutation pass (page.route value corruption). */
+    behavior?: boolean;
 }
 export declare const DEFAULT_TIMEOUT_MS: number;
 type RunVerdict = {
@@ -52,6 +79,32 @@ export declare function verdictFromReport(raw: string): RunVerdict;
 export declare function runMutation(specPath: string, index: number, mutation: Extract<Mutation, {
     mutatedSource: string;
 }>, opts?: RunOptions): Promise<RunVerdict>;
+type BehaviorRunVerdict = {
+    verdict: 'killed' | 'weak' | 'cannot-locate';
+    reason?: string;
+} | {
+    verdict: 'error';
+    reason: string;
+};
+/**
+ * Decide a behavior verdict from a Playwright JSON report and whether the
+ * injected handler recorded a hit (the named value was found and corrupted).
+ * Mirrors verdictFromReport's stats reading, then layers the behavior
+ * semantics: no hit means the value never reached a rewritable response
+ * (cannot-locate); a hit plus a red test is killed; a hit plus a green test is
+ * a hollow assertion (weak).
+ */
+export declare function behaviorVerdictFromReport(raw: string, hit: boolean): BehaviorRunVerdict;
+/**
+ * Run one behavior mutation: write the injected copy (route interception on top
+ * of the intact assertion), run Playwright against exactly that test, and read
+ * the verdict from the JSON report plus the hit-file the handler writes when it
+ * corrupts a response. The copy, the report dir and the hit-file are always
+ * removed.
+ */
+export declare function runBehaviorMutation(specPath: string, index: number, mutation: Extract<BehaviorMutation, {
+    mutatedSource: string;
+}>, opts?: RunOptions): Promise<BehaviorRunVerdict>;
 /**
  * Delete leftover mutation copies in a directory: files whose pid token
  * points at a process that no longer exists. Copies of live concurrent gate