playwright-mutation-gate 0.2.2 → 0.3.0

package/dist/run.js

@@ -1,7 +1,8 @@
 import { spawn } from 'node:child_process';
-import { mkdtempSync, readdirSync, readFileSync, rmSync, unlinkSync } from 'node:fs';
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, unlinkSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { basename, dirname, join, relative } from 'node:path';
+import { planBehaviorMutations } from './behavior.js';
 import { planSpecMutations, writeMutationFile } from './mutate.js';
 export const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
 const DEFAULT_COMMAND = ['npx', 'playwright', 'test'];
@@ -142,6 +143,107 @@ export async function runMutation(specPath, index, mutation, opts = {}) {
         rmSync(reportDir, { recursive: true, force: true });
     }
 }
+/**
+ * Decide a behavior verdict from a Playwright JSON report and whether the
+ * injected handler recorded a hit (the named value was found and corrupted).
+ * Mirrors verdictFromReport's stats reading, then layers the behavior
+ * semantics: no hit means the value never reached a rewritable response
+ * (cannot-locate); a hit plus a red test is killed; a hit plus a green test is
+ * a hollow assertion (weak).
+ */
+export function behaviorVerdictFromReport(raw, hit) {
+    let report;
+    try {
+        report = JSON.parse(raw);
+    }
+    catch {
+        return { verdict: 'error', reason: 'Playwright JSON report is not valid JSON' };
+    }
+    const stats = report.stats;
+    const expected = typeof stats?.expected === 'number' ? stats.expected : null;
+    const unexpected = typeof stats?.unexpected === 'number' ? stats.unexpected : null;
+    const flaky = typeof stats?.flaky === 'number' ? stats.flaky : null;
+    const skipped = typeof stats?.skipped === 'number' ? stats.skipped : 0;
+    if (expected === null || unexpected === null || flaky === null) {
+        return { verdict: 'error', reason: 'Playwright JSON report has no stats block' };
+    }
+    if (expected + unexpected + flaky === 0) {
+        if (skipped > 0) {
+            return {
+                verdict: 'cannot-locate',
+                reason: 'the test was skipped at runtime, so its behavior could not be observed',
+            };
+        }
+        const message = report.errors?.map((e) => e.message).find((m) => typeof m === 'string');
+        const detail = message ? `: ${message.split('\n', 1)[0]}` : '';
+        return { verdict: 'error', reason: `no tests ran under the mutation filter${detail}` };
+    }
+    if (!hit) {
+        return {
+            verdict: 'cannot-locate',
+            reason: 'the named value never appeared in an intercepted response, so nothing was corrupted',
+        };
+    }
+    if (unexpected + flaky > 0) {
+        return {
+            verdict: 'killed',
+            reason: 'the test failed once the named value was corrupted (a red here could also be flow breakage, not the assertion)',
+        };
+    }
+    return { verdict: 'weak', reason: 'the test stayed green while the value it names was broken' };
+}
+/**
+ * Run one behavior mutation: write the injected copy (route interception on top
+ * of the intact assertion), run Playwright against exactly that test, and read
+ * the verdict from the JSON report plus the hit-file the handler writes when it
+ * corrupts a response. The copy, the report dir and the hit-file are always
+ * removed.
+ */
+export async function runBehaviorMutation(specPath, index, mutation, opts = {}) {
+    const cwd = opts.cwd ?? process.cwd();
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const command = opts.playwrightCommand ?? DEFAULT_COMMAND;
+    const copyPath = writeMutationFile(specPath, index, mutation.mutatedSource);
+    const reportDir = mkdtempSync(join(tmpdir(), 'pmg-'));
+    const reportPath = join(reportDir, 'report.json');
+    const hitPath = join(reportDir, 'hit');
+    try {
+        const target = `${relative(cwd, copyPath) || copyPath}:${mutation.testLine}`;
+        const outcome = await runChild([...command, target, '--reporter=json', ...(opts.playwrightArgs ?? [])], cwd, { ...process.env, PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath, PMG_HIT_FILE: hitPath }, timeoutMs);
+        if (outcome.spawnError) {
+            return { verdict: 'error', reason: `could not launch ${command.join(' ')}: ${outcome.spawnError}` };
+        }
+        if (outcome.timedOut) {
+            return {
+                verdict: 'error',
+                reason: `Playwright run timed out after ${Math.round(timeoutMs / 1000)}s and was killed`,
+            };
+        }
+        let raw;
+        try {
+            raw = readFileSync(reportPath, 'utf8');
+        }
+        catch {
+            const tail = outcome.output.trim();
+            return {
+                verdict: 'error',
+                reason: `Playwright exited with code ${outcome.code ?? 'unknown'} without producing a report` +
+                    (tail === '' ? '' : `; output tail: ${tail.slice(-500)}`),
+            };
+        }
+        const hit = existsSync(hitPath) && statSync(hitPath).size > 0;
+        return behaviorVerdictFromReport(raw, hit);
+    }
+    finally {
+        try {
+            unlinkSync(copyPath);
+        }
+        catch {
+            // Already gone.
+        }
+        rmSync(reportDir, { recursive: true, force: true });
+    }
+}
 function pidIsAlive(pid) {
     try {
         process.kill(pid, 0);
@@ -217,6 +319,7 @@ export async function runSpec(specPath, opts = {}) {
                     assertLine: m.assertLine,
                     verdict: 'cannot-mutate',
                     reason: m.cannotMutate,
+                    ...(m.gateSkip !== undefined ? { gateSkip: m.gateSkip } : {}),
                 });
                 continue;
             }
@@ -230,11 +333,55 @@ export async function runSpec(specPath, opts = {}) {
             });
         }
     }
-    return {
+    const result = {
         specPath,
         results,
         unmarkedTests: plan.unmarkedTests,
         strayMarkerLines: plan.strayMarkerLines,
         malformedMarkerLines: plan.malformedMarkerLines,
+        malformedGateSkipLines: plan.malformedGateSkipLines,
     };
+    if (opts.behavior) {
+        result.behavior = await runBehaviorPass(specPath, source, plan.onlyTestLines, opts);
+    }
+    return result;
+}
+/**
+ * Run the behavior mutation pass for one spec: plan the route-interception
+ * mutations and run each one that can be located. A spec containing test.only
+ * is refused (same reason as inversion: .only makes every verdict fiction).
+ */
+async function runBehaviorPass(specPath, source, onlyTestLines, opts) {
+    const plan = planBehaviorMutations(source, { testIds: opts.testIds });
+    const results = [];
+    if (onlyTestLines.length > 0) {
+        const reason = `test.only on line ${onlyTestLines.join(', ')} of ${basename(specPath)} ` +
+            'filters out every other test, so no behavior verdict would be honest; remove .only first';
+        for (const m of plan) {
+            results.push({ testTitle: m.testTitle, testLine: m.testLine, assertLine: m.assertLine, verdict: 'error', reason });
+        }
+        return results;
+    }
+    for (const [index, m] of plan.entries()) {
+        if ('cannotLocate' in m) {
+            results.push({
+                testTitle: m.testTitle,
+                testLine: m.testLine,
+                assertLine: m.assertLine,
+                verdict: 'cannot-locate',
+                reason: m.cannotLocate,
+            });
+            continue;
+        }
+        const run = await runBehaviorMutation(specPath, index, m, opts);
+        results.push({
+            testTitle: m.testTitle,
+            testLine: m.testLine,
+            assertLine: m.assertLine,
+            verdict: run.verdict,
+            value: m.value,
+            ...('reason' in run && run.reason !== undefined ? { reason: run.reason } : {}),
+        });
+    }
+    return results;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "playwright-mutation-gate",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "description": "Mutation testing for Playwright assertions: invert each test's primary assertion and fail the run if the test stays green.",
   "keywords": [
     "playwright",