tewip-ai 0.16.3 → 0.17.0

package/README.md

@@ -282,6 +282,18 @@ Measured on **16 real commits** from two public projects, where the developers c
 its test: given only the source files, Tewip **named the right test in 16 of 16, ranked it first in
 12**.
 
+It also shows **the checks a change renegotiates** — every test assertion the diff rewrites,
+withdraws or renames, with the old text beside the new one. A test written earlier is behaviour
+someone agreed on, and this is the one thing in a review that the change's own description cannot
+account for: everything else reads the change's account of itself. It never passes judgement — tests
+change for good reasons — it asks which of them the author intended.
+
+This came out of a measurement rather than an idea. Of 18 real pull requests later found to have
+broken something, the one Tewip's test plan missed was one whose *intention* was wrong: it changed
+a freshness rule, changed the tests to match, and was reverted the next day. Run over those same 18
+pull requests, this check fires on 2 — **and both were reverted**, including the one the plan
+missed. No model is called, so it costs nothing.
+
 ## Which flaky tests to quarantine, and what they cost (new)
 
 ```bash

package/dist/engine/review/expectations.js

@@ -0,0 +1,118 @@
+// What a change does to the checks that were already agreed.
+//
+// Measuring test plans against 18 real pull requests that later turned out to have broken something
+// found one blind spot, and it was the same one everywhere in Tewip: the pull request's own
+// description is treated as the specification. In `budget-mvp#16` the author changed how an
+// indicator's freshness was judged, changed the existing test to match, and it was reverted a day
+// later because the new rule was wrong for a value that legitimately stops changing. Every layer
+// agreed with the change, because every layer was reading the change's own account of itself.
+//
+// A rewritten expectation is the one place where that assumption is checkable. A test someone wrote
+// earlier is behaviour the team agreed on; editing its assertion renegotiates that agreement, and
+// deleting it withdraws it. Neither is wrong - tests change for good reasons all the time - but
+// both deserve to be said out loud in a review, with the old text and the new one side by side, so
+// a person decides rather than a description deciding for them.
+//
+// This reads the diff only. No model, no network, nothing to configure.
+import { execFileSync } from 'node:child_process';
+const IS_TEST_FILE = /(\.spec\.|\.test\.|_test\.|test_|\/tests?\/|\/e2e\/|\/cypress\/|Test\.(java|kt|cs)$|_spec\.rb$)/i;
+/** A line that checks something: the sentence of the agreement. */
+const ASSERTION = /\b(?:expect|assert|should|verify|assertEquals|assertThat|assertTrue|assertFalse|toBe|toEqual|toMatch|toHaveBeenCalled|require\.\w+|is\.\w+)\b/;
+/** A line that names a test: the title people read in a report. */
+const TEST_TITLE = /\b(?:it|test|describe|context|scenario|def test_\w+|func Test\w+|@Test)\b\s*[( ]/;
+const title = (line) => /['"`]([^'"`]{4,120})['"`]/.exec(line)?.[1] ?? /\b(?:def|func)\s+(\w+)/.exec(line)?.[1] ?? '';
+/** The diff of test files only, which is all this looks at. */
+function testDiff(root, range) {
+    try {
+        return execFileSync('git', ['diff', '--unified=0', range, '--'], { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Expectations this change rewrites, withdraws or renames. A brand-new test is not reported: adding
+ * checks is not renegotiating anything, and a review that cried out about every added test would be
+ * ignored within a week.
+ */
+export function changedExpectations(root, range, diff = testDiff(root, range)) {
+    const out = [];
+    let file = '';
+    let isNewFile = false;
+    let removed = [];
+    let added = [];
+    const flush = () => {
+        if (!isNewFile && file) {
+            const gone = removed.filter((l) => ASSERTION.test(l));
+            const fresh = added.filter((l) => ASSERTION.test(l));
+            for (const [i, before] of gone.entries()) {
+                const after = fresh[i] ?? '';
+                out.push({
+                    file,
+                    kind: after ? 'rewritten' : 'withdrawn',
+                    test: title(removed.find((l) => TEST_TITLE.test(l)) ?? '') || title(added.find((l) => TEST_TITLE.test(l)) ?? ''),
+                    before: before.trim().slice(0, 160),
+                    after: after.trim().slice(0, 160),
+                });
+            }
+            const renamedFrom = removed.find((l) => TEST_TITLE.test(l) && title(l));
+            const renamedTo = added.find((l) => TEST_TITLE.test(l) && title(l));
+            if (renamedFrom && renamedTo && title(renamedFrom) !== title(renamedTo) && gone.length === 0) {
+                out.push({ file, kind: 'renamed', test: title(renamedFrom), before: title(renamedFrom), after: title(renamedTo) });
+            }
+        }
+        removed = [];
+        added = [];
+    };
+    for (const line of diff.split('\n')) {
+        if (line.startsWith('diff --git')) {
+            flush();
+            file = /b\/(\S+)/.exec(line)?.[1] ?? '';
+            isNewFile = false;
+            continue;
+        }
+        // A file added by this change had no agreed behaviour to renegotiate.
+        if (line.startsWith('new file mode'))
+            isNewFile = true;
+        if (line.startsWith('@@'))
+            flush();
+        if (!file || !IS_TEST_FILE.test(file))
+            continue;
+        if (line.startsWith('-') && !line.startsWith('---'))
+            removed.push(line.slice(1));
+        else if (line.startsWith('+') && !line.startsWith('+++'))
+            added.push(line.slice(1));
+    }
+    flush();
+    return out;
+}
+/** The sentences a reviewer reads. Never a verdict: this asks, it does not decide. */
+export function renderExpectations(changes) {
+    if (changes.length === 0)
+        return [];
+    const rewritten = changes.filter((c) => c.kind === 'rewritten');
+    const withdrawn = changes.filter((c) => c.kind === 'withdrawn');
+    const renamed = changes.filter((c) => c.kind === 'renamed');
+    const lines = ['## Checks this change renegotiates', ''];
+    lines.push(`This change edits ${changes.length} check(s) that were already agreed: ${[
+        rewritten.length ? `${rewritten.length} rewritten` : '',
+        withdrawn.length ? `${withdrawn.length} withdrawn` : '',
+        renamed.length ? `${renamed.length} renamed` : '',
+    ]
+        .filter(Boolean)
+        .join(', ')}. Tests change for good reasons, so this is not a fault - but a test written earlier is behaviour someone agreed on, and the change's own description cannot tell you whether replacing it was intended.`, '');
+    for (const change of [...withdrawn, ...rewritten, ...renamed].slice(0, 10)) {
+        const where = `${change.file.split('/').at(-1)}${change.test ? ` — ${change.test}` : ''}`;
+        if (change.kind === 'withdrawn') {
+            lines.push(`- **${where}** no longer checks: \`${change.before}\``);
+        }
+        else if (change.kind === 'renamed') {
+            lines.push(`- **${where}** was renamed to "${change.after}", so a report of it will no longer match the old name`);
+        }
+        else {
+            lines.push(`- **${where}**`, `  - was: \`${change.before}\``, `  - now: \`${change.after}\``);
+        }
+    }
+    lines.push('', 'Ask the author which of these the change intended, and which followed from it.');
+    return lines;
+}

package/dist/engine/review/review-command.js

@@ -6,6 +6,7 @@ import { chooseFramework, detectFrameworks } from '../quality/frameworks.js';
 import { readTriageRecords, datasetFiles } from '../../reporters/store.js';
 import { coverageFor, uncovered } from './coverage.js';
 import { renderReview, reviewChange } from './review.js';
+import { changedExpectations } from './expectations.js';
 /** Every failure Tewip has diagnosed in this project, from the history it keeps. */
 function history(root, only) {
     const dir = join(root, '.tewip', 'history');
@@ -25,7 +26,7 @@ export async function runReview(cmd) {
     const files = change.files.map((f) => f.split('\\').join('/'));
     const coverage = coverageFor(cmd.root, files);
     const past = history(cmd.root, cmd.name);
-    const review = reviewChange(cmd.diff, files, coverage, past, uncovered(files, coverage));
+    const review = reviewChange(cmd.diff, files, coverage, past, uncovered(files, coverage), changedExpectations(cmd.root, cmd.diff));
     // The command that runs just those tests - but only when the project's runner really owns them.
     // Printing a Playwright command for a file Playwright never runs is worse than printing nothing.
     const framework = chooseFramework(detectFrameworks(cmd.root), undefined, false) ?? detectFrameworks(cmd.root)[0] ?? null;

package/dist/engine/review/review.js

@@ -5,6 +5,7 @@
 // August, and that nothing at all covers the file you edited. That comes from Tewip's own history,
 // which is why this costs nothing and why nothing else can produce it.
 import { componentOf } from '../../outlets/items.js';
+import { renderExpectations } from './expectations.js';
 const fileOf = (record) => record.file ?? '';
 const countBy = (xs, key) => {
     const out = new Map();
@@ -13,7 +14,7 @@ const countBy = (xs, key) => {
     return out;
 };
 /** The review, assembled from the change, what covers it, and what has gone wrong here before. */
-export function reviewChange(range, changed, coverage, history, notCovered) {
+export function reviewChange(range, changed, coverage, history, notCovered, expectations = []) {
     const byFile = new Map();
     for (const record of history) {
         const file = fileOf(record).split('\\').join('/');
@@ -75,7 +76,14 @@ export function reviewChange(range, changed, coverage, history, notCovered) {
     if (tests.length === 0 && changed.length > 0) {
         notes.push('No test in this project reaches any of the changed files. Tewip can write one: `tewip plan "<what the change should do>"` then `tewip write`.');
     }
-    return { range, files: changed, tests, uncovered: notCovered, areas: [...areas.values()].sort((a, b) => b.realBugs - a.realBugs || b.failures - a.failures), runFirst, notes };
+    // A change that rewrites checks people already agreed on is worth saying before anything else:
+    // every other layer reads the change's own account of itself, and this is the one place where
+    // that account can be held against what the project had already decided.
+    if (expectations.length > 0) {
+        const withdrawn = expectations.filter((e) => e.kind === 'withdrawn').length;
+        notes.unshift(`This change edits ${expectations.length} check(s) that were already agreed${withdrawn > 0 ? `, ${withdrawn} of them withdrawn outright` : ''}. Read those first: the change's description cannot tell you whether replacing them was intended.`);
+    }
+    return { range, files: changed, tests, uncovered: notCovered, expectations, areas: [...areas.values()].sort((a, b) => b.realBugs - a.realBugs || b.failures - a.failures), runFirst, notes };
 }
 const when = (at) => {
     if (!at)
@@ -94,6 +102,9 @@ export function renderReview(review, runCommand) {
             lines.push(`- ${note}`);
         lines.push('');
     }
+    const renegotiated = renderExpectations(review.expectations);
+    if (renegotiated.length > 0)
+        lines.push(...renegotiated, '');
     lines.push('## Tests that cover this change', '');
     if (review.tests.length === 0)
         lines.push('None found.', '');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.16.3",
+  "version": "0.17.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",