tewip-ai 0.16.3 → 0.17.1

package/README.md

@@ -282,6 +282,28 @@ Measured on **16 real commits** from two public projects, where the developers c
 its test: given only the source files, Tewip **named the right test in 16 of 16, ranked it first in
 12**.
 
+It also shows **the checks a change renegotiates** — every test assertion the diff rewrites,
+withdraws or renames, with the old text beside the new one. A test written earlier is behaviour
+someone agreed on, and this is the one thing in a review that the change's own description cannot
+account for: everything else reads the change's account of itself. It never passes judgement — tests
+change for good reasons — it asks which of them the author intended.
+
+Alongside it, **the decisions a change alters outside its tests**: a guard that refused something,
+a value other code falls back on, a limit that decides behaviour under load, a version or runner
+pinned on purpose. Same rule - only things that already existed, never additions - and the same
+question rather than a verdict.
+
+Both came out of a measurement rather than an idea. Of 18 real pull requests later found to have
+broken something, the one Tewip's test plan missed was one whose *intention* was wrong: it changed
+a freshness rule, changed the tests to match, and was reverted the next day.
+
+**What these checks are not is a predictor**, and the measurement says so plainly. On 14 pull
+requests known to have caused a regression they speak up on 4; on 30 ordinary merged pull requests
+from the same repositories, on 9 - 29% against 30%. A change that alters an agreement is not more
+likely to be broken. What they do is put those alterations in front of a person: on the ordinary
+set they surfaced 23 of them, among which a `raise SystemExit` guard that was deleted, a semaphore
+dropped from 4 to 1, and a CI runner image moved. No model is called, so this costs nothing.
+
 ## Which flaky tests to quarantine, and what they cost (new)
 
 ```bash

package/dist/engine/review/agreements.js

@@ -0,0 +1,160 @@
+// Agreements a change renegotiates outside the tests.
+//
+// The test version of this reads assertions, because a test is an agreement written down. It is not
+// the only one. A default someone chose, a guard that refuses bad input, a version pinned on
+// purpose, a permission deliberately kept narrow - each is a decision the project already made, and
+// each can be undone in one line of a diff that reads like housekeeping. The pull request that
+// prompted all of this shipped a wrong rule with its tests updated to match; the same shape appears
+// when a timeout doubles or a `required` becomes optional and nobody says why.
+//
+// What this refuses to be is a linter. It reports only changes to things that were already decided
+// - never additions, never new files - and it asks rather than judges. The test that keeps it
+// honest is the noise measurement: run over eighteen real pull requests, it has to stay quiet on
+// the ordinary ones or a team will learn to scroll past it.
+import { execFileSync } from 'node:child_process';
+const NOT_SOURCE = /(\.spec\.|\.test\.|_test\.|test_|\/tests?\/|\/e2e\/|\/cypress\/|\.md$|\.snap$|lock(file)?$|package-lock\.json$|yarn\.lock$|\.sum$)/i;
+/** An exported or module-level constant, and a parameter's default: values other code relies on. */
+const DEFAULT_VALUE = /^\s*(?:export\s+)?(?:const|let|var|final|static|public|private)?\s*([A-Z][A-Z0-9_]{2,}|[\w$]+)\s*(?::[^=]+)?=\s*(.+?);?\s*$/;
+const PARAM_DEFAULT = /\b([\w$]+)\s*(?::[^=,)]+)?=\s*([^,)]+)[,)]/;
+/** A refusal: the code declining to accept something. */
+const GUARD = /\b(?:throw|raise|panic|abort|reject|assert|invariant|require)\b|\bif\s*\(![\w$.]+\)/;
+/** Who is allowed to do what. */
+const PERMISSION = /\b(?:permissions?|scope|role|allow[_-]?origin|cors|public|private|protected|readonly|auth|admin|policy|rls|chmod|0o?7[0-7][0-7])\b/i;
+/**
+ * A version deliberately pinned - and, in the same family, the thing a job is deliberately run on.
+ * `runs-on: macos-14` carries no dotted version but is exactly the same kind of decision, and one
+ * real pull request in the measurement set bumped it to `macos-26` by accident and was reverted
+ * for it, which is how this case was found.
+ */
+const PIN = /["']?[~^>=<]*\d+\.\d+(\.\d+)?["']?/;
+const RUNS_ON = /^\s*(?:-\s*)?(?:runs-on|image|container|uses|FROM|baseImage|node-version|python-version|go-version)\b\s*[:=]?\s*\S/i;
+const PIN_FILE = /(package\.json|requirements[\w.-]*\.txt|pyproject\.toml|go\.mod|Gemfile|Cargo\.toml|build\.gradle|pom\.xml|Dockerfile|\.ya?ml)$/i;
+/** Numbers that decide behaviour under load rather than logic. */
+const LIMIT = /\b(?:timeout|retries|retry|max|min|limit|ttl|interval|delay|backoff|threshold|concurrency|workers|pool|buffer|size)\b/i;
+/**
+ * Markup and styling, where the words this looks for mean something else entirely. Measured against
+ * 30 ordinary pull requests, `className="min-w-0 flex-1"` becoming `min-w-[9rem]` was reported as a
+ * limit that decides behaviour under load - which is the kind of noise that teaches people to
+ * scroll past a review.
+ */
+const MARKUP = /\bclassName\s*=|\bclass\s*=|\bstyle\s*=|<\/?[a-zA-Z][\w-]*[\s/>]|\.(?:css|scss|sass|less|html|svg)$/;
+/**
+ * The words in a line, with identifiers pulled apart. `MAX_RETRIES` and `maxRetries` are the same
+ * decision as `max retries`, but an underscore is a word character, so a plain \bretries\b matches
+ * neither - which silently classified a retry count as an ordinary default.
+ */
+const words = (line) => line.replace(/[_-]+/g, ' ').replace(/([a-z\d])([A-Z])/g, '$1 $2');
+const clean = (line) => line.trim().slice(0, 150);
+/** Pairs a removed line with the added line that replaced it, by position within the hunk. */
+function pairs(removed, added) {
+    return removed.map((line, i) => [line, added[i] ?? '']);
+}
+/** Two lines that are the same statement with a different value: that is a decision being changed. */
+function sameStatement(before, after) {
+    const shape = (s) => s.replace(/["'`][^"'`]*["'`]/g, 'S').replace(/\b\d+(\.\d+)*\b/g, 'N').replace(/\s+/g, ' ').trim();
+    return Boolean(after) && shape(before) === shape(after) && before.trim() !== after.trim();
+}
+function classify(file, before, after) {
+    const both = `${before} ${after}`;
+    if (MARKUP.test(both) || MARKUP.test(file))
+        return null;
+    if (!after) {
+        // Something was removed outright. Only a refusal is worth reporting: deleted code is the
+        // ordinary business of a diff, but a deleted guard changes what the system accepts.
+        return GUARD.test(before)
+            ? { file, kind: 'guard', before: clean(before), after: '', why: 'a check that refused something was removed, so input the project used to reject is now accepted' }
+            : null;
+    }
+    if (!sameStatement(before, after))
+        return null;
+    if (GUARD.test(before) || GUARD.test(after)) {
+        return { file, kind: 'guard', before: clean(before), after: clean(after), why: 'the condition on which the code refuses something changed' };
+    }
+    if (PERMISSION.test(words(both))) {
+        return { file, kind: 'permission', before: clean(before), after: clean(after), why: 'who or what is allowed here changed' };
+    }
+    if (PIN_FILE.test(file) && (PIN.test(before) || RUNS_ON.test(before))) {
+        return {
+            file,
+            kind: 'pin',
+            before: clean(before),
+            after: clean(after),
+            why: RUNS_ON.test(before) ? 'what this runs on changed' : 'a version that was pinned deliberately moved',
+        };
+    }
+    if (LIMIT.test(words(both))) {
+        return { file, kind: 'limit', before: clean(before), after: clean(after), why: 'a limit that decides behaviour under load changed' };
+    }
+    if (DEFAULT_VALUE.test(before) || PARAM_DEFAULT.test(before)) {
+        return { file, kind: 'default', before: clean(before), after: clean(after), why: 'a value other code falls back on changed' };
+    }
+    return null;
+}
+function sourceDiff(root, range) {
+    try {
+        return execFileSync('git', ['diff', '--unified=0', range, '--'], { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return '';
+    }
+}
+/** Decisions this change alters, outside the tests. Additions and new files are never reported. */
+export function changedAgreements(root, range, diff = sourceDiff(root, range)) {
+    const out = [];
+    let file = '';
+    let isNew = false;
+    let removed = [];
+    let added = [];
+    const flush = () => {
+        if (file && !isNew && !NOT_SOURCE.test(file)) {
+            for (const [before, after] of pairs(removed, added)) {
+                const found = classify(file, before, after);
+                if (found)
+                    out.push(found);
+            }
+        }
+        removed = [];
+        added = [];
+    };
+    for (const line of diff.split('\n')) {
+        if (line.startsWith('diff --git')) {
+            flush();
+            file = /b\/(\S+)/.exec(line)?.[1] ?? '';
+            isNew = false;
+            continue;
+        }
+        if (line.startsWith('new file mode'))
+            isNew = true;
+        if (line.startsWith('@@'))
+            flush();
+        if (line.startsWith('-') && !line.startsWith('---'))
+            removed.push(line.slice(1));
+        else if (line.startsWith('+') && !line.startsWith('+++'))
+            added.push(line.slice(1));
+    }
+    flush();
+    return out;
+}
+const HEADING = {
+    guard: 'checks that refused something',
+    permission: 'who is allowed what',
+    pin: 'versions pinned on purpose',
+    limit: 'limits that decide behaviour under load',
+    default: 'values other code falls back on',
+};
+/** Sentences for a reviewer. As with the tests, this asks; it never decides. */
+export function renderAgreements(agreements) {
+    if (agreements.length === 0)
+        return [];
+    const lines = ['## Decisions this change alters', ''];
+    const kinds = [...new Set(agreements.map((a) => a.kind))];
+    lines.push(`Outside the tests, this change alters ${agreements.length} thing(s) the project had already decided: ${kinds.map((k) => HEADING[k]).join(', ')}. Each may be exactly what was intended - the point is that a diff makes them look like housekeeping.`, '');
+    for (const agreement of agreements.slice(0, 10)) {
+        lines.push(`- **${agreement.file.split('/').at(-1)}** — ${agreement.why}`);
+        lines.push(`  - was: \`${agreement.before}\``);
+        if (agreement.after)
+            lines.push(`  - now: \`${agreement.after}\``);
+    }
+    lines.push('', 'Ask the author which of these the change needed.');
+    return lines;
+}

package/dist/engine/review/expectations.js

@@ -0,0 +1,118 @@
+// What a change does to the checks that were already agreed.
+//
+// Measuring test plans against 18 real pull requests that later turned out to have broken something
+// found one blind spot, and it was the same one everywhere in Tewip: the pull request's own
+// description is treated as the specification. In `budget-mvp#16` the author changed how an
+// indicator's freshness was judged, changed the existing test to match, and it was reverted a day
+// later because the new rule was wrong for a value that legitimately stops changing. Every layer
+// agreed with the change, because every layer was reading the change's own account of itself.
+//
+// A rewritten expectation is the one place where that assumption is checkable. A test someone wrote
+// earlier is behaviour the team agreed on; editing its assertion renegotiates that agreement, and
+// deleting it withdraws it. Neither is wrong - tests change for good reasons all the time - but
+// both deserve to be said out loud in a review, with the old text and the new one side by side, so
+// a person decides rather than a description deciding for them.
+//
+// This reads the diff only. No model, no network, nothing to configure.
+import { execFileSync } from 'node:child_process';
+const IS_TEST_FILE = /(\.spec\.|\.test\.|_test\.|test_|\/tests?\/|\/e2e\/|\/cypress\/|Test\.(java|kt|cs)$|_spec\.rb$)/i;
+/** A line that checks something: the sentence of the agreement. */
+const ASSERTION = /\b(?:expect|assert|should|verify|assertEquals|assertThat|assertTrue|assertFalse|toBe|toEqual|toMatch|toHaveBeenCalled|require\.\w+|is\.\w+)\b/;
+/** A line that names a test: the title people read in a report. */
+const TEST_TITLE = /\b(?:it|test|describe|context|scenario|def test_\w+|func Test\w+|@Test)\b\s*[( ]/;
+const title = (line) => /['"`]([^'"`]{4,120})['"`]/.exec(line)?.[1] ?? /\b(?:def|func)\s+(\w+)/.exec(line)?.[1] ?? '';
+/** The diff of test files only, which is all this looks at. */
+function testDiff(root, range) {
+    try {
+        return execFileSync('git', ['diff', '--unified=0', range, '--'], { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Expectations this change rewrites, withdraws or renames. A brand-new test is not reported: adding
+ * checks is not renegotiating anything, and a review that cried out about every added test would be
+ * ignored within a week.
+ */
+export function changedExpectations(root, range, diff = testDiff(root, range)) {
+    const out = [];
+    let file = '';
+    let isNewFile = false;
+    let removed = [];
+    let added = [];
+    const flush = () => {
+        if (!isNewFile && file) {
+            const gone = removed.filter((l) => ASSERTION.test(l));
+            const fresh = added.filter((l) => ASSERTION.test(l));
+            for (const [i, before] of gone.entries()) {
+                const after = fresh[i] ?? '';
+                out.push({
+                    file,
+                    kind: after ? 'rewritten' : 'withdrawn',
+                    test: title(removed.find((l) => TEST_TITLE.test(l)) ?? '') || title(added.find((l) => TEST_TITLE.test(l)) ?? ''),
+                    before: before.trim().slice(0, 160),
+                    after: after.trim().slice(0, 160),
+                });
+            }
+            const renamedFrom = removed.find((l) => TEST_TITLE.test(l) && title(l));
+            const renamedTo = added.find((l) => TEST_TITLE.test(l) && title(l));
+            if (renamedFrom && renamedTo && title(renamedFrom) !== title(renamedTo) && gone.length === 0) {
+                out.push({ file, kind: 'renamed', test: title(renamedFrom), before: title(renamedFrom), after: title(renamedTo) });
+            }
+        }
+        removed = [];
+        added = [];
+    };
+    for (const line of diff.split('\n')) {
+        if (line.startsWith('diff --git')) {
+            flush();
+            file = /b\/(\S+)/.exec(line)?.[1] ?? '';
+            isNewFile = false;
+            continue;
+        }
+        // A file added by this change had no agreed behaviour to renegotiate.
+        if (line.startsWith('new file mode'))
+            isNewFile = true;
+        if (line.startsWith('@@'))
+            flush();
+        if (!file || !IS_TEST_FILE.test(file))
+            continue;
+        if (line.startsWith('-') && !line.startsWith('---'))
+            removed.push(line.slice(1));
+        else if (line.startsWith('+') && !line.startsWith('+++'))
+            added.push(line.slice(1));
+    }
+    flush();
+    return out;
+}
+/** The sentences a reviewer reads. Never a verdict: this asks, it does not decide. */
+export function renderExpectations(changes) {
+    if (changes.length === 0)
+        return [];
+    const rewritten = changes.filter((c) => c.kind === 'rewritten');
+    const withdrawn = changes.filter((c) => c.kind === 'withdrawn');
+    const renamed = changes.filter((c) => c.kind === 'renamed');
+    const lines = ['## Checks this change renegotiates', ''];
+    lines.push(`This change edits ${changes.length} check(s) that were already agreed: ${[
+        rewritten.length ? `${rewritten.length} rewritten` : '',
+        withdrawn.length ? `${withdrawn.length} withdrawn` : '',
+        renamed.length ? `${renamed.length} renamed` : '',
+    ]
+        .filter(Boolean)
+        .join(', ')}. Tests change for good reasons, so this is not a fault - but a test written earlier is behaviour someone agreed on, and the change's own description cannot tell you whether replacing it was intended.`, '');
+    for (const change of [...withdrawn, ...rewritten, ...renamed].slice(0, 10)) {
+        const where = `${change.file.split('/').at(-1)}${change.test ? ` — ${change.test}` : ''}`;
+        if (change.kind === 'withdrawn') {
+            lines.push(`- **${where}** no longer checks: \`${change.before}\``);
+        }
+        else if (change.kind === 'renamed') {
+            lines.push(`- **${where}** was renamed to "${change.after}", so a report of it will no longer match the old name`);
+        }
+        else {
+            lines.push(`- **${where}**`, `  - was: \`${change.before}\``, `  - now: \`${change.after}\``);
+        }
+    }
+    lines.push('', 'Ask the author which of these the change intended, and which followed from it.');
+    return lines;
+}

package/dist/engine/review/review-command.js

@@ -6,6 +6,8 @@ import { chooseFramework, detectFrameworks } from '../quality/frameworks.js';
 import { readTriageRecords, datasetFiles } from '../../reporters/store.js';
 import { coverageFor, uncovered } from './coverage.js';
 import { renderReview, reviewChange } from './review.js';
+import { changedExpectations } from './expectations.js';
+import { changedAgreements } from './agreements.js';
 /** Every failure Tewip has diagnosed in this project, from the history it keeps. */
 function history(root, only) {
     const dir = join(root, '.tewip', 'history');
@@ -25,7 +27,7 @@ export async function runReview(cmd) {
     const files = change.files.map((f) => f.split('\\').join('/'));
     const coverage = coverageFor(cmd.root, files);
     const past = history(cmd.root, cmd.name);
-    const review = reviewChange(cmd.diff, files, coverage, past, uncovered(files, coverage));
+    const review = reviewChange(cmd.diff, files, coverage, past, uncovered(files, coverage), changedExpectations(cmd.root, cmd.diff), changedAgreements(cmd.root, cmd.diff));
     // The command that runs just those tests - but only when the project's runner really owns them.
     // Printing a Playwright command for a file Playwright never runs is worse than printing nothing.
     const framework = chooseFramework(detectFrameworks(cmd.root), undefined, false) ?? detectFrameworks(cmd.root)[0] ?? null;

package/dist/engine/review/review.js

@@ -5,6 +5,8 @@
 // August, and that nothing at all covers the file you edited. That comes from Tewip's own history,
 // which is why this costs nothing and why nothing else can produce it.
 import { componentOf } from '../../outlets/items.js';
+import { renderExpectations } from './expectations.js';
+import { renderAgreements } from './agreements.js';
 const fileOf = (record) => record.file ?? '';
 const countBy = (xs, key) => {
     const out = new Map();
@@ -13,7 +15,7 @@ const countBy = (xs, key) => {
     return out;
 };
 /** The review, assembled from the change, what covers it, and what has gone wrong here before. */
-export function reviewChange(range, changed, coverage, history, notCovered) {
+export function reviewChange(range, changed, coverage, history, notCovered, expectations = [], agreements = []) {
     const byFile = new Map();
     for (const record of history) {
         const file = fileOf(record).split('\\').join('/');
@@ -75,7 +77,17 @@ export function reviewChange(range, changed, coverage, history, notCovered) {
     if (tests.length === 0 && changed.length > 0) {
         notes.push('No test in this project reaches any of the changed files. Tewip can write one: `tewip plan "<what the change should do>"` then `tewip write`.');
     }
-    return { range, files: changed, tests, uncovered: notCovered, areas: [...areas.values()].sort((a, b) => b.realBugs - a.realBugs || b.failures - a.failures), runFirst, notes };
+    // A change that rewrites checks people already agreed on is worth saying before anything else:
+    // every other layer reads the change's own account of itself, and this is the one place where
+    // that account can be held against what the project had already decided.
+    if (expectations.length > 0) {
+        const withdrawn = expectations.filter((e) => e.kind === 'withdrawn').length;
+        notes.unshift(`This change edits ${expectations.length} check(s) that were already agreed${withdrawn > 0 ? `, ${withdrawn} of them withdrawn outright` : ''}. Read those first: the change's description cannot tell you whether replacing them was intended.`);
+    }
+    if (agreements.length > 0) {
+        notes.unshift(`This change alters ${agreements.length} thing(s) the project had already decided outside its tests (${[...new Set(agreements.map((a) => a.kind))].join(', ')}). They are listed below with what they were.`);
+    }
+    return { range, files: changed, tests, uncovered: notCovered, expectations, agreements, areas: [...areas.values()].sort((a, b) => b.realBugs - a.realBugs || b.failures - a.failures), runFirst, notes };
 }
 const when = (at) => {
     if (!at)
@@ -94,6 +106,12 @@ export function renderReview(review, runCommand) {
             lines.push(`- ${note}`);
         lines.push('');
     }
+    const renegotiated = renderExpectations(review.expectations);
+    if (renegotiated.length > 0)
+        lines.push(...renegotiated, '');
+    const altered = renderAgreements(review.agreements);
+    if (altered.length > 0)
+        lines.push(...altered, '');
     lines.push('## Tests that cover this change', '');
     if (review.tests.length === 0)
         lines.push('None found.', '');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.16.3",
+  "version": "0.17.1",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",