tewip-ai 0.17.0 → 0.17.1

package/README.md

@@ -288,11 +288,21 @@ someone agreed on, and this is the one thing in a review that the change's own d
 account for: everything else reads the change's account of itself. It never passes judgement — tests
 change for good reasons — it asks which of them the author intended.
 
-This came out of a measurement rather than an idea. Of 18 real pull requests later found to have
+Alongside it, **the decisions a change alters outside its tests**: a guard that refused something,
+a value other code falls back on, a limit that decides behaviour under load, a version or runner
+pinned on purpose. Same rule - only things that already existed, never additions - and the same
+question rather than a verdict.
+
+Both came out of a measurement rather than an idea. Of 18 real pull requests later found to have
 broken something, the one Tewip's test plan missed was one whose *intention* was wrong: it changed
-a freshness rule, changed the tests to match, and was reverted the next day. Run over those same 18
-pull requests, this check fires on 2 — **and both were reverted**, including the one the plan
-missed. No model is called, so it costs nothing.
+a freshness rule, changed the tests to match, and was reverted the next day.
+
+**What these checks are not is a predictor**, and the measurement says so plainly. On 14 pull
+requests known to have caused a regression they speak up on 4; on 30 ordinary merged pull requests
+from the same repositories, on 9 - 29% against 30%. A change that alters an agreement is not more
+likely to be broken. What they do is put those alterations in front of a person: on the ordinary
+set they surfaced 23 of them, among which a `raise SystemExit` guard that was deleted, a semaphore
+dropped from 4 to 1, and a CI runner image moved. No model is called, so this costs nothing.
 
 ## Which flaky tests to quarantine, and what they cost (new)
 

package/dist/engine/review/agreements.js

@@ -0,0 +1,160 @@
+// Agreements a change renegotiates outside the tests.
+//
+// The test version of this reads assertions, because a test is an agreement written down. It is not
+// the only one. A default someone chose, a guard that refuses bad input, a version pinned on
+// purpose, a permission deliberately kept narrow - each is a decision the project already made, and
+// each can be undone in one line of a diff that reads like housekeeping. The pull request that
+// prompted all of this shipped a wrong rule with its tests updated to match; the same shape appears
+// when a timeout doubles or a `required` becomes optional and nobody says why.
+//
+// What this refuses to be is a linter. It reports only changes to things that were already decided
+// - never additions, never new files - and it asks rather than judges. The test that keeps it
+// honest is the noise measurement: run over eighteen real pull requests, it has to stay quiet on
+// the ordinary ones or a team will learn to scroll past it.
+import { execFileSync } from 'node:child_process';
+const NOT_SOURCE = /(\.spec\.|\.test\.|_test\.|test_|\/tests?\/|\/e2e\/|\/cypress\/|\.md$|\.snap$|lock(file)?$|package-lock\.json$|yarn\.lock$|\.sum$)/i;
+/** An exported or module-level constant, and a parameter's default: values other code relies on. */
+const DEFAULT_VALUE = /^\s*(?:export\s+)?(?:const|let|var|final|static|public|private)?\s*([A-Z][A-Z0-9_]{2,}|[\w$]+)\s*(?::[^=]+)?=\s*(.+?);?\s*$/;
+const PARAM_DEFAULT = /\b([\w$]+)\s*(?::[^=,)]+)?=\s*([^,)]+)[,)]/;
+/** A refusal: the code declining to accept something. */
+const GUARD = /\b(?:throw|raise|panic|abort|reject|assert|invariant|require)\b|\bif\s*\(![\w$.]+\)/;
+/** Who is allowed to do what. */
+const PERMISSION = /\b(?:permissions?|scope|role|allow[_-]?origin|cors|public|private|protected|readonly|auth|admin|policy|rls|chmod|0o?7[0-7][0-7])\b/i;
+/**
+ * A version deliberately pinned - and, in the same family, the thing a job is deliberately run on.
+ * `runs-on: macos-14` carries no dotted version but is exactly the same kind of decision, and one
+ * real pull request in the measurement set bumped it to `macos-26` by accident and was reverted
+ * for it, which is how this case was found.
+ */
+const PIN = /["']?[~^>=<]*\d+\.\d+(\.\d+)?["']?/;
+const RUNS_ON = /^\s*(?:-\s*)?(?:runs-on|image|container|uses|FROM|baseImage|node-version|python-version|go-version)\b\s*[:=]?\s*\S/i;
+const PIN_FILE = /(package\.json|requirements[\w.-]*\.txt|pyproject\.toml|go\.mod|Gemfile|Cargo\.toml|build\.gradle|pom\.xml|Dockerfile|\.ya?ml)$/i;
+/** Numbers that decide behaviour under load rather than logic. */
+const LIMIT = /\b(?:timeout|retries|retry|max|min|limit|ttl|interval|delay|backoff|threshold|concurrency|workers|pool|buffer|size)\b/i;
+/**
+ * Markup and styling, where the words this looks for mean something else entirely. Measured against
+ * 30 ordinary pull requests, `className="min-w-0 flex-1"` becoming `min-w-[9rem]` was reported as a
+ * limit that decides behaviour under load - which is the kind of noise that teaches people to
+ * scroll past a review.
+ */
+const MARKUP = /\bclassName\s*=|\bclass\s*=|\bstyle\s*=|<\/?[a-zA-Z][\w-]*[\s/>]|\.(?:css|scss|sass|less|html|svg)$/;
+/**
+ * The words in a line, with identifiers pulled apart. `MAX_RETRIES` and `maxRetries` are the same
+ * decision as `max retries`, but an underscore is a word character, so a plain \bretries\b matches
+ * neither - which silently classified a retry count as an ordinary default.
+ */
+const words = (line) => line.replace(/[_-]+/g, ' ').replace(/([a-z\d])([A-Z])/g, '$1 $2');
+const clean = (line) => line.trim().slice(0, 150);
+/** Pairs a removed line with the added line that replaced it, by position within the hunk. */
+function pairs(removed, added) {
+    return removed.map((line, i) => [line, added[i] ?? '']);
+}
+/** Two lines that are the same statement with a different value: that is a decision being changed. */
+function sameStatement(before, after) {
+    const shape = (s) => s.replace(/["'`][^"'`]*["'`]/g, 'S').replace(/\b\d+(\.\d+)*\b/g, 'N').replace(/\s+/g, ' ').trim();
+    return Boolean(after) && shape(before) === shape(after) && before.trim() !== after.trim();
+}
+function classify(file, before, after) {
+    const both = `${before} ${after}`;
+    if (MARKUP.test(both) || MARKUP.test(file))
+        return null;
+    if (!after) {
+        // Something was removed outright. Only a refusal is worth reporting: deleted code is the
+        // ordinary business of a diff, but a deleted guard changes what the system accepts.
+        return GUARD.test(before)
+            ? { file, kind: 'guard', before: clean(before), after: '', why: 'a check that refused something was removed, so input the project used to reject is now accepted' }
+            : null;
+    }
+    if (!sameStatement(before, after))
+        return null;
+    if (GUARD.test(before) || GUARD.test(after)) {
+        return { file, kind: 'guard', before: clean(before), after: clean(after), why: 'the condition on which the code refuses something changed' };
+    }
+    if (PERMISSION.test(words(both))) {
+        return { file, kind: 'permission', before: clean(before), after: clean(after), why: 'who or what is allowed here changed' };
+    }
+    if (PIN_FILE.test(file) && (PIN.test(before) || RUNS_ON.test(before))) {
+        return {
+            file,
+            kind: 'pin',
+            before: clean(before),
+            after: clean(after),
+            why: RUNS_ON.test(before) ? 'what this runs on changed' : 'a version that was pinned deliberately moved',
+        };
+    }
+    if (LIMIT.test(words(both))) {
+        return { file, kind: 'limit', before: clean(before), after: clean(after), why: 'a limit that decides behaviour under load changed' };
+    }
+    if (DEFAULT_VALUE.test(before) || PARAM_DEFAULT.test(before)) {
+        return { file, kind: 'default', before: clean(before), after: clean(after), why: 'a value other code falls back on changed' };
+    }
+    return null;
+}
+function sourceDiff(root, range) {
+    try {
+        return execFileSync('git', ['diff', '--unified=0', range, '--'], { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return '';
+    }
+}
+/** Decisions this change alters, outside the tests. Additions and new files are never reported. */
+export function changedAgreements(root, range, diff = sourceDiff(root, range)) {
+    const out = [];
+    let file = '';
+    let isNew = false;
+    let removed = [];
+    let added = [];
+    const flush = () => {
+        if (file && !isNew && !NOT_SOURCE.test(file)) {
+            for (const [before, after] of pairs(removed, added)) {
+                const found = classify(file, before, after);
+                if (found)
+                    out.push(found);
+            }
+        }
+        removed = [];
+        added = [];
+    };
+    for (const line of diff.split('\n')) {
+        if (line.startsWith('diff --git')) {
+            flush();
+            file = /b\/(\S+)/.exec(line)?.[1] ?? '';
+            isNew = false;
+            continue;
+        }
+        if (line.startsWith('new file mode'))
+            isNew = true;
+        if (line.startsWith('@@'))
+            flush();
+        if (line.startsWith('-') && !line.startsWith('---'))
+            removed.push(line.slice(1));
+        else if (line.startsWith('+') && !line.startsWith('+++'))
+            added.push(line.slice(1));
+    }
+    flush();
+    return out;
+}
+const HEADING = {
+    guard: 'checks that refused something',
+    permission: 'who is allowed what',
+    pin: 'versions pinned on purpose',
+    limit: 'limits that decide behaviour under load',
+    default: 'values other code falls back on',
+};
+/** Sentences for a reviewer. As with the tests, this asks; it never decides. */
+export function renderAgreements(agreements) {
+    if (agreements.length === 0)
+        return [];
+    const lines = ['## Decisions this change alters', ''];
+    const kinds = [...new Set(agreements.map((a) => a.kind))];
+    lines.push(`Outside the tests, this change alters ${agreements.length} thing(s) the project had already decided: ${kinds.map((k) => HEADING[k]).join(', ')}. Each may be exactly what was intended - the point is that a diff makes them look like housekeeping.`, '');
+    for (const agreement of agreements.slice(0, 10)) {
+        lines.push(`- **${agreement.file.split('/').at(-1)}** — ${agreement.why}`);
+        lines.push(`  - was: \`${agreement.before}\``);
+        if (agreement.after)
+            lines.push(`  - now: \`${agreement.after}\``);
+    }
+    lines.push('', 'Ask the author which of these the change needed.');
+    return lines;
+}

package/dist/engine/review/review-command.js

@@ -7,6 +7,7 @@ import { readTriageRecords, datasetFiles } from '../../reporters/store.js';
 import { coverageFor, uncovered } from './coverage.js';
 import { renderReview, reviewChange } from './review.js';
 import { changedExpectations } from './expectations.js';
+import { changedAgreements } from './agreements.js';
 /** Every failure Tewip has diagnosed in this project, from the history it keeps. */
 function history(root, only) {
     const dir = join(root, '.tewip', 'history');
@@ -26,7 +27,7 @@ export async function runReview(cmd) {
     const files = change.files.map((f) => f.split('\\').join('/'));
     const coverage = coverageFor(cmd.root, files);
     const past = history(cmd.root, cmd.name);
-    const review = reviewChange(cmd.diff, files, coverage, past, uncovered(files, coverage), changedExpectations(cmd.root, cmd.diff));
+    const review = reviewChange(cmd.diff, files, coverage, past, uncovered(files, coverage), changedExpectations(cmd.root, cmd.diff), changedAgreements(cmd.root, cmd.diff));
     // The command that runs just those tests - but only when the project's runner really owns them.
     // Printing a Playwright command for a file Playwright never runs is worse than printing nothing.
     const framework = chooseFramework(detectFrameworks(cmd.root), undefined, false) ?? detectFrameworks(cmd.root)[0] ?? null;

package/dist/engine/review/review.js

@@ -6,6 +6,7 @@
 // which is why this costs nothing and why nothing else can produce it.
 import { componentOf } from '../../outlets/items.js';
 import { renderExpectations } from './expectations.js';
+import { renderAgreements } from './agreements.js';
 const fileOf = (record) => record.file ?? '';
 const countBy = (xs, key) => {
     const out = new Map();
@@ -14,7 +15,7 @@ const countBy = (xs, key) => {
     return out;
 };
 /** The review, assembled from the change, what covers it, and what has gone wrong here before. */
-export function reviewChange(range, changed, coverage, history, notCovered, expectations = []) {
+export function reviewChange(range, changed, coverage, history, notCovered, expectations = [], agreements = []) {
     const byFile = new Map();
     for (const record of history) {
         const file = fileOf(record).split('\\').join('/');
@@ -83,7 +84,10 @@ export function reviewChange(range, changed, coverage, history, notCovered, expe
         const withdrawn = expectations.filter((e) => e.kind === 'withdrawn').length;
         notes.unshift(`This change edits ${expectations.length} check(s) that were already agreed${withdrawn > 0 ? `, ${withdrawn} of them withdrawn outright` : ''}. Read those first: the change's description cannot tell you whether replacing them was intended.`);
     }
-    return { range, files: changed, tests, uncovered: notCovered, expectations, areas: [...areas.values()].sort((a, b) => b.realBugs - a.realBugs || b.failures - a.failures), runFirst, notes };
+    if (agreements.length > 0) {
+        notes.unshift(`This change alters ${agreements.length} thing(s) the project had already decided outside its tests (${[...new Set(agreements.map((a) => a.kind))].join(', ')}). They are listed below with what they were.`);
+    }
+    return { range, files: changed, tests, uncovered: notCovered, expectations, agreements, areas: [...areas.values()].sort((a, b) => b.realBugs - a.realBugs || b.failures - a.failures), runFirst, notes };
 }
 const when = (at) => {
     if (!at)
@@ -105,6 +109,9 @@ export function renderReview(review, runCommand) {
     const renegotiated = renderExpectations(review.expectations);
     if (renegotiated.length > 0)
         lines.push(...renegotiated, '');
+    const altered = renderAgreements(review.agreements);
+    if (altered.length > 0)
+        lines.push(...altered, '');
     lines.push('## Tests that cover this change', '');
     if (review.tests.length === 0)
         lines.push('None found.', '');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.17.0",
+  "version": "0.17.1",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",