tewip-ai 0.70.16 → 0.70.17

package/dist/engine/flaky/flaky.js

@@ -95,6 +95,21 @@ export function decide(test, { minRuns, alwaysFailsAbove }) {
     //
     // Phase 56 is where this comes from: of the signals available when a test fails, non-determinism
     // is the one that is genuinely decisive, and it costs nothing to read.
+    // This is also what makes quarantine safe, and not the rule above it.
+    //
+    // The rail that refuses to quarantine a test whose failures looked like the application being
+    // wrong reads the *diagnosed* category, and on real stored history there is none: across
+    // gutenberg, gutenberg-aug2 and supabase, all 3,249 harvested failures have no diagnosis at all,
+    // because nobody ran `tewip explain` over them. So the rail never fires.
+    //
+    // Measured anyway, against 2,773 hand labels (`node eval/flaky-safety.ts`): of 241 tests
+    // recommended for quarantine, 218 had a labelled failure and **none of them was ever labelled a
+    // real bug**. All 68 tests that were catching real bugs came back `watch`, never `quarantine`.
+    //
+    // The safety is real and its source is this condition, not the rail: a test that failed and then
+    // passed on a retry of the same commit did both on the same code. A test catching a real bug does
+    // not pass on a retry. That is an observation, which is why it still works when every diagnosis
+    // is missing.
     if (test.recoveredOnRetry > 0 && test.failedRuns >= minRuns) {
         const times = test.recoveredOnRetry === 1 ? 'once' : `${test.recoveredOnRetry} times`;
         const spread = test.branches.length > 1 ? ` It failed on ${test.branches.length} different branches, so it is the test rather than one person's change.` : '';

package/dist/engine/review/review-command.js

@@ -5,7 +5,7 @@ import { changeSummary } from '../quality/grounding.js';
 import { chooseFramework, detectFrameworks } from '../quality/frameworks.js';
 import { readTriageRecords, datasetFiles } from '../../reporters/store.js';
 import { coverageFor, uncovered } from './coverage.js';
-import { renderReview, reviewChange } from './review.js';
+import { renderReview, reviewChange, runFrom } from './review.js';
 import { changedExpectations } from './expectations.js';
 import { changedAgreements } from './agreements.js';
 import { changedTestLines, vouchChanged } from '../vouch/changed.js';
@@ -35,7 +35,13 @@ export async function runReview(cmd) {
     // Printing a Playwright command for a file Playwright never runs is worse than printing nothing.
     const framework = chooseFramework(detectFrameworks(cmd.root), undefined, false) ?? detectFrameworks(cmd.root)[0] ?? null;
     const ownsThem = framework ? review.runFirst.some((f) => resolve(cmd.root, f).startsWith(resolve(framework.testDir))) : false;
-    const runCommand = framework && ownsThem ? (chosen) => chosen.map((f) => framework.command(resolve(cmd.root, f))).join(' && ') : undefined;
+    // And from the directory the runner actually runs in. `framework.command` builds paths relative
+    // to the framework's own cwd, which in a layered project is not where the report is read: this
+    // printed `node --test "test/flaky.test.ts"` for a file at engine/test/flaky.test.ts, and copying
+    // it gave "Could not find 'test/flaky.test.ts'". The one actionable line in a review has to work
+    // where somebody pastes it.
+    const prefix = framework ? runFrom(cmd.root, framework.cwd) : '';
+    const runCommand = framework && ownsThem ? (chosen) => prefix + chosen.map((f) => framework.command(resolve(cmd.root, f))).join(' && ') : undefined;
     // Whether the tests in this change could ever fail. Costs nothing, calls no model, and is the one
     // question an agent-written diff most needs asked of it.
     const vouched = changedTestLines(vouchChanged(cmd.root, files));

package/dist/engine/review/review.js

@@ -1,3 +1,4 @@
+import { relative, sep } from 'node:path';
 // The risk review of a change, from what Tewip already knows about this project.
 //
 // A general coding agent can read a diff. What it cannot do is tell you that the two tests covering
@@ -98,6 +99,19 @@ const when = (at) => {
     return days <= 1 ? 'yesterday' : days < 14 ? `${days} days ago` : days < 60 ? `${Math.round(days / 7)} weeks ago` : `${Math.round(days / 30)} months ago`;
 };
 /** The review as a person reads it, before they read the diff. */
+/**
+ * The `cd` a suggested command needs, when the runner does not run from the repository root.
+ *
+ * A framework builds its commands relative to its own working directory, which in a layered
+ * project is not where the report is read. `tewip review` printed
+ * `node --test "test/flaky.test.ts"` for a file at engine/test/flaky.test.ts, and copying it gave
+ * "Could not find 'test/flaky.test.ts'". The command is the one line in a review somebody acts on;
+ * it has to work where they paste it.
+ */
+export function runFrom(root, cwd) {
+    const where = relative(root, cwd).split(sep).join('/');
+    return where && where !== '.' ? `cd ${where} && ` : '';
+}
 export function renderReview(review, runCommand) {
     const lines = [`# Risk review: ${review.range}`, '', `${review.files.length} changed file(s).`, ''];
     if (review.notes.length > 0) {

package/dist/engine/trace/trace-command.js

@@ -51,7 +51,7 @@ export function traceReport(cmd) {
     const found = plans(cmd.root, cmd.only);
     const { runs, records } = history(cmd.root, cmd.name);
     const traced = found.map((plan) => tracePlan(plan, runs, records));
-    return { traced, report: renderTrace(traced), plans: found.length, runs: runs.length };
+    return { traced, report: renderTrace(traced, runs.length), plans: found.length, runs: runs.length };
 }
 export function runTrace(cmd) {
     const { traced, report, plans: found, runs } = traceReport(cmd);

package/dist/engine/trace/trace.js

@@ -78,7 +78,16 @@ export function tracePlan(plan, runs, records) {
 }
 const MARK = { passing: 'passing', failing: 'FAILING', 'never-run': 'never run', 'not-covered': 'NOT COVERED' };
 /** The answer to "is this covered, and is it passing", written so it can be disagreed with. */
-export function renderTrace(traced) {
+/**
+ * The report.
+ *
+ * `runsKept` is passed separately because the scenarios cannot carry it. With no run history at
+ * all, every scenario comes back `never run` - true of each one and misleading as a whole: it reads
+ * as a measurement of a suite nobody has exercised, when the truth is that nothing has been
+ * recorded here to measure. The footer made it worse, claiming the lines came from "a run it
+ * recorded" when there was no run.
+ */
+export function renderTrace(traced, runsKept = 1) {
     const all = traced.flatMap((t) => t.scenarios);
     const count = (s) => all.filter((x) => x.standing === s).length;
     const lines = ['# From what was asked for, to what is passing', ''];
@@ -90,7 +99,13 @@ export function renderTrace(traced) {
             'writes one, and `tewip write` turns its scenarios into tests.',
         ].join('\n');
     }
-    lines.push(`${all.length} scenario(s) across ${traced.length} requirement(s): ${count('passing')} passing, ${count('failing')} failing, ${count('never-run')} never run, ${count('not-covered')} not covered.`, '', 'A scenario counts as covered only when a real test can be named, and as passing only when a real', 'run attempted it. Everything else says what is missing instead.', '');
+    lines.push(`${all.length} scenario(s) across ${traced.length} requirement(s): ${count('passing')} passing, ${count('failing')} failing, ${count('never-run')} never run, ${count('not-covered')} not covered.`, '');
+    if (runsKept === 0) {
+        lines.push('**No run has been kept here, so nothing below says whether anything passes.** Every scenario', 'reads "never run" because there is no run to have attempted it - not because the tests were', 'written and then left alone. Run `tewip explain <report>` on a real run first, and this becomes', 'an answer rather than a list.', '');
+    }
+    else {
+        lines.push('A scenario counts as covered only when a real test can be named, and as passing only when a real', 'run attempted it. Everything else says what is missing instead.', '');
+    }
     for (const requirement of traced) {
         lines.push(`## ${requirement.requirement}${requirement.ticket ? ` (${requirement.ticket})` : ''}`, '');
         for (const s of requirement.scenarios) {
@@ -112,6 +127,8 @@ export function renderTrace(traced) {
             lines.push(`- ${s.standing === 'failing' ? `fix or triage ${s.test}` : `write a test for ${s.id}: ${s.title}`}`);
         lines.push('');
     }
-    lines.push('Every line above comes from a plan Tewip wrote and a run it recorded. Nothing is inferred.');
+    lines.push(runsKept === 0
+        ? 'Every line above comes from a plan Tewip wrote. No run was recorded here, so nothing above was checked against one.'
+        : 'Every line above comes from a plan Tewip wrote and a run it recorded. Nothing is inferred.');
     return lines.join('\n');
 }

package/dist/engine/verdict/evidence.js

@@ -0,0 +1,12 @@
+// What a verdict is made of: the questions, the standings, and everything `decide` is allowed to
+// read. Split from verdict.ts when it passed this project's 300-line rule.
+//
+// Worth keeping together, because the shape of this file is the argument. Every field that can be
+// absent is nullable or counted, and the comments say which absence means what - an empty list and
+// a question nobody answered look identical otherwise, and four separate phases found the gate
+// reading one as the other.
+export {};
+/**
+ * The verdict, in a fixed order: what is broken first, then what cannot be believed, then what is
+ * missing, and only then "ready".
+ */

package/dist/engine/verdict/render.js

@@ -0,0 +1,22 @@
+export function renderDecision(range, decision) {
+    const mark = { met: 'yes', 'not-met': '**no**', 'not-applicable': 'n/a', 'not-checked': 'not checked' };
+    const title = decision.verdict === 'ready' ? 'Ready, as far as this can be shown' : decision.verdict === 'not-ready' ? 'Not ready' : 'Cannot say';
+    const lines = [
+        `# Is this work done? ${title}`,
+        '',
+        `For \`${range}\`. ${decision.headline}.`,
+        '',
+        '| Question | Answer |',
+        '| --- | --- |',
+        ...decision.checks.map((c) => `| ${c.question} | ${mark[c.standing]} |`),
+        '',
+    ];
+    for (const check of decision.checks) {
+        if (check.standing === 'not-applicable')
+            continue;
+        const said = `${mark[check.standing] === '**no**' ? 'No' : mark[check.standing] === 'yes' ? 'Yes' : 'Not checked'}: ${check.answer}`;
+        lines.push(`### ${check.question}`, '', /[.!?]$/.test(said) ? said : `${said}.`, '', `_Decided by ${check.decidedBy}._`, '');
+    }
+    lines.push('## What this says nothing about', '', ...decision.notChecked.map((n) => `- ${n}`), '', '---', '', 'Nothing above is judged here: each answer comes from a layer with its own measured accuracy, and', 'this only puts them in a fixed order - what is broken, then what cannot be believed, then what is', 'missing. "Cannot say" is a real answer and is never rounded up to ready.');
+    return lines.join(String.fromCharCode(10));
+}

package/dist/engine/verdict/verdict-command.js

@@ -37,13 +37,23 @@ function history(root, only) {
     return { runs, records };
 }
 /** Scenarios of kept plans that this change relates to and nothing covers. Null when no plans exist. */
+/**
+ * Scenarios this change touches that no test covers, and what stopped the question being answered.
+ *
+ * The gaps alone cannot carry it. An empty list means one of three different things: every scenario
+ * is covered, no plan here is about this change, or every plan failed to parse and was silently
+ * skipped - and the gate read all three as "every scenario this change relates to has a test".
+ * Measured on a project with one plan file containing the word "refunds" and no valid JSON.
+ */
 function uncoveredFor(root, changed, attempted) {
     const dir = join(root, '.tewip', 'plans');
     if (!existsSync(dir))
-        return null;
+        return { gaps: null, unreadable: 0, considered: 0 };
     const words = new Set(changed.flatMap((f) => (f.toLowerCase().match(/[a-z][a-z0-9]{3,}/g) ?? [])));
     const clean = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '');
     const out = [];
+    let unreadable = 0;
+    let considered = 0;
     for (const file of readdirSync(dir).filter((f) => f.endsWith('.json'))) {
         try {
             const saved = JSON.parse(readFileSync(join(dir, file), 'utf8'));
@@ -55,6 +65,7 @@ function uncoveredFor(root, changed, attempted) {
             const about = (`${plan.title ?? ''} ${saved.feature ?? ''}`.toLowerCase().match(/[a-z][a-z0-9]{3,}/g) ?? []).some((w) => words.has(w));
             if (!about)
                 continue;
+            considered += 1;
             for (const scenario of plan.scenarios) {
                 const covered = (scenario.covered_by ?? '').trim() || attempted.find((t) => clean(t).includes(clean(scenario.id)) && clean(scenario.id).length >= 4);
                 if (!covered)
@@ -62,10 +73,12 @@ function uncoveredFor(root, changed, attempted) {
             }
         }
         catch {
-            // Not a readable plan.
+            // Skipped in silence until Phase 112: an unparseable plan left the gap list empty,
+            // which the gate read as full coverage.
+            unreadable += 1;
         }
     }
-    return out;
+    return { gaps: out, unreadable, considered };
 }
 export function gather(cmd) {
     const change = changeSummary(cmd.root, cmd.diff);
@@ -112,6 +125,7 @@ export function gather(cmd) {
     })
         .map((r) => ({ testId: r.testId, category: String(r.diagnosis?.category ?? 'UNKNOWN') }));
     const standards = loadStandards(cmd.root);
+    const plans = uncoveredFor(cmd.root, changed, Object.keys(newest?.attempts ?? {}));
     const brokenRules = standards.length === 0 ? null : checkChanged(cmd.root, standards, changed).map((v) => ({ standard: v.standard, file: v.file, line: v.line, why: v.rule.because }));
     const proved = cmd.prove ? proveChanged(cmd.root, changedSource, { budgetSeconds: cmd.proveBudgetSeconds ?? 300, onStep: (m) => console.error(`  ${m}`) }) : null;
     return {
@@ -123,10 +137,15 @@ export function gather(cmd) {
             coverage,
             cannotFail: vouched.cannotFail.map((j) => ({ name: j.name, file: j.file, line: j.line, why: j.why })),
             crashOnly: vouched.crashOnly.length,
+            couldNotJudge: vouched.couldNotJudge.length,
+            proofsTried: proved ? proved.proofs.reduce((n, pr) => n + pr.tried.length, 0) : 0,
+            proofSkipped: proved?.skipped ?? '',
             unnoticed: proved ? proved.proofs.flatMap((p) => p.tried.filter((t) => t.fate === 'unnoticed').map((t) => ({ file: p.file, line: t.mutant.line, kind: t.mutant.kind }))) : null,
             failingNow,
             noHistory: runs.length === 0,
-            uncovered: uncoveredFor(cmd.root, changed, Object.keys(newest?.attempts ?? {})),
+            uncovered: plans.gaps,
+            plansUnreadable: plans.unreadable,
+            plansConsidered: plans.considered,
             brokenRules,
         },
     };

package/dist/engine/verdict/verdict.js

@@ -1,25 +1,27 @@
 /** The questions that cannot be skipped on the way to "ready". */
-const ESSENTIAL = ['Is anything failing?', 'Is the changed code tested?'];
-/**
- * The verdict, in a fixed order: what is broken first, then what cannot be believed, then what is
- * missing, and only then "ready".
- */
+const ESSENTIAL = ['Is everything passing?', 'Is the changed code tested?'];
 export function decide(evidence) {
     const checks = [];
     // 1. Is anything failing right now? Nothing else matters while it is.
+    //
+    // Asked as "Is everything passing?" so that a yes is good news, like every other row. Phrased
+    // the other way round, the table said **no** to "Is anything failing?" when a test really was
+    // failing - true of the standing, backwards as an answer, and the table is the part a reviewer
+    // scans before anything else. The headline said Not ready and named the test either way; the
+    // row beside it still read like reassurance.
     if (evidence.noHistory) {
-        checks.push({ question: 'Is anything failing?', standing: 'not-checked', answer: 'no run has been kept for this project, so Tewip has not seen a test result here. Run `tewip explain <report>` in CI and it will know.', decidedBy: 'kept history' });
+        checks.push({ question: 'Is everything passing?', standing: 'not-checked', answer: 'no run has been kept for this project, so Tewip has not seen a test result here. Run `tewip explain <report>` in CI and it will know.', decidedBy: 'kept history' });
     }
     else if (evidence.failingNow.length > 0) {
         checks.push({
-            question: 'Is anything failing?',
+            question: 'Is everything passing?',
             standing: 'not-met',
             answer: `${evidence.failingNow.length} test(s) touching this change failed in the last run: ${evidence.failingNow.slice(0, 3).map((f) => `${f.testId} (${f.category})`).join(', ')}`,
             decidedBy: 'the triage rules and model, measured on real CI data',
         });
     }
     else {
-        checks.push({ question: 'Is anything failing?', standing: 'met', answer: 'nothing touching this change failed in the last kept run', decidedBy: 'kept history' });
+        checks.push({ question: 'Is everything passing?', standing: 'met', answer: 'nothing touching this change failed in the last kept run', decidedBy: 'kept history' });
     }
     // 2. Does anything test the change at all?
     if (evidence.changedSource.length === 0) {
@@ -96,6 +98,19 @@ export function decide(evidence) {
             decidedBy: 'tewip vouch, measured on 41,523 real tests in the six public projects in corpus/expected.json',
         });
     }
+    else if (evidence.couldNotJudge > 0 && evidence.couldNotJudge >= evidence.changedTests.length) {
+        // Nothing was established. `cannotFail` is empty because vouch declined every test in the
+        // change, not because it found them sound - three tests that call the function under test and
+        // assert nothing come back exactly like three good ones. Answering "yes" there is a clean
+        // answer produced by looking at nothing, in the question that matters most on a change somebody
+        // did not write by hand.
+        checks.push({
+            question: 'Can the tests in this change fail?',
+            standing: 'not-checked',
+            answer: `vouch would not judge any of the ${evidence.couldNotJudge} test(s) here: each body is one call to something it cannot follow, so whether they assert anything is unknown`,
+            decidedBy: 'tewip vouch',
+        });
+    }
     else {
         checks.push({
             question: 'Can the tests in this change fail?',
@@ -116,6 +131,19 @@ export function decide(evidence) {
             decidedBy: 'tewip prove, by running the tests against a broken copy of the code',
         });
     }
+    else if (evidence.proofsTried === 0) {
+        // Nothing was tried, so nothing was established. The reason is carried through rather than
+        // flattened into a shrug: "no unit test framework was found here" is a different thing for a
+        // reader to act on than "this costs minutes, ask for it".
+        checks.push({
+            question: 'Would the tests catch a bug in this code?',
+            standing: 'not-checked',
+            answer: evidence.proofSkipped
+                ? `nothing could be proved here: ${evidence.proofSkipped}`
+                : 'nothing was proved here, so whether the tests would catch a bug is unknown',
+            decidedBy: 'tewip prove',
+        });
+    }
     else {
         checks.push({ question: 'Would the tests catch a bug in this code?', standing: 'met', answer: 'every deliberate change made to this code was caught by a covering test', decidedBy: 'tewip prove' });
     }
@@ -147,6 +175,19 @@ export function decide(evidence) {
             decidedBy: 'tewip trace, which counts a scenario covered only when a real test can be named',
         });
     }
+    else if (evidence.plansUnreadable > 0 || evidence.plansConsidered === 0) {
+        // An empty gap list means one of three things and only one of them is good news. A plan that
+        // would not parse was skipped in silence, and a project whose plans are all about other
+        // features has not had this question answered at all.
+        checks.push({
+            question: 'Is what was asked for covered?',
+            standing: 'not-checked',
+            answer: evidence.plansUnreadable > 0
+                ? `${evidence.plansUnreadable} plan file(s) here could not be read, so their scenarios were never compared with this change`
+                : 'no plan this project keeps is about this change, so there was nothing to compare it with',
+            decidedBy: 'tewip trace',
+        });
+    }
     else {
         checks.push({ question: 'Is what was asked for covered?', standing: 'met', answer: 'every scenario this change relates to has a test that can be named', decidedBy: 'tewip trace' });
     }
@@ -179,25 +220,6 @@ export function decide(evidence) {
         ],
     };
 }
-export function renderDecision(range, decision) {
-    const mark = { met: 'yes', 'not-met': '**no**', 'not-applicable': 'n/a', 'not-checked': 'not checked' };
-    const title = decision.verdict === 'ready' ? 'Ready, as far as this can be shown' : decision.verdict === 'not-ready' ? 'Not ready' : 'Cannot say';
-    const lines = [
-        `# Is this work done? ${title}`,
-        '',
-        `For \`${range}\`. ${decision.headline}.`,
-        '',
-        '| Question | Answer |',
-        '| --- | --- |',
-        ...decision.checks.map((c) => `| ${c.question} | ${mark[c.standing]} |`),
-        '',
-    ];
-    for (const check of decision.checks) {
-        if (check.standing === 'not-applicable')
-            continue;
-        const said = `${mark[check.standing] === '**no**' ? 'No' : mark[check.standing] === 'yes' ? 'Yes' : 'Not checked'}: ${check.answer}`;
-        lines.push(`### ${check.question}`, '', /[.!?]$/.test(said) ? said : `${said}.`, '', `_Decided by ${check.decidedBy}._`, '');
-    }
-    lines.push('## What this says nothing about', '', ...decision.notChecked.map((n) => `- ${n}`), '', '---', '', 'Nothing above is judged here: each answer comes from a layer with its own measured accuracy, and', 'this only puts them in a fixed order - what is broken, then what cannot be believed, then what is', 'missing. "Cannot say" is a real answer and is never rounded up to ready.');
-    return lines.join(String.fromCharCode(10));
-}
+// The report lives in render.ts; re-exported here so nothing that reads a verdict has to know
+// where it is drawn.
+export { renderDecision } from './render.js';

package/dist/engine/vouch/changed.js

@@ -9,7 +9,7 @@ import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { CANNOT_FAIL, vouchFile } from './vouch.js';
 export function vouchChanged(root, changed) {
-    const out = { files: 0, read: 0, cannotFail: [], crashOnly: [] };
+    const out = { files: 0, read: 0, cannotFail: [], crashOnly: [], couldNotJudge: [] };
     for (const file of changed) {
         const full = join(root, file);
         if (!existsSync(full))
@@ -28,6 +28,7 @@ export function vouchChanged(root, changed) {
         out.read += vouched.judgements.length;
         out.cannotFail.push(...vouched.judgements.filter((j) => CANNOT_FAIL.includes(j.verdict)));
         out.crashOnly.push(...vouched.judgements.filter((j) => j.verdict === 'crash-only'));
+        out.couldNotJudge.push(...vouched.judgements.filter((j) => j.verdict === 'cannot-tell'));
     }
     return out;
 }

package/dist/engine/vouch/text.js

@@ -37,9 +37,22 @@ export function withoutComments(body) {
         }
         if (ch === '/' && next === '*') {
             const close = body.indexOf('*/', i + 2);
-            // An unclosed block comment runs to the end, which is what a compiler would do too.
+            // Only when it closes. An unclosed one was treated as running to the end of the body, on the
+            // reasoning that a compiler would do the same - but a compiler is reading a language that has
+            // block comments, and this is handed every language. Ruby has none, and discourse writes
+            //
+            //     %r{https://matrix.org/_matrix/client/r0/rooms/!blah/send/m.room.message/*}
+            //
+            // where the `/*` is the end of a regex literal. Running to the end of the body deleted the
+            // three assertions below it and reported two thoroughly checked tests as crash-only. A file
+            // that really ends inside a block comment does not compile; a regex containing `/*` is
+            // ordinary. The old regex required the closing `*/` and was right to.
+            if (close === -1) {
+                out += ch;
+                continue;
+            }
             out += ' ';
-            i = close === -1 ? body.length : close + 1;
+            i = close + 1;
             continue;
         }
         const atLineStart = i === 0 || /\s/.test(body[i - 1] ?? '');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.70.16",
+  "version": "0.70.17",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [