tewip-ai 0.64.0 → 0.66.0

package/dist/engine/flaky/flaky.js

@@ -70,7 +70,9 @@ const flakyShare = (test) => {
  *   quarantine needs the test to have failed in several runs, and to be unreliable rather than
  *   consistently wrong.
  */
-function decide(test, { minRuns, alwaysFailsAbove }) {
+/** The verdict for one test, and the sentence that justifies it. Exported so it can be tested
+ * against the shapes real projects produce rather than only through a whole report. */
+export function decide(test, { minRuns, alwaysFailsAbove }) {
     const rate = share(test);
     const percent = `${Math.round(rate * 100)}% of the ${test.totalRuns} run(s) it was in`;
     const realBugs = test.categories.find((c) => c.category === 'REAL_BUG')?.count ?? 0;
@@ -80,6 +82,27 @@ function decide(test, { minRuns, alwaysFailsAbove }) {
     if (rate > alwaysFailsAbove && test.totalRuns >= minRuns) {
         return { verdict: 'fix-now', why: `it fails in ${percent}. That is not flakiness, it is a test that does not work - hiding it would hide whatever it is telling you.` };
     }
+    // Observed, not inferred. A test that failed and then passed on a retry within the same run did
+    // both on the same code, same commit, same machine: that is not evidence of flakiness, it is
+    // flakiness, and no diagnosis is needed to see it.
+    //
+    // This mattered more than it looks. Everything below rests on `flakyShare`, which counts how many
+    // failures a *previous* `tewip explain` already called flaky - so on a project's raw history,
+    // where nothing has been diagnosed yet, no test could ever qualify and the command recommended
+    // quarantining nothing at all. Measured on 1,305 real failures from WordPress/gutenberg: **124 of
+    // the 180 failing tests had recovered on retry, and all 124 were filed under "watch"**. One of
+    // them had done it 93 separate times.
+    //
+    // Phase 56 is where this comes from: of the signals available when a test fails, non-determinism
+    // is the one that is genuinely decisive, and it costs nothing to read.
+    if (test.recoveredOnRetry > 0 && test.failedRuns >= minRuns) {
+        const times = test.recoveredOnRetry === 1 ? 'once' : `${test.recoveredOnRetry} times`;
+        const spread = test.branches.length > 1 ? ` It failed on ${test.branches.length} different branches, so it is the test rather than one person's change.` : '';
+        return {
+            verdict: 'quarantine',
+            why: `it failed and then passed on a retry in the same run ${times}: the same code did both, which is what a flaky test is. It failed in ${percent}.${spread}`,
+        };
+    }
     if (test.failedRuns >= minRuns && flakyShare(test) >= 0.5) {
         const spread = test.branches.length > 1 ? ` It failed on ${test.branches.length} different branches, so it is the test rather than one person's change.` : '';
         return { verdict: 'quarantine', why: `it failed in ${percent}, and Tewip read most of those as the test being unreliable.${spread}` };

package/dist/engine/llm/llm-diagnose.js

@@ -50,6 +50,28 @@ export const LLM_CONFIDENCE_CEILING = 0.45;
 export function contradictsHistory(category, context) {
     if (category !== 'REAL_BUG' && category !== 'LOCATOR_DRIFT')
         return null;
+    // A timeout is not evidence of a bug, and that is measured rather than argued.
+    //
+    // Phase 56 took 471 real labelled CI failures from WordPress/gutenberg and asked, for each signal
+    // a classifier can actually see when the test fails, how far it moves the label from its base
+    // rate. Three signals move it a long way:
+    //
+    //   error says Timeout/exceeded   ->  TIMING 91%   (37% when absent)
+    //   error is an assertion         ->  TIMING 93%   (ENV 86% when absent)
+    //   error mentions a locator      ->  TIMING 94%
+    //
+    // and REAL_BUG sits at 3-6% in **every single bucket** - its base rate, undisturbed. Nothing in
+    // the evidence available at failure time distinguishes a real bug from flakiness. That is the
+    // whole explanation for a model layer scoring 17.8%: it reaches for the one category the evidence
+    // cannot support, and is wrong about 94% of the time when it does.
+    //
+    // So a verdict of REAL_BUG resting on nothing but a timeout is refused. Six per cent of those
+    // really are bugs, and they are not lost: the case goes to a person with the model's reasoning
+    // attached, which is what this project does with every question it cannot answer.
+    const message = context.error?.message ?? '';
+    if (category === 'REAL_BUG' && /timeout|exceeded/i.test(message)) {
+        return 'refused: the only evidence is a timeout, and across 471 labelled failures a timeout was TIMING 91% of the time and REAL_BUG 6%';
+    }
     if (context.thisRun?.anotherAttemptPassed) {
         return `refused: another attempt of the same code passed in this run, so the failure is not deterministic and cannot be ${category}`;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.64.0",
+  "version": "0.66.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [