tewip-ai 0.63.0 → 0.65.0

package/dist/engine/llm/llm-diagnose.js

@@ -23,13 +23,100 @@ export function parseLlmOutput(raw) {
         evidence: value.evidence.slice(0, 4).map((e) => truncate(e, 300)),
     };
 }
+/**
+ * A verdict the run history already rules out, whatever the model says.
+ *
+ * The prompt has told the model since llm-6 that a failure which passed on another attempt of the
+ * same code cannot be a real bug or a drifted locator - both of those fail every time. Measured
+ * against 547 real labelled CI failures from WordPress/gutenberg, it does not obey: the model layer
+ * decided 87 of them and got **9 right**, and almost every miss was a flaky timing failure called
+ * REAL_BUG or LOCATOR_DRIFT at a confidence of 0.9 or higher.
+ *
+ * Telling a team to hunt a bug that is not there is the most expensive mistake this tool can make,
+ * and an instruction in a prompt is a request, not a guarantee. The same fact is checked here in
+ * code, where it holds.
+ *
+ * It refuses rather than rewrites. "This cannot be REAL_BUG" is not "this is TIMING" - ENV and
+ * TEST_DATA both survive a retry too - so the answer becomes UNKNOWN, which routes the failure to a
+ * person with the model's reasoning attached rather than replacing one guess with another.
+ */
+/**
+ * The most this layer may claim, until a measured run says otherwise.
+ *
+ * Below 0.5 so that it routes to a person by the rule the prompt already states: "Low confidence
+ * routes the failure to a human". See the comment at the call site for the measurement.
+ */
+export const LLM_CONFIDENCE_CEILING = 0.45;
+export function contradictsHistory(category, context) {
+    if (category !== 'REAL_BUG' && category !== 'LOCATOR_DRIFT')
+        return null;
+    // A timeout is not evidence of a bug, and that is measured rather than argued.
+    //
+    // Phase 56 took 471 real labelled CI failures from WordPress/gutenberg and asked, for each signal
+    // a classifier can actually see when the test fails, how far it moves the label from its base
+    // rate. Three signals move it a long way:
+    //
+    //   error says Timeout/exceeded   ->  TIMING 91%   (37% when absent)
+    //   error is an assertion         ->  TIMING 93%   (ENV 86% when absent)
+    //   error mentions a locator      ->  TIMING 94%
+    //
+    // and REAL_BUG sits at 3-6% in **every single bucket** - its base rate, undisturbed. Nothing in
+    // the evidence available at failure time distinguishes a real bug from flakiness. That is the
+    // whole explanation for a model layer scoring 17.8%: it reaches for the one category the evidence
+    // cannot support, and is wrong about 94% of the time when it does.
+    //
+    // So a verdict of REAL_BUG resting on nothing but a timeout is refused. Six per cent of those
+    // really are bugs, and they are not lost: the case goes to a person with the model's reasoning
+    // attached, which is what this project does with every question it cannot answer.
+    const message = context.error?.message ?? '';
+    if (category === 'REAL_BUG' && /timeout|exceeded/i.test(message)) {
+        return 'refused: the only evidence is a timeout, and across 471 labelled failures a timeout was TIMING 91% of the time and REAL_BUG 6%';
+    }
+    if (context.thisRun?.anotherAttemptPassed) {
+        return `refused: another attempt of the same code passed in this run, so the failure is not deterministic and cannot be ${category}`;
+    }
+    if (/both failed and passed|not deterministic/i.test(context.history?.pattern ?? '')) {
+        return `refused: the history says "${context.history.pattern}", so the failure is not deterministic and cannot be ${category}`;
+    }
+    return null;
+}
 export async function llmDiagnose(context, rules, client, now = new Date()) {
     try {
         const raw = await client.complete({ system: SYSTEM_PROMPT, user: JSON.stringify(context), schemaName: OUTPUT_SCHEMA_NAME, schema: OUTPUT_SCHEMA });
         const out = parseLlmOutput(raw);
+        const refused = contradictsHistory(out.category, context);
+        if (refused) {
+            return {
+                category: 'UNKNOWN',
+                confidence: 0,
+                source: 'llm',
+                model: client.model,
+                engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,
+                diagnosedAt: now.toISOString(),
+                evidence: [
+                    { source: 'llm', message: `the model said ${out.category} (${out.confidence.toFixed(2)}): ${out.reasoning}` },
+                    { source: 'rules', message: refused },
+                ],
+            };
+        }
         return {
             category: out.category,
-            confidence: out.confidence,
+            // Capped, because the number it reports is not the number it earns.
+            //
+            // Measured against 752 real labelled CI failures from WordPress/gutenberg - labels taken from
+            // what the developer actually changed to fix each one - this layer decided 90 and got 16
+            // right: **17.8%**. Seventy per cent of those failures are TIMING, so answering "TIMING" to
+            // everything would have scored about 70%. The model is four times worse than the most trivial
+            // baseline there is, and it was reporting 0.90 to 0.96 while being wrong.
+            //
+            // The rules layer, on the same data, is 98.5% and keeps its own confidence untouched.
+            //
+            // Nothing is discarded: the category and the reasoning are still reported, and a person still
+            // sees them. What changes is that no consumer downstream can mistake this for a settled
+            // answer - 0.7 is where a rule becomes actionable and 0.85 is where a fix may be attempted,
+            // and a layer at 17.8% has no business clearing either. Raise this when a measured run earns
+            // it, and not before.
+            confidence: Math.min(out.confidence, LLM_CONFIDENCE_CEILING),
             source: 'llm',
             model: client.model,
             engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.63.0",
+  "version": "0.65.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [