tewip-ai 0.63.0 → 0.64.0

package/dist/engine/llm/llm-diagnose.js

@@ -23,13 +23,78 @@ export function parseLlmOutput(raw) {
         evidence: value.evidence.slice(0, 4).map((e) => truncate(e, 300)),
     };
 }
+/**
+ * A verdict the run history already rules out, whatever the model says.
+ *
+ * The prompt has told the model since llm-6 that a failure which passed on another attempt of the
+ * same code cannot be a real bug or a drifted locator - both of those fail every time. Measured
+ * against 547 real labelled CI failures from WordPress/gutenberg, it does not obey: the model layer
+ * decided 87 of them and got **9 right**, and almost every miss was a flaky timing failure called
+ * REAL_BUG or LOCATOR_DRIFT at a confidence of 0.9 or higher.
+ *
+ * Telling a team to hunt a bug that is not there is the most expensive mistake this tool can make,
+ * and an instruction in a prompt is a request, not a guarantee. The same fact is checked here in
+ * code, where it holds.
+ *
+ * It refuses rather than rewrites. "This cannot be REAL_BUG" is not "this is TIMING" - ENV and
+ * TEST_DATA both survive a retry too - so the answer becomes UNKNOWN, which routes the failure to a
+ * person with the model's reasoning attached rather than replacing one guess with another.
+ */
+/**
+ * The most this layer may claim, until a measured run says otherwise.
+ *
+ * Below 0.5 so that it routes to a person by the rule the prompt already states: "Low confidence
+ * routes the failure to a human". See the comment at the call site for the measurement.
+ */
+export const LLM_CONFIDENCE_CEILING = 0.45;
+export function contradictsHistory(category, context) {
+    if (category !== 'REAL_BUG' && category !== 'LOCATOR_DRIFT')
+        return null;
+    if (context.thisRun?.anotherAttemptPassed) {
+        return `refused: another attempt of the same code passed in this run, so the failure is not deterministic and cannot be ${category}`;
+    }
+    if (/both failed and passed|not deterministic/i.test(context.history?.pattern ?? '')) {
+        return `refused: the history says "${context.history.pattern}", so the failure is not deterministic and cannot be ${category}`;
+    }
+    return null;
+}
 export async function llmDiagnose(context, rules, client, now = new Date()) {
     try {
         const raw = await client.complete({ system: SYSTEM_PROMPT, user: JSON.stringify(context), schemaName: OUTPUT_SCHEMA_NAME, schema: OUTPUT_SCHEMA });
         const out = parseLlmOutput(raw);
+        const refused = contradictsHistory(out.category, context);
+        if (refused) {
+            return {
+                category: 'UNKNOWN',
+                confidence: 0,
+                source: 'llm',
+                model: client.model,
+                engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,
+                diagnosedAt: now.toISOString(),
+                evidence: [
+                    { source: 'llm', message: `the model said ${out.category} (${out.confidence.toFixed(2)}): ${out.reasoning}` },
+                    { source: 'rules', message: refused },
+                ],
+            };
+        }
         return {
             category: out.category,
-            confidence: out.confidence,
+            // Capped, because the number it reports is not the number it earns.
+            //
+            // Measured against 752 real labelled CI failures from WordPress/gutenberg - labels taken from
+            // what the developer actually changed to fix each one - this layer decided 90 and got 16
+            // right: **17.8%**. Seventy per cent of those failures are TIMING, so answering "TIMING" to
+            // everything would have scored about 70%. The model is four times worse than the most trivial
+            // baseline there is, and it was reporting 0.90 to 0.96 while being wrong.
+            //
+            // The rules layer, on the same data, is 98.5% and keeps its own confidence untouched.
+            //
+            // Nothing is discarded: the category and the reasoning are still reported, and a person still
+            // sees them. What changes is that no consumer downstream can mistake this for a settled
+            // answer - 0.7 is where a rule becomes actionable and 0.85 is where a fix may be attempted,
+            // and a layer at 17.8% has no business clearing either. Raise this when a measured run earns
+            // it, and not before.
+            confidence: Math.min(out.confidence, LLM_CONFIDENCE_CEILING),
             source: 'llm',
             model: client.model,
             engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.63.0",
+  "version": "0.64.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [