tewip-ai 0.62.0 → 0.64.0

package/dist/engine/llm/llm-diagnose.js

@@ -23,13 +23,78 @@ export function parseLlmOutput(raw) {
         evidence: value.evidence.slice(0, 4).map((e) => truncate(e, 300)),
     };
 }
+/**
+ * A verdict the run history already rules out, whatever the model says.
+ *
+ * The prompt has told the model since llm-6 that a failure which passed on another attempt of the
+ * same code cannot be a real bug or a drifted locator - both of those fail every time. Measured
+ * against 547 real labelled CI failures from WordPress/gutenberg, it does not obey: the model layer
+ * decided 87 of them and got **9 right**, and almost every miss was a flaky timing failure called
+ * REAL_BUG or LOCATOR_DRIFT at a confidence of 0.9 or higher.
+ *
+ * Telling a team to hunt a bug that is not there is the most expensive mistake this tool can make,
+ * and an instruction in a prompt is a request, not a guarantee. The same fact is checked here in
+ * code, where it holds.
+ *
+ * It refuses rather than rewrites. "This cannot be REAL_BUG" is not "this is TIMING" - ENV and
+ * TEST_DATA both survive a retry too - so the answer becomes UNKNOWN, which routes the failure to a
+ * person with the model's reasoning attached rather than replacing one guess with another.
+ */
+/**
+ * The most this layer may claim, until a measured run says otherwise.
+ *
+ * Below 0.5 so that it routes to a person by the rule the prompt already states: "Low confidence
+ * routes the failure to a human". See the comment at the call site for the measurement.
+ */
+export const LLM_CONFIDENCE_CEILING = 0.45;
+export function contradictsHistory(category, context) {
+    if (category !== 'REAL_BUG' && category !== 'LOCATOR_DRIFT')
+        return null;
+    if (context.thisRun?.anotherAttemptPassed) {
+        return `refused: another attempt of the same code passed in this run, so the failure is not deterministic and cannot be ${category}`;
+    }
+    if (/both failed and passed|not deterministic/i.test(context.history?.pattern ?? '')) {
+        return `refused: the history says "${context.history.pattern}", so the failure is not deterministic and cannot be ${category}`;
+    }
+    return null;
+}
 export async function llmDiagnose(context, rules, client, now = new Date()) {
     try {
         const raw = await client.complete({ system: SYSTEM_PROMPT, user: JSON.stringify(context), schemaName: OUTPUT_SCHEMA_NAME, schema: OUTPUT_SCHEMA });
         const out = parseLlmOutput(raw);
+        const refused = contradictsHistory(out.category, context);
+        if (refused) {
+            return {
+                category: 'UNKNOWN',
+                confidence: 0,
+                source: 'llm',
+                model: client.model,
+                engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,
+                diagnosedAt: now.toISOString(),
+                evidence: [
+                    { source: 'llm', message: `the model said ${out.category} (${out.confidence.toFixed(2)}): ${out.reasoning}` },
+                    { source: 'rules', message: refused },
+                ],
+            };
+        }
         return {
             category: out.category,
-            confidence: out.confidence,
+            // Capped, because the number it reports is not the number it earns.
+            //
+            // Measured against 752 real labelled CI failures from WordPress/gutenberg - labels taken from
+            // what the developer actually changed to fix each one - this layer decided 90 and got 16
+            // right: **17.8%**. Seventy per cent of those failures are TIMING, so answering "TIMING" to
+            // everything would have scored about 70%. The model is four times worse than the most trivial
+            // baseline there is, and it was reporting 0.90 to 0.96 while being wrong.
+            //
+            // The rules layer, on the same data, is 98.5% and keeps its own confidence untouched.
+            //
+            // Nothing is discarded: the category and the reasoning are still reported, and a person still
+            // sees them. What changes is that no consumer downstream can mistake this for a settled
+            // answer - 0.7 is where a rule becomes actionable and 0.85 is where a fix may be attempted,
+            // and a layer at 17.8% has no business clearing either. Raise this when a measured run earns
+            // it, and not before.
+            confidence: Math.min(out.confidence, LLM_CONFIDENCE_CEILING),
             source: 'llm',
             model: client.model,
             engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,

package/dist/reporters/store.js

@@ -1,5 +1,5 @@
 // JSONL persistence shared by the reporter, the history script and future layers.
-import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { openSync, readSync, closeSync, fstatSync, appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 export const PROJECT_ROOT = join(import.meta.dirname, '..');
 export const DATA_DIR = join(PROJECT_ROOT, 'data');
@@ -13,23 +13,68 @@ export function appendJsonl(file, rows) {
     mkdirSync(dirname(file), { recursive: true });
     appendFileSync(file, rows.map((row) => JSON.stringify(row)).join('\n') + '\n', 'utf8');
 }
-/** Reads a JSONL file; missing file yields []. Malformed lines are skipped with a warning. */
+/**
+ * Reads a JSONL file; missing file yields []. Malformed lines are skipped with a warning.
+ *
+ * Read in chunks rather than whole. `readFileSync(file, 'utf8')` builds one string, and a string in
+ * Node cannot exceed 0x1fffffe8 characters - about 512MB. These files only ever grow: the weekly
+ * shadow run appends every CI failure it harvests from a public project and keeps the lot, so that
+ * accuracy is measured against everything ever collected rather than one week of it. gutenberg's
+ * dataset crossed the limit and **every run since has died on this line**, for weeks, while the
+ * workflow's only symptom was a red tick nobody was looking at.
+ *
+ * A UTF-8 character can straddle a chunk boundary, so the last partial line is carried into the next
+ * read rather than decoded on its own - decoding a split character gives U+FFFD and a line that no
+ * longer parses.
+ */
 export function readJsonl(file) {
     if (!existsSync(file))
         return [];
     const rows = [];
-    readFileSync(file, 'utf8')
-        .split(/\r?\n/)
-        .forEach((line, i) => {
-        if (!line.trim())
-            return;
-        try {
-            rows.push(JSON.parse(line));
+    const handle = openSync(file, 'r');
+    try {
+        const size = fstatSync(handle).size;
+        const buffer = Buffer.allocUnsafe(8 * 1024 * 1024);
+        let carry = Buffer.alloc(0);
+        let line = 0;
+        let offset = 0;
+        while (offset < size) {
+            const read = readSync(handle, buffer, 0, buffer.length, offset);
+            if (read <= 0)
+                break;
+            offset += read;
+            const chunk = Buffer.concat([carry, buffer.subarray(0, read)]);
+            let from = 0;
+            for (let i = 0; i < chunk.length; i += 1) {
+                if (chunk[i] !== 0x0a)
+                    continue;
+                const text = chunk.subarray(from, i).toString('utf8').replace(/\r$/, '');
+                from = i + 1;
+                line += 1;
+                if (!text.trim())
+                    continue;
+                try {
+                    rows.push(JSON.parse(text));
+                }
+                catch {
+                    console.warn(`[triage] skipping malformed line ${line} in ${file}`);
+                }
+            }
+            carry = chunk.subarray(from);
         }
-        catch {
-            console.warn(`[triage] skipping malformed line ${i + 1} in ${file}`);
+        const last = carry.toString('utf8').replace(/\r$/, '');
+        if (last.trim()) {
+            try {
+                rows.push(JSON.parse(last));
+            }
+            catch {
+                console.warn(`[triage] skipping malformed line ${line + 1} in ${file}`);
+            }
         }
-    });
+    }
+    finally {
+        closeSync(handle);
+    }
     return rows;
 }
 export const readTriageRecords = (file = TRIAGE_FILE) => readJsonl(file);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.62.0",
+  "version": "0.64.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [