tewip-ai 0.69.4 → 0.69.6

package/dist/engine/rules.js

@@ -157,6 +157,19 @@ const envServerError = {
  * Declines when the server failed during the test: the wrong value may come from the
  * backend (ENV) or from a test that provokes errors on purpose, which only the LLM layer,
  * seeing the whole trace, can tell apart (rules-3).
+ *
+ * **Do not remove the history requirement.** Phase 64 found that `tewip bugs` files almost nothing
+ * from one CI run - 1 ticket out of 1,964 real failures - because every test in a single run has
+ * `runs < 2` and this rule declines. The proposed fix was to let a plain value mismatch stand on its
+ * own: "expected 4, received 3 needs no model" is an obvious-sounding argument, and it was measured
+ * in Phase 65 over **2,767 hand-labelled real failures from nine projects**:
+ *
+ *     293 value mismatches -> TIMING 265 (90.4%), REAL_BUG 26 (8.9%), ENV 2 (0.7%)
+ *
+ * Per project it is 0%, 0%, 0%, 4%, 5%, 6%, 7%, 11%, 39%. **A flaky test fails its assertion too -
+ * that is what failing looks like.** The mismatch is not the signal; the history around it is, which
+ * is what the streak conditions below are reading and what one CI run cannot supply. Re-derive with
+ * `node eval/mismatch-precision.ts` before changing any of this.
  */
 const realBugAssertion = {
     id: 'real-bug-assertion',

package/dist/engine/work/bug-command.js

@@ -19,6 +19,59 @@ import { TEWIP_LABEL } from './jira-command.js';
 import { trackerFromEnv, trackerHelp } from './trackers.js';
 /** Bugs Tewip filed carry this as well, so a second run finds them and does not file again. */
 export const BUG_LABEL = 'tewip-bug';
+/**
+ * How sure Tewip has to be before it puts a person's name on a ticket.
+ *
+ * Named and exported because of what it collides with. Phase 55 capped a model-only diagnosis at
+ * **0.45** - the model is not reliable enough for its word alone to be acted on - and this bar has
+ * been 0.8 since Phase 10. The two numbers were set eight weeks apart and never compared, and
+ * together they mean **a bug only the model recognised can never be filed**, whatever it says.
+ *
+ * Measured over 1,964 real failures from five public projects (gutenberg, supabase, lxdui, penpot,
+ * jupyterlab) with the rules alone: **20 REAL_BUG verdicts, of which 1 reaches 0.8.** Four of the
+ * five projects produce none at all. On a real jest report of a plain value mismatch - "Expected: 4,
+ * Received: 3", which is as clear as a real bug gets - the verdict is REAL_BUG at 0.45 from the
+ * model, and `bugs` files nothing.
+ *
+ * The bar is not lowered here. A ticket opened on a model's unsupported word is the thing this
+ * project exists not to do, and 0.45 is the honest reading of how often that word is right. What
+ * changes is that the refusal now says which of the two reasons applies, because "none of them is a
+ * real bug Tewip is confident about" was being printed for twenty failures Tewip had called real
+ * bugs.
+ */
+export const CONFIDENT_ENOUGH = 0.8;
+/**
+ * Why nothing was filed - the two cases told apart.
+ *
+ * The old sentence was one line for both: `${items.length} failure(s), and none of them is a real
+ * bug Tewip is confident about (${kinds})`, where `kinds` was every category present. On a real
+ * report it printed **"none of them is a real bug Tewip is confident about (REAL_BUG, ENV)"** -
+ * listing REAL_BUG inside the sentence denying there was one. A reader cannot tell from that whether
+ * Tewip found nothing or found something it would not act on, and those call for opposite next steps.
+ */
+export function whyNothingWasFiled(items, unsure) {
+    const kinds = [...new Set(items.filter((i) => i.diagnosis.category !== 'REAL_BUG').map((i) => i.diagnosis.category))];
+    if (unsure.length === 0) {
+        return (`${items.length} failure(s), and none of them looks like the application being wrong` +
+            `${kinds.length > 0 ? ` - they are ${kinds.join(', ')}` : ''}. Nothing was filed, which is the point.`);
+    }
+    const highest = Math.max(...unsure.map((i) => i.diagnosis.confidence));
+    const byModel = unsure.filter((i) => /AI|gpt|claude|model/i.test(i.source)).length;
+    return [
+        `${items.length} failure(s). ${unsure.length} of them Tewip reads as a real bug, and none of those reaches the ${CONFIDENT_ENOUGH.toFixed(2)} confidence it requires before opening a ticket in somebody's name - the highest is ${highest.toFixed(2)}.`,
+        byModel > 0
+            ? `${byModel === unsure.length ? 'All' : `${byModel}`} of them came from the AI layer, which is capped at 0.45 on purpose: a model's unsupported word is not enough to file on.`
+            : '',
+        // What a CI user can actually do about it. The rules decline a value mismatch without a stable
+        // failing streak, and on one run there is no streak to read - which is not over-caution:
+        // measured over 2,767 labelled real failures, a value mismatch is TIMING 90% of the time and a
+        // real bug 9% of the time (`node eval/mismatch-precision.ts`). History is what separates them.
+        'What would change this: history. The rules will not call a wrong value a bug without a stable failing streak behind it, because a flaky test produces the same wrong value - on real labelled data a value mismatch is a flaky test nine times out of ten. Keep this project\'s runs (`--name <project>`, or let the Action keep them) and the same failure in a second and third run is what the rules read. A person\'s correction on the failure in CI settles it immediately.',
+        `Nothing was filed. These are listed in \`tewip explain\` with their evidence${kinds.length > 0 ? `, alongside ${kinds.join(', ')} failures that are deliberately never filed` : ''}.`,
+    ]
+        .filter(Boolean)
+        .join('\n');
+}
 const isBlobDir = (path) => statSync(path).isDirectory() && readdirSync(path).some((f) => /^report.*\.zip$/.test(f));
 /** The bug report a developer actually wants: what failed, what it did, and why Tewip believes it. */
 export function bugBody(item, runUrl) {
@@ -76,15 +129,15 @@ export async function runBugs(cmd) {
     const items = await buildItems(imported.records, index, { dataset: `local: ${name}`, llm: llmFromEnv(), corrections: loadCorrections(historyDir) });
     // Only what Tewip itself calls a bug, and only when it is confident enough to put a person's name
     // on it. Everything else is deliberately left out.
-    const bugs = items.filter((i) => i.diagnosis.category === 'REAL_BUG' && i.diagnosis.confidence >= 0.8);
+    const bugs = items.filter((i) => i.diagnosis.category === 'REAL_BUG' && i.diagnosis.confidence >= CONFIDENT_ENOUGH);
+    const unsure = items.filter((i) => i.diagnosis.category === 'REAL_BUG' && i.diagnosis.confidence < CONFIDENT_ENOUGH);
     const lines = [`Report: ${cmd.report}`, `Project: ${cmd.projectKey}`, ''];
     if (items.length === 0) {
         console.log('Every test in that report passed. Nothing to file.');
         return [];
     }
     if (bugs.length === 0) {
-        const kinds = [...new Set(items.map((i) => i.diagnosis.category))].join(', ');
-        console.log(`${items.length} failure(s), and none of them is a real bug Tewip is confident about (${kinds}). Nothing was filed, which is the point.`);
+        console.log(whyNothingWasFiled(items, unsure));
         return [];
     }
     const tracker = chosen?.tracker.canWrite() ? chosen.tracker : null;
@@ -122,8 +175,17 @@ export async function runBugs(cmd) {
         lines.push(`  filed ${key}: ${summary}`);
         filed++;
     }
-    const left = items.length - bugs.length;
-    lines.push('', `${bugs.length} of ${items.length} failure(s) look like real bugs; ${left} were flaky tests, environment or test data and were deliberately not filed.`);
+    // `items.length - bugs.length` was described as "flaky tests, environment or test data", which is
+    // untrue of a REAL_BUG sitting below the bar: a bug Tewip is 0.45 sure of was being counted, in
+    // print, as a flaky test. And the `max` cap was silent - the same mistake `prove` made in Phase 58,
+    // where a sampled answer read as a whole one.
+    const notBugs = items.length - bugs.length - unsure.length;
+    const beyondCap = Math.max(0, bugs.length - cmd.max);
+    lines.push('', `${bugs.length} of ${items.length} failure(s) look like real bugs.` +
+        (notBugs > 0 ? ` ${notBugs} were flaky tests, environment or test data and were deliberately not filed.` : '') +
+        (unsure.length > 0 ? ` ${unsure.length} more look like real bugs but below the ${CONFIDENT_ENOUGH.toFixed(2)} confidence needed to file one, so they were not.` : ''));
+    if (beyondCap > 0)
+        lines.push(`${beyondCap} were past the --max ${cmd.max} limit and were not looked at. Raise it with \`--max ${bugs.length}\`.`);
     lines.push(cmd.create ? `${filed} ticket(s) created in ${tracker?.name}. Tewip never moves, assigns or closes one.` : 'Nothing was created. Add --create to file them.');
     console.log(lines.join('\n'));
     return bugs;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.69.4",
+  "version": "0.69.6",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [