tewip-ai 0.69.5 → 0.69.7

package/dist/engine/rules.js

@@ -157,6 +157,19 @@ const envServerError = {
  * Declines when the server failed during the test: the wrong value may come from the
  * backend (ENV) or from a test that provokes errors on purpose, which only the LLM layer,
  * seeing the whole trace, can tell apart (rules-3).
+ *
+ * **Do not remove the history requirement.** Phase 64 found that `tewip bugs` files almost nothing
+ * from one CI run - 1 ticket out of 1,964 real failures - because every test in a single run has
+ * `runs < 2` and this rule declines. The proposed fix was to let a plain value mismatch stand on its
+ * own: "expected 4, received 3 needs no model" is an obvious-sounding argument, and it was measured
+ * in Phase 65 over **2,767 hand-labelled real failures from nine projects**:
+ *
+ *     293 value mismatches -> TIMING 265 (90.4%), REAL_BUG 26 (8.9%), ENV 2 (0.7%)
+ *
+ * Per project it is 0%, 0%, 0%, 4%, 5%, 6%, 7%, 11%, 39%. **A flaky test fails its assertion too -
+ * that is what failing looks like.** The mismatch is not the signal; the history around it is, which
+ * is what the streak conditions below are reading and what one CI run cannot supply. Re-derive with
+ * `node eval/mismatch-precision.ts` before changing any of this.
  */
 const realBugAssertion = {
     id: 'real-bug-assertion',

package/dist/engine/trace/trace.js

@@ -21,20 +21,44 @@ export function testFor(scenario, attempted) {
     const id = clean(scenario.id);
     return id.length >= 4 ? (attempted.find((t) => clean(t).includes(id)) ?? '') : '';
 }
+/**
+ * Why no test could be named - and what that does and does not establish.
+ *
+ * "no test in this project can be named for it, so nothing is checking it" was printed whatever the
+ * reason, including when **no run had been kept here at all**. Field-tested on this repository, which
+ * has 1,095 passing tests and a kept history belonging to a different project: all twelve scenarios
+ * of a real plan came back "nothing is checking it". Tewip was not looking at this project's tests,
+ * because it only ever sees the tests a kept run names, and it said so as though it had looked.
+ *
+ * The distinction matters because the two cases call for opposite actions: write a test, or keep
+ * your runs so the ones you have can be seen.
+ */
+function notCoveredBecause(runs, knownTests) {
+    if (runs.length === 0) {
+        return 'no run has been kept for this project, so Tewip has no list of its tests to match against - this is not a claim that nothing checks it. Run `tewip explain <report>` once and ask again';
+    }
+    return `no test among the ${knownTests} that the ${runs.length} kept run(s) name matches it, so nothing Tewip can see is checking it`;
+}
 /** Joins one plan to what the runs actually say. Nothing here is inferred from anything else. */
 export function tracePlan(plan, runs, records) {
     const newest = runs.at(-1);
     const attempted = newest ? Object.keys(newest.attempts ?? {}) : [];
     const everAttempted = new Set(runs.flatMap((r) => Object.keys(r.attempts ?? {})));
+    // A scenario's own id was matched against the **newest run only**, so a test that ran last week
+    // and not today could not be found at all - and the answer for it was "no test in this project can
+    // be named for it, so nothing is checking it", which is a different and much stronger claim than
+    // "it did not run today". The branch below that exists to say exactly that ("ran before but not in
+    // the last run") was unreachable for every scenario the plan did not name a test for.
+    const everyTest = [...everAttempted];
     const failures = new Map();
     for (const record of records)
         if (record.runId === newest?.runId)
             failures.set(record.testId, record);
     const scenarios = plan.scenarios.map((scenario) => {
-        const test = testFor(scenario, attempted);
+        const test = testFor(scenario, everyTest);
         const base = { id: scenario.id, title: scenario.title, priority: scenario.priority ?? '', test, runId: newest?.runId ?? '', when: newest?.finishedAt ?? '' };
         if (!test) {
-            return { ...base, standing: 'not-covered', diagnosis: '', why: 'no test in this project can be named for it, so nothing is checking it' };
+            return { ...base, standing: 'not-covered', diagnosis: '', why: notCoveredBecause(runs, everAttempted.size) };
         }
         if (!everAttempted.has(test)) {
             return { ...base, standing: 'never-run', runId: '', when: '', diagnosis: '', why: `${test} is named as the cover, but no run kept here has attempted it` };

package/dist/engine/work/bug-command.js

@@ -60,8 +60,13 @@ export function whyNothingWasFiled(items, unsure) {
     return [
         `${items.length} failure(s). ${unsure.length} of them Tewip reads as a real bug, and none of those reaches the ${CONFIDENT_ENOUGH.toFixed(2)} confidence it requires before opening a ticket in somebody's name - the highest is ${highest.toFixed(2)}.`,
         byModel > 0
-            ? `${byModel === unsure.length ? 'All' : `${byModel}`} of them came from the AI layer, which is capped at 0.45 on purpose: a model's unsupported word is not enough to file on. A person's correction on the failure in CI, or a rule that agrees, is what would make it enough.`
+            ? `${byModel === unsure.length ? 'All' : `${byModel}`} of them came from the AI layer, which is capped at 0.45 on purpose: a model's unsupported word is not enough to file on.`
             : '',
+        // What a CI user can actually do about it. The rules decline a value mismatch without a stable
+        // failing streak, and on one run there is no streak to read - which is not over-caution:
+        // measured over 2,767 labelled real failures, a value mismatch is TIMING 90% of the time and a
+        // real bug 9% of the time (`node eval/mismatch-precision.ts`). History is what separates them.
+        'What would change this: history. The rules will not call a wrong value a bug without a stable failing streak behind it, because a flaky test produces the same wrong value - on real labelled data a value mismatch is a flaky test nine times out of ten. Keep this project\'s runs (`--name <project>`, or let the Action keep them) and the same failure in a second and third run is what the rules read. A person\'s correction on the failure in CI settles it immediately.',
         `Nothing was filed. These are listed in \`tewip explain\` with their evidence${kinds.length > 0 ? `, alongside ${kinds.join(', ')} failures that are deliberately never filed` : ''}.`,
     ]
         .filter(Boolean)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.69.5",
+  "version": "0.69.7",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [