tewip-ai 0.69.3 → 0.69.5

package/dist/engine/perf/compare.js

@@ -17,6 +17,7 @@
 // and deliberately conservative: it is better to say "cannot say" to a real 4% regression than to
 // cry wolf at noise, because the second one costs the team the tool.
 import { MAIN_TIMING } from './k6.js';
+import { show } from './run.js';
 /** The fewest baseline runs from which run-to-run noise can be estimated at all. */
 export const ENOUGH_BASELINES = 3;
 /**
@@ -25,6 +26,22 @@ export const ENOUGH_BASELINES = 3;
  * worth a person's afternoon.
  */
 const SMALLEST_CHANGE = 0.05;
+/**
+ * The smallest difference in milliseconds this will call a regression, whatever the percentage.
+ *
+ * Found by running the same unchanged page four times through real Lighthouse and comparing the
+ * fourth against the other three: **"1 metric is slower than the baseline runs can explain."** The
+ * metric was `server-response-time`, the difference was **2.8ms on an 11ms baseline** - +25%, which
+ * clears the 5% floor easily - and Lighthouse itself reports all four of those runs as *"Root
+ * document took 10 ms"*. Tewip was claiming a regression at a precision finer than the tool it read
+ * reports, on a page that had not changed.
+ *
+ * The number is not derived from theory and is not pretending to be: it sits between a false
+ * positive this was measured producing (2.8ms) and the smallest true positive the suite already
+ * requires it to catch (10ms on a 50ms baseline). A relative floor cannot separate those two - both
+ * are about +20% - which is why the floor here is absolute.
+ */
+const MS_NOISE_FLOOR = 5;
 const median = (values) => {
     const sorted = [...values].sort((a, b) => a - b);
     const mid = Math.floor(sorted.length / 2);
@@ -37,13 +54,19 @@ const round = (n) => Math.round(n * 10) / 10;
  * `baseline` is every stored value for the metric, oldest first. It is not averaged into a single
  * number, because its spread is the only evidence available about what noise looks like here.
  */
-export function compareMetric(metric, now, baseline) {
+export function compareMetric(metric, now, baseline, unit = 'ms') {
     const runs = baseline.length;
+    // Every number in `why` goes through the same formatter the rest of the report uses. These
+    // sentences used to end "ms" whatever the metric was, so a Lighthouse score of 100 was explained
+    // as **"1ms is within what 3 baseline runs already did on their own (1-1ms)"** - on the line
+    // directly below one reading `performance-score 100`. Two different numbers and a unit that does
+    // not exist, for the same measurement, in the same bullet.
+    const u = (n) => show({ value: n, unit, rating: null });
     const before = runs > 0 ? median(baseline) : 0;
     const low = runs > 0 ? Math.min(...baseline) : 0;
     const high = runs > 0 ? Math.max(...baseline) : 0;
     const spread = high - low;
-    const margin = Math.max(spread, before * SMALLEST_CHANGE);
+    const margin = Math.max(spread, before * SMALLEST_CHANGE, unit === 'ms' ? MS_NOISE_FLOOR : 0);
     const base = {
         metric,
         now: round(now),
@@ -68,7 +91,7 @@ export function compareMetric(metric, now, baseline) {
         return {
             ...base,
             direction: 'slower',
-            why: `${round(now)}ms is outside everything ${runs} baseline runs produced (${round(low)}-${round(high)}ms) ` +
+            why: `${u(now)} is outside everything ${runs} baseline runs produced (${u(low)}-${u(high)}) ` +
                 `by more than their own spread, so the runs cannot explain it. That is ${percent > 0 ? '+' : ''}${percent}% on the middle run.`,
         };
     }
@@ -76,14 +99,14 @@ export function compareMetric(metric, now, baseline) {
         return {
             ...base,
             direction: 'faster',
-            why: `${round(now)}ms is below everything ${runs} baseline runs produced (${round(low)}-${round(high)}ms) by more than their own spread.`,
+            why: `${u(now)} is below everything ${runs} baseline runs produced (${u(low)}-${u(high)}) by more than their own spread.`,
         };
     }
     return {
         ...base,
         direction: 'cannot-say',
-        why: `${round(now)}ms is within what ${runs} baseline runs already did on their own (${round(low)}-${round(high)}ms, ` +
-            `allowing ${round(margin)}ms either side). A difference the baseline runs produce by themselves is not evidence about this change.`,
+        why: `${u(now)} is within what ${runs} baseline runs already did on their own (${u(low)}-${u(high)}, ` +
+            `allowing ${u(margin)} either side). A difference the baseline runs produce by themselves is not evidence about this change.`,
     };
 }
 /** Enough requests for a percentile to mean anything. Below this, p95 is three data points. */

package/dist/engine/perf/perf-command.js

@@ -92,10 +92,18 @@ export function renderPerf(run, comparisons, label, baselineCount, recorded, man
     }
     for (const c of [...slower, ...faster, ...unsure]) {
         const verdict = c.direction === 'slower' ? '**slower**' : c.direction === 'faster' ? 'faster' : 'cannot say';
+        // `show` and nothing else. This had a local formatter - `unit === 'ms' ? n + 'ms' : n` - which
+        // knew nothing about Lighthouse's 0-1 score, so one real report said **performance-score: 100**
+        // in the section above and **performance-score 1** here. Anyone reading the second number takes
+        // it for 1 out of 100: a perfect page reported as a catastrophic one, in the same report.
+        // `server-response-time` came out as 13ms above and 12.6ms here for the same reason.
         const unit = run.metrics[c.metric]?.unit ?? 'ms';
-        const u = (n) => (unit === 'ms' ? `${n}ms` : `${n}`);
+        const u = (n) => show({ ...(run.metrics[c.metric] ?? { unit, rating: null }), value: n });
         lines.push(`- \`${c.metric}\` ${u(c.now)} — ${verdict}`);
-        lines.push(`  - baseline ${u(c.baselineLow)}–${u(c.baselineHigh)} over ${c.baselineRuns} runs, middle ${u(c.before)}`);
+        // With no baseline there is no range, and "baseline 0ms-0ms over 0 runs, middle 0ms" is four
+        // zeros that read as measurements. The line below already says there is nothing to compare to.
+        if (c.baselineRuns > 0)
+            lines.push(`  - baseline ${u(c.baselineLow)}–${u(c.baselineHigh)} over ${c.baselineRuns} runs, middle ${u(c.before)}`);
         lines.push(`  - ${c.why}`);
     }
     lines.push('');
@@ -174,7 +182,7 @@ export function runPerf(cmd) {
         }
     }
     const comparisons = metricsInOrder(run)
-        .map((metric) => compareMetric(metric, run.metrics[metric].value, baselineFor(history, metric)))
+        .map((metric) => compareMetric(metric, run.metrics[metric].value, baselineFor(history, metric), run.metrics[metric].unit))
         // A metric that is zero now and was zero every time before says nothing, and printing
         // "0ms is within 0-0ms" three times buries the real lines.
         .filter((c) => c.now > 0 || c.baselineHigh > 0);

package/dist/engine/work/bug-command.js

@@ -19,6 +19,54 @@ import { TEWIP_LABEL } from './jira-command.js';
 import { trackerFromEnv, trackerHelp } from './trackers.js';
 /** Bugs Tewip filed carry this as well, so a second run finds them and does not file again. */
 export const BUG_LABEL = 'tewip-bug';
+/**
+ * How sure Tewip has to be before it puts a person's name on a ticket.
+ *
+ * Named and exported because of what it collides with. Phase 55 capped a model-only diagnosis at
+ * **0.45** - the model is not reliable enough for its word alone to be acted on - and this bar has
+ * been 0.8 since Phase 10. The two numbers were set eight weeks apart and never compared, and
+ * together they mean **a bug only the model recognised can never be filed**, whatever it says.
+ *
+ * Measured over 1,964 real failures from five public projects (gutenberg, supabase, lxdui, penpot,
+ * jupyterlab) with the rules alone: **20 REAL_BUG verdicts, of which 1 reaches 0.8.** Four of the
+ * five projects produce none at all. On a real jest report of a plain value mismatch - "Expected: 4,
+ * Received: 3", which is as clear as a real bug gets - the verdict is REAL_BUG at 0.45 from the
+ * model, and `bugs` files nothing.
+ *
+ * The bar is not lowered here. A ticket opened on a model's unsupported word is the thing this
+ * project exists not to do, and 0.45 is the honest reading of how often that word is right. What
+ * changes is that the refusal now says which of the two reasons applies, because "none of them is a
+ * real bug Tewip is confident about" was being printed for twenty failures Tewip had called real
+ * bugs.
+ */
+export const CONFIDENT_ENOUGH = 0.8;
+/**
+ * Why nothing was filed - the two cases told apart.
+ *
+ * The old sentence was one line for both: `${items.length} failure(s), and none of them is a real
+ * bug Tewip is confident about (${kinds})`, where `kinds` was every category present. On a real
+ * report it printed **"none of them is a real bug Tewip is confident about (REAL_BUG, ENV)"** -
+ * listing REAL_BUG inside the sentence denying there was one. A reader cannot tell from that whether
+ * Tewip found nothing or found something it would not act on, and those call for opposite next steps.
+ */
+export function whyNothingWasFiled(items, unsure) {
+    const kinds = [...new Set(items.filter((i) => i.diagnosis.category !== 'REAL_BUG').map((i) => i.diagnosis.category))];
+    if (unsure.length === 0) {
+        return (`${items.length} failure(s), and none of them looks like the application being wrong` +
+            `${kinds.length > 0 ? ` - they are ${kinds.join(', ')}` : ''}. Nothing was filed, which is the point.`);
+    }
+    const highest = Math.max(...unsure.map((i) => i.diagnosis.confidence));
+    const byModel = unsure.filter((i) => /AI|gpt|claude|model/i.test(i.source)).length;
+    return [
+        `${items.length} failure(s). ${unsure.length} of them Tewip reads as a real bug, and none of those reaches the ${CONFIDENT_ENOUGH.toFixed(2)} confidence it requires before opening a ticket in somebody's name - the highest is ${highest.toFixed(2)}.`,
+        byModel > 0
+            ? `${byModel === unsure.length ? 'All' : `${byModel}`} of them came from the AI layer, which is capped at 0.45 on purpose: a model's unsupported word is not enough to file on. A person's correction on the failure in CI, or a rule that agrees, is what would make it enough.`
+            : '',
+        `Nothing was filed. These are listed in \`tewip explain\` with their evidence${kinds.length > 0 ? `, alongside ${kinds.join(', ')} failures that are deliberately never filed` : ''}.`,
+    ]
+        .filter(Boolean)
+        .join('\n');
+}
 const isBlobDir = (path) => statSync(path).isDirectory() && readdirSync(path).some((f) => /^report.*\.zip$/.test(f));
 /** The bug report a developer actually wants: what failed, what it did, and why Tewip believes it. */
 export function bugBody(item, runUrl) {
@@ -76,15 +124,15 @@ export async function runBugs(cmd) {
     const items = await buildItems(imported.records, index, { dataset: `local: ${name}`, llm: llmFromEnv(), corrections: loadCorrections(historyDir) });
     // Only what Tewip itself calls a bug, and only when it is confident enough to put a person's name
     // on it. Everything else is deliberately left out.
-    const bugs = items.filter((i) => i.diagnosis.category === 'REAL_BUG' && i.diagnosis.confidence >= 0.8);
+    const bugs = items.filter((i) => i.diagnosis.category === 'REAL_BUG' && i.diagnosis.confidence >= CONFIDENT_ENOUGH);
+    const unsure = items.filter((i) => i.diagnosis.category === 'REAL_BUG' && i.diagnosis.confidence < CONFIDENT_ENOUGH);
     const lines = [`Report: ${cmd.report}`, `Project: ${cmd.projectKey}`, ''];
     if (items.length === 0) {
         console.log('Every test in that report passed. Nothing to file.');
         return [];
     }
     if (bugs.length === 0) {
-        const kinds = [...new Set(items.map((i) => i.diagnosis.category))].join(', ');
-        console.log(`${items.length} failure(s), and none of them is a real bug Tewip is confident about (${kinds}). Nothing was filed, which is the point.`);
+        console.log(whyNothingWasFiled(items, unsure));
         return [];
     }
     const tracker = chosen?.tracker.canWrite() ? chosen.tracker : null;
@@ -122,8 +170,17 @@ export async function runBugs(cmd) {
         lines.push(`  filed ${key}: ${summary}`);
         filed++;
     }
-    const left = items.length - bugs.length;
-    lines.push('', `${bugs.length} of ${items.length} failure(s) look like real bugs; ${left} were flaky tests, environment or test data and were deliberately not filed.`);
+    // `items.length - bugs.length` was described as "flaky tests, environment or test data", which is
+    // untrue of a REAL_BUG sitting below the bar: a bug Tewip is 0.45 sure of was being counted, in
+    // print, as a flaky test. And the `max` cap was silent - the same mistake `prove` made in Phase 58,
+    // where a sampled answer read as a whole one.
+    const notBugs = items.length - bugs.length - unsure.length;
+    const beyondCap = Math.max(0, bugs.length - cmd.max);
+    lines.push('', `${bugs.length} of ${items.length} failure(s) look like real bugs.` +
+        (notBugs > 0 ? ` ${notBugs} were flaky tests, environment or test data and were deliberately not filed.` : '') +
+        (unsure.length > 0 ? ` ${unsure.length} more look like real bugs but below the ${CONFIDENT_ENOUGH.toFixed(2)} confidence needed to file one, so they were not.` : ''));
+    if (beyondCap > 0)
+        lines.push(`${beyondCap} were past the --max ${cmd.max} limit and were not looked at. Raise it with \`--max ${bugs.length}\`.`);
     lines.push(cmd.create ? `${filed} ticket(s) created in ${tracker?.name}. Tewip never moves, assigns or closes one.` : 'Nothing was created. Add --create to file them.');
     console.log(lines.join('\n'));
     return bugs;

package/dist/mcp/tools-prove.js

@@ -118,7 +118,7 @@ export function registerProofTools(server, { answer, readOnly, project }) {
         const name = label ?? 'default';
         const history = fromSameTool(storedRuns(root, name), run.source);
         const comparisons = metricsInOrder(run)
-            .map((m) => compareMetric(m, run.metrics[m].value, baselineFor(history, m)))
+            .map((m) => compareMetric(m, run.metrics[m].value, baselineFor(history, m), run.metrics[m].unit))
             .filter((c) => c.now > 0 || c.baselineHigh > 0);
         return renderPerf(run, comparisons, name, history.length, false, manual);
     }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.69.3",
+  "version": "0.69.5",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [