tewip-ai 0.66.0 → 0.68.0

package/dist/cli/tewip.js

@@ -199,7 +199,7 @@ async function main() {
             if (!question)
                 throw new Error('usage: tewip recall "<an error, or a question>" [--test "<test id>"] [--limit 5]');
             const { runRecall } = await import('../engine/recall/recall-command.js');
-            runRecall({ root: process.cwd(), question, testId: flag(args, 'test'), limit: Number(flag(args, 'limit')) || undefined, name: flag(args, 'name'), out: flag(args, 'out') });
+            await runRecall({ root: process.cwd(), question, testId: flag(args, 'test'), limit: Number(flag(args, 'limit')) || undefined, name: flag(args, 'name'), out: flag(args, 'out') });
             return;
         }
         case 'incident': {

package/dist/engine/learn/learn-command.js

@@ -24,7 +24,13 @@ function gather(root, only) {
 }
 export function runLearn(cmd) {
     const { corrections, records } = gather(cmd.root, cmd.name);
-    const learned = whatWeLearned(corrections, loadVerdicts(cmd.root), records);
+    // The denominator is diagnoses, not kept failures. A store can hold thousands of failures that
+    // nothing has diagnosed yet - the harvest writes exactly that - and counting those as decisions
+    // divides the corrections by a number far larger than the decisions Tewip actually made, making
+    // it look more accurate than it is. Found next to the same mistake in `recall`, which called 1,305
+    // undiagnosed failures "searched".
+    const diagnosed = records.filter((r) => r.diagnosis?.category).length;
+    const learned = whatWeLearned(corrections, loadVerdicts(cmd.root), records, diagnosed);
     const report = renderLearning(learned);
     console.log(report);
     if (cmd.labels) {

package/dist/engine/prove/mutants.js

@@ -96,6 +96,18 @@ function inPattern(line, at) {
  * changes to one function say much less about a suite than five changes in five places.
  */
 export function mutantsFor(text, limit = 6) {
+    return spread(candidateMutants(text), limit);
+}
+/**
+ * Every change this file allows, before any sampling.
+ *
+ * The caller needs this number, not only the sample. `prove engine/verdict/verdict.ts --limit 6`
+ * used to report "6 change(s) were made... 0 went unnoticed", and the file allows 20: eight of the
+ * twelve it never tried were in code the tests had never been asked about. A clean report that
+ * silently covers a third of the file reads exactly like a clean report that covers all of it,
+ * which is the one thing this project will not print.
+ */
+export function candidateMutants(text) {
     const ranges = textRanges(text);
     const lines = text.split(String.fromCharCode(10));
     const offsets = [];
@@ -128,14 +140,14 @@ export function mutantsFor(text, limit = 6) {
             break; // one change per line keeps each answer about one thing
         }
     }
-    return spread(found, limit);
+    return found;
 }
 /**
  * Takes at most `limit`, as far apart in the file as they can be. A suite that catches everything in
  * one function and nothing in the rest of the file should not look like a suite that catches
  * everything, and taking the first six mutants in order is how that happens.
  */
-function spread(all, limit) {
+export function spread(all, limit) {
     if (all.length <= limit)
         return all;
     const step = all.length / limit;

package/dist/engine/prove/prove-command.js

@@ -24,6 +24,36 @@ export function filesToProve(cmd) {
         return [];
     return change.files.map((f) => f.split('\\').join('/')).filter((f) => SOURCE.test(f) && !IS_TEST.test(f) && existsSync(resolve(cmd.root, f)));
 }
+/**
+ * " of the 20" - the part of the headline that keeps a clean answer honest.
+ *
+ * `--limit` samples the file, so "6 change(s) were made... 0 went unnoticed" was true of six lines
+ * and read as true of the file. Found on Tewip's own `engine/verdict/verdict.ts`: a report of six
+ * caught out of six, in a file that allows twenty. Silent when everything the file allows was tried,
+ * because then there is nothing to warn about.
+ */
+function ofHowMany(proofs, tried) {
+    const found = proofs.reduce((total, p) => total + p.found, 0);
+    return found > tried ? ` of the ${found}` : '';
+}
+/**
+ * The two versions of a line, cut so that the difference between them is visible.
+ *
+ * Both used to be truncated at 120 characters from the start, which on a long line put the change
+ * past the cut and printed an identical pair. Found on Tewip's own `verdict.ts:112`, where a real
+ * hole - nothing tests the "and 4 more" count - was reported as `was: X` / `became: X`. A reader
+ * seeing that concludes the tool is broken, and stops reading the findings that are right.
+ */
+export function aroundTheChange(was, now, width = 120) {
+    if (was.length <= width && now.length <= width)
+        return [was, now];
+    let at = 0;
+    while (at < was.length && at < now.length && was[at] === now[at])
+        at += 1;
+    const from = Math.max(0, at - Math.floor(width / 3));
+    const cut = (text) => (from > 0 ? '...' : '') + text.slice(from, from + width) + (from + width < text.length ? '...' : '');
+    return [cut(was), cut(now)];
+}
 export function renderProof(proofs) {
     const tried = proofs.flatMap((p) => p.tried);
     const unnoticed = tried.filter((t) => t.fate === 'unnoticed');
@@ -36,7 +66,7 @@ export function renderProof(proofs) {
         '',
         tried.length === 0
             ? 'Nothing was tried. See below for why.'
-            : `${tried.length} change(s) were made to your code and the covering tests were run against each. **${unnoticed.length} went unnoticed.** ${noticed.length} were caught. Took ${seconds}s.`,
+            : `${tried.length}${ofHowMany(proofs, tried.length)} change(s) were made to your code and the covering tests were run against each. **${unnoticed.length} went unnoticed.** ${noticed.length} were caught. Took ${seconds}s.`,
         '',
     ];
     if (unnoticed.length > 0) {
@@ -47,9 +77,10 @@ export function renderProof(proofs) {
                 continue;
             lines.push(`### ${proof.file}`, '');
             for (const gap of gaps) {
+                const [was, now] = aroundTheChange(gap.mutant.was, gap.mutant.now);
                 lines.push(`- **line ${gap.mutant.line}: ${gap.mutant.kind}**`);
-                lines.push(`  - was: \`${gap.mutant.was.slice(0, 120)}\``);
-                lines.push(`  - became: \`${gap.mutant.now.slice(0, 120)}\``);
+                lines.push(`  - was: \`${was}\``);
+                lines.push(`  - became: \`${now}\``);
             }
             lines.push('', `Run by: ${proof.covering.join(', ')}`, '');
         }
@@ -67,7 +98,8 @@ export function renderProof(proofs) {
             lines.push(`- **${p.file}**: ${p.blocked}`);
         lines.push('');
     }
-    if (invalid.length > 0 || broke.length > 0 || proofs.some((p) => p.notTried > 0 || p.setAside.length > 0)) {
+    const neverSampled = proofs.filter((p) => p.found > p.tried.length + p.notTried);
+    if (invalid.length > 0 || broke.length > 0 || neverSampled.length > 0 || proofs.some((p) => p.notTried > 0 || p.setAside.length > 0)) {
         lines.push('## Not counted', '');
         if (invalid.length > 0)
             lines.push(`- ${invalid.length} change(s) did not compile, so they say nothing about the tests. That is Tewip's fault, not yours.`);
@@ -79,6 +111,9 @@ export function renderProof(proofs) {
         const left = proofs.reduce((total, p) => total + p.notTried, 0);
         if (left > 0)
             lines.push(`- ${left} change(s) were never tried: the time budget ran out. Raise it with \`--budget <seconds>\`.`);
+        for (const p of neverSampled) {
+            lines.push(`- **${p.file}** allows ${p.found} change(s) and ${p.tried.length + p.notTried} were taken, spread across the file. Nothing above says anything about the other ${p.found - p.tried.length - p.notTried}. To ask about all of them: \`--limit ${p.found}\`.`);
+        }
         lines.push('');
     }
     lines.push('---', '', 'Every line above was proved by running your own tests against your own code, one small change at', 'a time, and every file was put back exactly as it was. No model was called.', '', 'What this does not say: that the unnoticed changes are bugs, or that the caught ones mean the', 'suite is good. It says only which of these particular mistakes your tests would have stopped -', 'which is more than a coverage percentage can tell you, and less than a proof that the code works.');
@@ -121,7 +156,7 @@ export function runProve(cmd) {
     const proofs = [];
     for (const file of files) {
         if (!existsSync(resolve(cmd.root, file))) {
-            proofs.push({ file, covering: [], tried: [], blocked: 'no such file here', setAside: [], notTried: 0, seconds: 0 });
+            proofs.push({ file, covering: [], tried: [], blocked: 'no such file here', setAside: [], notTried: 0, found: 0, seconds: 0 });
             continue;
         }
         // Nothing importing it by name is the normal case in a layered codebase, not the absence of

package/dist/engine/prove/prove.js

@@ -13,7 +13,7 @@ import { spawnSync } from 'node:child_process';
 import { readFileSync, writeFileSync } from 'node:fs';
 import { join, relative, resolve } from 'node:path';
 import { runTest } from '../quality/write-verify.js';
-import { applyMutant, mutantsFor } from './mutants.js';
+import { applyMutant, candidateMutants, spread } from './mutants.js';
 /**
  * Does this mutated file still parse?
  *
@@ -74,14 +74,18 @@ export function proveFile(root, file, covering, framework, opts = {}) {
     const runMs = (opts.runSeconds ?? 180) * 1000;
     const say = opts.onStep ?? (() => { });
     const full = resolve(root, file);
-    const proof = { file, covering, tried: [], blocked: '', setAside: [], notTried: 0, seconds: 0 };
+    const proof = { file, covering, tried: [], blocked: '', setAside: [], notTried: 0, found: 0, seconds: 0 };
     if (covering.length === 0) {
         proof.blocked = 'no test imports this file, so no test could notice any change to it. That is the finding: this code is not covered by anything Tewip can name.';
         return finish(proof, started);
     }
     const original = readFileSync(full, 'utf8');
     const stopGuarding = guardAgainstBeingKilled(full, original);
-    const mutants = mutantsFor(original, limit);
+    // The whole set first, then the sample: `found` is what the file allows and `mutants` is what
+    // there was time to ask about, and the report has to be able to tell the reader the difference.
+    const candidates = candidateMutants(original);
+    const mutants = spread(candidates, limit);
+    proof.found = candidates.length;
     if (mutants.length === 0) {
         proof.blocked = 'nothing in this file could be changed in a way a test should be expected to catch - no comparison, no condition, no returned value outside of strings and comments';
         return finish(proof, started, stopGuarding);

package/dist/engine/recall/recall-command.js

@@ -1,43 +1,82 @@
 // `tewip recall "<an error, or a question>"`: what this project already concluded.
 import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
-import { datasetFiles, readTriageRecords } from '../../reporters/store.js';
+import { buildItems } from '../../outlets/items.js';
+import { datasetFiles, readRunSummaries, readTriageRecords } from '../../reporters/store.js';
 import { loadCorrections } from '../corrections.js';
+import { HistoryIndex } from '../history.js';
 import { memories, recall, renderRecall } from './recall.js';
 /** Everything kept for this project: its failures, and the times a person overruled Tewip. */
 function kept(root, only) {
     const dir = join(root, '.tewip', 'history');
     if (!existsSync(dir))
-        return { records: [], corrections: [] };
+        return { records: [], runs: [], corrections: [] };
     const records = [];
+    const runs = [];
     const corrections = [];
     for (const project of only ? [only] : readdirSync(dir)) {
         const where = join(dir, project);
         if (!existsSync(where))
             continue;
-        records.push(...readTriageRecords(datasetFiles(where).triage));
+        const files = datasetFiles(where);
+        records.push(...readTriageRecords(files.triage));
+        runs.push(...readRunSummaries(files.runs));
         corrections.push(...loadCorrections(where));
     }
-    return { records, corrections };
+    return { records, runs, corrections };
+}
+/**
+ * The kept failures with a category on each, diagnosing the ones that do not have one stored.
+ *
+ * This is the whole bug Phase 59 was looking for. `memories()` keeps only records that carry a
+ * `diagnosis`, and **nothing on the collection path writes one**: the reporter and the harvest both
+ * store the raw failure, and the category is worked out when a report is rendered. So on a real
+ * 1,305-failure history, `recall` had nothing to search - and said "Nothing in this project's
+ * history resembles it... this is new here", footed by "Searched 1305 kept failure(s)". Both halves
+ * false: nothing was searched, and 660 of those 1,305 failures were the timeout being asked about.
+ *
+ * `note`, `flaky` and the risk digest had all been diagnosing on read for months. Recall was the
+ * one retrieval layer that demanded a field nobody writes, which made it permanently empty on every
+ * project that had not been through `tewip explain` - and empty in a way that read like an answer.
+ *
+ * Rules only, never a model: `recall`'s promise is no model, no network, nothing to set up, and a
+ * question asked of local history must not start costing money because of how it is indexed.
+ */
+async function diagnosed(records, runs, corrections) {
+    if (records.length === 0)
+        return [];
+    const items = await buildItems(records, new HistoryIndex(records, runs), { dataset: 'recall', llm: null, reuseStoredDiagnoses: true, corrections });
+    const byCase = new Map(items.map((item) => [`${item.runId}::${item.testId}`, item.diagnosis]));
+    return records.map((record) => (record.diagnosis ? record : { ...record, diagnosis: byCase.get(`${record.runId}::${record.testId}`) }));
 }
 /**
  * The report, without printing it. An MCP server speaks JSON-RPC over stdout, so anything one of
  * its tools calls has to return its answer: a stray console.log corrupts every response the
  * assistant receives, and the corruption looks like the tool being broken rather than noisy.
  */
-export function recallReport(cmd) {
-    const { records, corrections } = kept(cmd.root, cmd.name);
+export async function recallReport(cmd) {
+    const { records, runs, corrections } = kept(cmd.root, cmd.name);
     if (records.length === 0) {
         return { report: 'Nothing has been kept here yet, so there is no history to search. Run `tewip explain <report>` first.', searched: 0, corrections: 0 };
     }
-    const found = recall(cmd.question, memories(records, corrections), { testId: cmd.testId, limit: cmd.limit ?? 5 });
-    return { report: renderRecall(cmd.question, found), searched: records.length, corrections: corrections.length };
+    // The count reported is the count searched. It used to be `records.length` whatever `memories`
+    // actually indexed, which is how an empty index came to be described as 1,305 failures searched.
+    const mem = memories(await diagnosed(records, runs, corrections), corrections);
+    if (mem.length === 0) {
+        return {
+            report: `${records.length} failure(s) are kept here, but none of them could be given a category, so there is nothing to compare this against. Recall searches what Tewip concluded about a failure, and a failure it cannot read is not a conclusion.`,
+            searched: 0,
+            corrections: corrections.length,
+        };
+    }
+    const found = recall(cmd.question, mem, { testId: cmd.testId, limit: cmd.limit ?? 5 });
+    return { report: renderRecall(cmd.question, found, mem.length), searched: mem.length, corrections: corrections.length };
 }
-export function runRecall(cmd) {
-    const { report, searched, corrections } = recallReport(cmd);
+export async function runRecall(cmd) {
+    const { report, searched, corrections } = await recallReport(cmd);
     console.log(report);
     if (searched > 0)
-        console.error(`\nSearched ${searched} kept failure(s) and ${corrections} correction(s). No model was called.`);
+        console.error(`\nSearched ${searched} diagnosed failure(s) and ${corrections} correction(s). No model was called.`);
     if (cmd.out) {
         mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
         writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');

package/dist/engine/recall/recall.js

@@ -88,10 +88,20 @@ export function recall(question, all, { testId, limit = 5 } = {}) {
         .slice(0, limit);
 }
 /** What a person reads. Every line names the run it came from, so nothing has to be believed. */
-export function renderRecall(question, found) {
+/**
+ * What a person reads.
+ *
+ * `searched` is not decoration. "Nothing in this project's history resembles it" is only an answer
+ * if something was actually compared against, and on a real 1,305-failure history this said exactly
+ * that while the searchable set was **empty** - see `recallReport`. The number belongs in the report
+ * body rather than only on stderr, because an MCP client and a `> file` both keep the body and throw
+ * stderr away, which is precisely where a reader would have caught it.
+ */
+export function renderRecall(question, found, searched) {
     const lines = [`# What this project already knows`, '', `Asked: ${question.replace(/\s+/g, ' ').slice(0, 160)}`, ''];
     if (found.length === 0) {
-        lines.push('Nothing in this project\'s history resembles it. That is an answer, not a failure: it means', 'this is new here, and there is no earlier decision to follow.');
+        const against = searched === undefined ? 'this project\'s history' : `the ${searched} diagnosed failure(s) kept here`;
+        lines.push(`Nothing in ${against} resembles it. That is an answer, not a failure: it means`, 'this is new here, and there is no earlier decision to follow.');
         return lines.join('\n');
     }
     const corrected = found.filter((f) => f.correctedBy);

package/dist/mcp/server.js

@@ -111,7 +111,7 @@ server.registerTool('recall', {
     if (!isAbsolute(root) || !existsSync(root))
         throw new Error(`project must be an existing absolute path, got "${root}"`);
     const { recallReport } = await import('../engine/recall/recall-command.js');
-    return recallReport({ root, question, testId: test }).report;
+    return (await recallReport({ root, question, testId: test })).report;
 }));
 server.registerTool('trace_requirements', {
     title: 'Is this requirement covered, and is it passing?',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.66.0",
+  "version": "0.68.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [