tewip-ai 0.66.0 → 0.67.0

package/dist/engine/prove/mutants.js

@@ -96,6 +96,18 @@ function inPattern(line, at) {
  * changes to one function say much less about a suite than five changes in five places.
  */
 export function mutantsFor(text, limit = 6) {
+    return spread(candidateMutants(text), limit);
+}
+/**
+ * Every change this file allows, before any sampling.
+ *
+ * The caller needs this number, not only the sample. `prove engine/verdict/verdict.ts --limit 6`
+ * used to report "6 change(s) were made... 0 went unnoticed", and the file allows 20: eight of the
+ * twelve it never tried were in code the tests had never been asked about. A clean report that
+ * silently covers a third of the file reads exactly like a clean report that covers all of it,
+ * which is the one thing this project will not print.
+ */
+export function candidateMutants(text) {
     const ranges = textRanges(text);
     const lines = text.split(String.fromCharCode(10));
     const offsets = [];
@@ -128,14 +140,14 @@ export function mutantsFor(text, limit = 6) {
             break; // one change per line keeps each answer about one thing
         }
     }
-    return spread(found, limit);
+    return found;
 }
 /**
  * Takes at most `limit`, as far apart in the file as they can be. A suite that catches everything in
  * one function and nothing in the rest of the file should not look like a suite that catches
  * everything, and taking the first six mutants in order is how that happens.
  */
-function spread(all, limit) {
+export function spread(all, limit) {
     if (all.length <= limit)
         return all;
     const step = all.length / limit;

package/dist/engine/prove/prove-command.js

@@ -24,6 +24,36 @@ export function filesToProve(cmd) {
         return [];
     return change.files.map((f) => f.split('\\').join('/')).filter((f) => SOURCE.test(f) && !IS_TEST.test(f) && existsSync(resolve(cmd.root, f)));
 }
+/**
+ * " of the 20" - the part of the headline that keeps a clean answer honest.
+ *
+ * `--limit` samples the file, so "6 change(s) were made... 0 went unnoticed" was true of six lines
+ * and read as true of the file. Found on Tewip's own `engine/verdict/verdict.ts`: a report of six
+ * caught out of six, in a file that allows twenty. Silent when everything the file allows was tried,
+ * because then there is nothing to warn about.
+ */
+function ofHowMany(proofs, tried) {
+    const found = proofs.reduce((total, p) => total + p.found, 0);
+    return found > tried ? ` of the ${found}` : '';
+}
+/**
+ * The two versions of a line, cut so that the difference between them is visible.
+ *
+ * Both used to be truncated at 120 characters from the start, which on a long line put the change
+ * past the cut and printed an identical pair. Found on Tewip's own `verdict.ts:112`, where a real
+ * hole - nothing tests the "and 4 more" count - was reported as `was: X` / `became: X`. A reader
+ * seeing that concludes the tool is broken, and stops reading the findings that are right.
+ */
+export function aroundTheChange(was, now, width = 120) {
+    if (was.length <= width && now.length <= width)
+        return [was, now];
+    let at = 0;
+    while (at < was.length && at < now.length && was[at] === now[at])
+        at += 1;
+    const from = Math.max(0, at - Math.floor(width / 3));
+    const cut = (text) => (from > 0 ? '...' : '') + text.slice(from, from + width) + (from + width < text.length ? '...' : '');
+    return [cut(was), cut(now)];
+}
 export function renderProof(proofs) {
     const tried = proofs.flatMap((p) => p.tried);
     const unnoticed = tried.filter((t) => t.fate === 'unnoticed');
@@ -36,7 +66,7 @@ export function renderProof(proofs) {
         '',
         tried.length === 0
             ? 'Nothing was tried. See below for why.'
-            : `${tried.length} change(s) were made to your code and the covering tests were run against each. **${unnoticed.length} went unnoticed.** ${noticed.length} were caught. Took ${seconds}s.`,
+            : `${tried.length}${ofHowMany(proofs, tried.length)} change(s) were made to your code and the covering tests were run against each. **${unnoticed.length} went unnoticed.** ${noticed.length} were caught. Took ${seconds}s.`,
         '',
     ];
     if (unnoticed.length > 0) {
@@ -47,9 +77,10 @@ export function renderProof(proofs) {
                 continue;
             lines.push(`### ${proof.file}`, '');
             for (const gap of gaps) {
+                const [was, now] = aroundTheChange(gap.mutant.was, gap.mutant.now);
                 lines.push(`- **line ${gap.mutant.line}: ${gap.mutant.kind}**`);
-                lines.push(`  - was: \`${gap.mutant.was.slice(0, 120)}\``);
-                lines.push(`  - became: \`${gap.mutant.now.slice(0, 120)}\``);
+                lines.push(`  - was: \`${was}\``);
+                lines.push(`  - became: \`${now}\``);
             }
             lines.push('', `Run by: ${proof.covering.join(', ')}`, '');
         }
@@ -67,7 +98,8 @@ export function renderProof(proofs) {
             lines.push(`- **${p.file}**: ${p.blocked}`);
         lines.push('');
     }
-    if (invalid.length > 0 || broke.length > 0 || proofs.some((p) => p.notTried > 0 || p.setAside.length > 0)) {
+    const neverSampled = proofs.filter((p) => p.found > p.tried.length + p.notTried);
+    if (invalid.length > 0 || broke.length > 0 || neverSampled.length > 0 || proofs.some((p) => p.notTried > 0 || p.setAside.length > 0)) {
         lines.push('## Not counted', '');
         if (invalid.length > 0)
             lines.push(`- ${invalid.length} change(s) did not compile, so they say nothing about the tests. That is Tewip's fault, not yours.`);
@@ -79,6 +111,9 @@ export function renderProof(proofs) {
         const left = proofs.reduce((total, p) => total + p.notTried, 0);
         if (left > 0)
             lines.push(`- ${left} change(s) were never tried: the time budget ran out. Raise it with \`--budget <seconds>\`.`);
+        for (const p of neverSampled) {
+            lines.push(`- **${p.file}** allows ${p.found} change(s) and ${p.tried.length + p.notTried} were taken, spread across the file. Nothing above says anything about the other ${p.found - p.tried.length - p.notTried}. To ask about all of them: \`--limit ${p.found}\`.`);
+        }
         lines.push('');
     }
     lines.push('---', '', 'Every line above was proved by running your own tests against your own code, one small change at', 'a time, and every file was put back exactly as it was. No model was called.', '', 'What this does not say: that the unnoticed changes are bugs, or that the caught ones mean the', 'suite is good. It says only which of these particular mistakes your tests would have stopped -', 'which is more than a coverage percentage can tell you, and less than a proof that the code works.');
@@ -121,7 +156,7 @@ export function runProve(cmd) {
     const proofs = [];
     for (const file of files) {
         if (!existsSync(resolve(cmd.root, file))) {
-            proofs.push({ file, covering: [], tried: [], blocked: 'no such file here', setAside: [], notTried: 0, seconds: 0 });
+            proofs.push({ file, covering: [], tried: [], blocked: 'no such file here', setAside: [], notTried: 0, found: 0, seconds: 0 });
             continue;
         }
         // Nothing importing it by name is the normal case in a layered codebase, not the absence of

package/dist/engine/prove/prove.js

@@ -13,7 +13,7 @@ import { spawnSync } from 'node:child_process';
 import { readFileSync, writeFileSync } from 'node:fs';
 import { join, relative, resolve } from 'node:path';
 import { runTest } from '../quality/write-verify.js';
-import { applyMutant, mutantsFor } from './mutants.js';
+import { applyMutant, candidateMutants, spread } from './mutants.js';
 /**
  * Does this mutated file still parse?
  *
@@ -74,14 +74,18 @@ export function proveFile(root, file, covering, framework, opts = {}) {
     const runMs = (opts.runSeconds ?? 180) * 1000;
     const say = opts.onStep ?? (() => { });
     const full = resolve(root, file);
-    const proof = { file, covering, tried: [], blocked: '', setAside: [], notTried: 0, seconds: 0 };
+    const proof = { file, covering, tried: [], blocked: '', setAside: [], notTried: 0, found: 0, seconds: 0 };
     if (covering.length === 0) {
         proof.blocked = 'no test imports this file, so no test could notice any change to it. That is the finding: this code is not covered by anything Tewip can name.';
         return finish(proof, started);
     }
     const original = readFileSync(full, 'utf8');
     const stopGuarding = guardAgainstBeingKilled(full, original);
-    const mutants = mutantsFor(original, limit);
+    // The whole set first, then the sample: `found` is what the file allows and `mutants` is what
+    // there was time to ask about, and the report has to be able to tell the reader the difference.
+    const candidates = candidateMutants(original);
+    const mutants = spread(candidates, limit);
+    proof.found = candidates.length;
     if (mutants.length === 0) {
         proof.blocked = 'nothing in this file could be changed in a way a test should be expected to catch - no comparison, no condition, no returned value outside of strings and comments';
         return finish(proof, started, stopGuarding);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.66.0",
+  "version": "0.67.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [