tewip-ai 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -2
- package/dist/cli/help.js +18 -1
- package/dist/cli/tewip.js +20 -0
- package/dist/engine/attest/attest.js +5 -2
- package/dist/engine/prove/mutants.js +148 -0
- package/dist/engine/prove/prove-command.js +152 -0
- package/dist/engine/prove/prove.js +179 -0
- package/dist/engine/prove/reaches.js +102 -0
- package/dist/engine/pursue/act.js +1 -0
- package/dist/engine/pursue/policy.js +14 -0
- package/dist/engine/pursue/pursue-command.js +23 -1
- package/dist/engine/quality/frameworks.js +33 -0
- package/dist/engine/quality/write-verify.js +12 -1
- package/dist/engine/review/coverage.js +27 -4
- package/dist/engine/review/review-command.js +5 -1
- package/dist/engine/vouch/assertions.js +138 -0
- package/dist/engine/vouch/changed.js +58 -0
- package/dist/engine/vouch/run-vouch.js +35 -0
- package/dist/engine/vouch/tests-in.js +275 -0
- package/dist/engine/vouch/vouch-command.js +135 -0
- package/dist/engine/vouch/vouch.js +193 -0
- package/dist/mcp/server.js +32 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -71,6 +71,13 @@ On every run, the action:
|
|
|
71
71
|
- **Developers:** posts one diagnosis comment on the pull request and keeps it updated. For each failure it says whether the change likely caused it, why, and what to do next.
|
|
72
72
|
- **SDETs:** writes the triage board into the job summary.
|
|
73
73
|
- **DevOps:** collects environment failures (server errors, connection resets, outages) in **one** GitHub issue, and each new run adds a single grouped comment to it.
|
|
74
|
+
- **The tests in the pull request:** on a PR it reads the test files the change touched and says which
|
|
75
|
+
of them could never fail - an unfinished expectation, a value compared with itself, a test that
|
|
76
|
+
discards its own failure, one that never runs. It says nothing when they are all sound, and it says
|
|
77
|
+
it **even on a green run**, because "all tests passed" is the exact sentence that needs qualifying
|
|
78
|
+
when one of them cannot fail. Needs `fetch-depth: 0` on `actions/checkout` so the diff can be read;
|
|
79
|
+
without it the section is skipped rather than guessed at. Any such test also enters the attestation
|
|
80
|
+
as a refusal, so a release decision taken from the bundle cannot rest on a test that proves nothing.
|
|
74
81
|
- **Evidence, every run:** the Action writes an attestation into its artifact - the claims with
|
|
75
82
|
their evidence and which layer decided each, what Tewip **refused** to decide, and what it did
|
|
76
83
|
not check at all. Nothing enters it that Tewip did not verify itself, so a release decision can
|
|
@@ -496,6 +503,89 @@ Joins the test plans Tewip wrote to the tests that cover them and the runs that
|
|
|
496
503
|
scenario counts as covered only when a real test can be named, and passing only when a real run
|
|
497
504
|
attempted it; everything else says what is missing. No model, no cost.
|
|
498
505
|
|
|
506
|
+
## Would your tests catch a bug? (Phase 23)
|
|
507
|
+
|
|
508
|
+
```bash
|
|
509
|
+
tewip prove src/pricing.js # or --diff main...HEAD for what a change touched
|
|
510
|
+
tewip prove src/pricing.js --strict # exits non-zero when something goes unnoticed
|
|
511
|
+
```
|
|
512
|
+
|
|
513
|
+
Coverage says a line ran. It does not say anybody checked it. `prove` settles that by **execution**:
|
|
514
|
+
it breaks your code in small, deliberate ways - a boundary moved by one, a condition inverted, an
|
|
515
|
+
`and` that became an `or`, a returned flag flipped, a pattern loosened - runs the tests that reach
|
|
516
|
+
that file, and reports every change **none of them noticed**.
|
|
517
|
+
|
|
518
|
+
```
|
|
519
|
+
4 change(s) were made to your code and the covering tests were run against each. 3 went unnoticed.
|
|
520
|
+
|
|
521
|
+
- line 2: a boundary moved by one
|
|
522
|
+
- was: if (total >= 100 && isMember) return 0.2;
|
|
523
|
+
- became: if (total > 100 && isMember) return 0.2;
|
|
524
|
+
- line 9: a sign reversed
|
|
525
|
+
- was: return total - total * off;
|
|
526
|
+
- became: return total + total * off;
|
|
527
|
+
```
|
|
528
|
+
|
|
529
|
+
Both of those are real: the tests used 150 and 10 and never the boundary itself, and the test calling
|
|
530
|
+
`finalPrice` asserted nothing at all. Neither gap is visible in a coverage report, because both lines
|
|
531
|
+
were covered.
|
|
532
|
+
|
|
533
|
+
It refuses more often than it reports, and the refusals are the reason the reports are worth reading:
|
|
534
|
+
|
|
535
|
+
- **Your file is put back exactly as it was**, byte for byte, line endings included, in a `finally` -
|
|
536
|
+
and the restore is verified, not assumed. A restore that did not work is an error, not a shrug.
|
|
537
|
+
- **Nothing that can hang is touched.** Loop headers are never changed: a flipped comparison in a
|
|
538
|
+
`while` does not fail a test, it runs forever.
|
|
539
|
+
- **A red suite proves nothing.** Every covering test is run untouched first. Ones that do not pass are
|
|
540
|
+
set aside by name, and if none pass, it says so and stops - a test that was already failing would
|
|
541
|
+
otherwise appear to "catch" every change.
|
|
542
|
+
- **It checks that your tests actually load the file.** If breaking the file outright does not fail a
|
|
543
|
+
single test, they are reading an installed copy or a build output, and it says that instead of
|
|
544
|
+
reporting a page of holes that do not exist. This caught a real case on the first Python project it
|
|
545
|
+
met, where `import humanize` resolved to a different checkout entirely.
|
|
546
|
+
- **Strings and comments are never changed**, because changing a message is not a bug a test should be
|
|
547
|
+
expected to catch.
|
|
548
|
+
|
|
549
|
+
It follows imports, so a file reached *through* another module is still proved - in a layered codebase
|
|
550
|
+
that is most of them. It needs a unit test framework (jest, vitest, mocha, `node --test`, pytest, go
|
|
551
|
+
test, maven, gradle, dotnet), calls no model, and costs nothing but time. It is also an MCP tool, so a
|
|
552
|
+
coding agent can ask whether the tests it is relying on would have stopped its own mistake.
|
|
553
|
+
|
|
554
|
+
## Could these tests ever fail? (Phase 22)
|
|
555
|
+
|
|
556
|
+
```bash
|
|
557
|
+
tewip vouch tests/ # or a file, or the whole project
|
|
558
|
+
tewip vouch tests/ --strict # exits non-zero, for a merge gate
|
|
559
|
+
```
|
|
560
|
+
|
|
561
|
+
Most tests in a pull request are now written by an agent, and the question that decides whether they
|
|
562
|
+
are worth anything goes unasked: **could this test ever fail?** Coverage counts a test that checks
|
|
563
|
+
nothing. CI goes green on it. A reviewer skims something plausible and approves it.
|
|
564
|
+
|
|
565
|
+
`vouch` reads tests somebody else wrote and answers per test, quoting the line it rests on. It finds
|
|
566
|
+
tests that leave an expectation unfinished (`expect(value);` with no matcher - it reads exactly like a
|
|
567
|
+
check), tests that compare a value with itself, tests that catch and discard their own failure, tests
|
|
568
|
+
that never run, and tests that compare no value at all. No model, no cost.
|
|
569
|
+
|
|
570
|
+
It is also an MCP tool, so a coding agent can ask about the tests it just wrote before claiming a
|
|
571
|
+
change is covered.
|
|
572
|
+
|
|
573
|
+
**Measured on 5,448 tests in 8 public projects across 5 languages** (express, axios, chalk, requests,
|
|
574
|
+
click, gorilla/mux, cobra, gson): 19 tests were called unable to fail, and **all 19 were correct** when
|
|
575
|
+
read against the files - one unconditional skip in psf/requests, eighteen `@Ignore` methods in gson.
|
|
576
|
+
|
|
577
|
+
Two things it refuses to do, which is why the list is worth reading:
|
|
578
|
+
|
|
579
|
+
- **It will not guess.** A test whose checking happens inside a helper it cannot follow is reported as
|
|
580
|
+
undecided, by name, never as worthless. A wrong accusation about somebody's test costs more than a
|
|
581
|
+
missed one.
|
|
582
|
+
- **It will not overclaim.** A test with no assertion still fails when the code throws - in axios, a
|
|
583
|
+
test that compiles a TypeScript fixture and runs it asserts nothing *on purpose*. Those are reported
|
|
584
|
+
separately as "compares no value", not as broken, and they do not trip `--strict`.
|
|
585
|
+
|
|
586
|
+
Honest limit, also printed at the bottom of every report: its **recall is unmeasured**. A clean report
|
|
587
|
+
means nothing it found was wrong, not that every test in the file is worth something.
|
|
588
|
+
|
|
499
589
|
## A goal instead of a verb (Phases 19 and 21)
|
|
500
590
|
|
|
501
591
|
```bash
|
|
@@ -505,8 +595,9 @@ tewip pursue "keep the checkout flow green" --act
|
|
|
505
595
|
|
|
506
596
|
Give Tewip a goal and it says what it would do next, and why, from what it can actually see. The
|
|
507
597
|
order is fixed, published and testable - **not chosen by a model**: a real bug first, then the
|
|
508
|
-
environment, then flakiness, then anything failing it cannot explain, then a
|
|
509
|
-
then is the goal worth attesting. Deciding the same state twice gives
|
|
598
|
+
environment, then flakiness, then anything failing it cannot explain, then a test that cannot fail,
|
|
599
|
+
then a coverage gap, and only then is the goal worth attesting. Deciding the same state twice gives
|
|
600
|
+
the same answer.
|
|
510
601
|
|
|
511
602
|
With `--act` it carries the decision out, and this is where the refusals matter most:
|
|
512
603
|
|
package/dist/cli/help.js
CHANGED
|
@@ -105,11 +105,28 @@ Usage:
|
|
|
105
105
|
green" - and it says what it would do next and why, from what
|
|
106
106
|
it can actually see. The order is fixed and written down: a
|
|
107
107
|
real bug first, then the environment, then flakiness, then a
|
|
108
|
-
coverage gap. It decides; it does
|
|
108
|
+
test that cannot fail, then a coverage gap. It decides; it does
|
|
109
|
+
not act unless you ask. With
|
|
109
110
|
--act it carries out the two safe ones - writing a missing test,
|
|
110
111
|
or attesting a green goal - by running the command that already
|
|
111
112
|
proves them. It will not touch a real bug, and it writes what it
|
|
112
113
|
did or refused to .tewip/actions.jsonl.
|
|
114
|
+
tewip prove <file> [--diff main...HEAD] [--limit 6] [--budget 600] [--strict]
|
|
115
|
+
What would your tests NOT catch? Breaks your code in small,
|
|
116
|
+
deliberate ways - a boundary moved by one, a condition inverted,
|
|
117
|
+
an and that became an or - runs the tests that import that file,
|
|
118
|
+
and reports every change nobody noticed. The line was covered; it
|
|
119
|
+
was not checked, and no coverage number can tell you that. Every
|
|
120
|
+
file is put back exactly as it was. Slow by nature, and it needs
|
|
121
|
+
a unit test framework. No model, no cost.
|
|
122
|
+
tewip vouch [<paths>] [--strict] [--out vouch.md] [--json vouch.json]
|
|
123
|
+
Could these tests ever fail? Reads tests somebody else wrote -
|
|
124
|
+
an agent, a contractor, last year's team - and names the ones
|
|
125
|
+
that check nothing, compare a value with itself, leave an
|
|
126
|
+
expectation unfinished, swallow their own failure, or never run.
|
|
127
|
+
Where the checking is in a helper it cannot follow it says so
|
|
128
|
+
instead of accusing. --strict exits non-zero on a finding, for a
|
|
129
|
+
gate. No model, no cost.
|
|
113
130
|
tewip trace [--only "<ticket or title>"] [--out trace.md]
|
|
114
131
|
Is this requirement covered, and is it passing right now? Joins
|
|
115
132
|
the plans Tewip wrote to the tests that cover them and the runs
|
package/dist/cli/tewip.js
CHANGED
|
@@ -185,6 +185,26 @@ async function main() {
|
|
|
185
185
|
runPursue({ root: process.cwd(), goal, act: args.includes('--act'), name: flag(args, 'name'), out: flag(args, 'out') });
|
|
186
186
|
return;
|
|
187
187
|
}
|
|
188
|
+
case 'prove': {
|
|
189
|
+
const paths = args.filter((a, i) => !a.startsWith('--') && !['--out', '--diff', '--limit', '--budget'].includes(args[i - 1] ?? ''));
|
|
190
|
+
const { runProve } = await import('../engine/prove/prove-command.js');
|
|
191
|
+
runProve({
|
|
192
|
+
root: process.cwd(),
|
|
193
|
+
files: paths,
|
|
194
|
+
diff: flag(args, 'diff'),
|
|
195
|
+
limit: Number(flag(args, 'limit') ?? '6'),
|
|
196
|
+
budgetSeconds: Number(flag(args, 'budget') ?? '600'),
|
|
197
|
+
out: flag(args, 'out'),
|
|
198
|
+
strict: args.includes('--strict'),
|
|
199
|
+
});
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
case 'vouch': {
|
|
203
|
+
const paths = args.filter((a, i) => !a.startsWith('--') && !['--out', '--json'].includes(args[i - 1] ?? ''));
|
|
204
|
+
const { runVouch } = await import('../engine/vouch/run-vouch.js');
|
|
205
|
+
runVouch({ root: process.cwd(), paths: paths.length > 0 ? paths : ['.'], out: flag(args, 'out'), json: flag(args, 'json'), strict: args.includes('--strict') });
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
188
208
|
case 'trace': {
|
|
189
209
|
const { runTrace } = await import('../engine/trace/trace-command.js');
|
|
190
210
|
runTrace({ root: process.cwd(), only: flag(args, 'only'), name: flag(args, 'name'), out: flag(args, 'out') });
|
|
@@ -11,9 +11,12 @@ function evidenceOf(record) {
|
|
|
11
11
|
* The bundle for one run. `records` and `run` are what Tewip itself wrote when it read the report,
|
|
12
12
|
* so everything here is traceable; nothing is inferred and nothing is re-judged.
|
|
13
13
|
*/
|
|
14
|
-
export function attestRun(project, run, records, { checked, notChecked, tool, engine }) {
|
|
14
|
+
export function attestRun(project, run, records, { checked, notChecked, tool, engine, refusals: given = [], }) {
|
|
15
15
|
const claims = [];
|
|
16
|
-
|
|
16
|
+
// Refusals the caller already established - a test in this change that cannot fail, for instance -
|
|
17
|
+
// belong here beside the ones the records produce. They are the same kind of statement: something
|
|
18
|
+
// this bundle will not vouch for.
|
|
19
|
+
const refusals = [...given];
|
|
17
20
|
for (const record of records) {
|
|
18
21
|
const diagnosis = record.diagnosis;
|
|
19
22
|
const category = diagnosis?.category ?? 'UNKNOWN';
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// Phase 23: small, deliberate breakages of real code, to find out what the tests would not notice.
|
|
2
|
+
//
|
|
3
|
+
// `vouch` reads a test and says whether it *could* fail. This asks the harder question, and answers
|
|
4
|
+
// it by execution rather than by reading: break the code the test covers, run the test, and see. A
|
|
5
|
+
// change nobody notices is a hole in the suite that no coverage percentage will ever show, because
|
|
6
|
+
// the line was covered - it just was not checked.
|
|
7
|
+
//
|
|
8
|
+
// Three rules shape what may be changed, and all three exist to keep the answer trustworthy:
|
|
9
|
+
//
|
|
10
|
+
// 1. **Only code.** Strings and comments are skipped, reusing the same mask `vouch` uses. Changing a
|
|
11
|
+
// word in a message is not a bug a test should be expected to catch.
|
|
12
|
+
// 2. **Nothing that can hang.** Loop headers are never touched. A flipped comparison in a `while`
|
|
13
|
+
// condition does not fail a test, it runs forever, and a tool that hangs a developer's machine
|
|
14
|
+
// gets uninstalled. This is the same lesson as the fork bomb in Phase 21, learned once.
|
|
15
|
+
// 3. **One change at a time**, always restored. A mutant is a question, not an edit.
|
|
16
|
+
import { textRanges } from '../vouch/tests-in.js';
|
|
17
|
+
/** A line that could loop: never touched, because a wrong comparison there hangs instead of failing. */
|
|
18
|
+
const COULD_LOOP = /\b(for|while|do|loop|repeat|until)\b\s*[({:]|\bfor\b.*\bin\b|\.\s*(forEach|map|filter|reduce|while)\s*\(/;
|
|
19
|
+
/** A line that only reports, logs or types: changing it says nothing about behaviour. */
|
|
20
|
+
const NOT_BEHAVIOUR = /^\s*(\/\/|#|\*|\/\*|@|import\b|from\b|export\s+(type|interface)\b|use\s|package\b|using\b|require\b|const\s+\w+\s*=\s*require)|console\s*\.|print\s*\(|println|log(ger)?\s*\.|process\.(stdout|stderr)|fmt\s*\.\s*Print/;
|
|
21
|
+
/**
|
|
22
|
+
* The changes worth making, in the order they are worth making them. Each is a mistake a person or an
|
|
23
|
+
* agent really makes: a boundary off by one, a condition inverted, an `and` that should be an `or`, a
|
|
24
|
+
* sign the wrong way round, a flag returned the wrong way.
|
|
25
|
+
*/
|
|
26
|
+
const OPERATORS = [
|
|
27
|
+
{ kind: 'a boundary moved by one', find: /(?<![<>=!])<=(?!=)/, swap: () => '<' },
|
|
28
|
+
{ kind: 'a boundary moved by one', find: /(?<![<>=!])>=(?!=)/, swap: () => '>' },
|
|
29
|
+
{ kind: 'a boundary moved by one', find: /(?<![<>=!+-])<(?![<=])/, swap: () => '<=' },
|
|
30
|
+
{ kind: 'a boundary moved by one', find: /(?<![<>=!+-])>(?![>=])/, swap: () => '>=' },
|
|
31
|
+
{ kind: 'a comparison inverted', find: /===/, swap: () => '!==' },
|
|
32
|
+
{ kind: 'a comparison inverted', find: /(?<![!=<>])==(?!=)/, swap: () => '!=' },
|
|
33
|
+
{ kind: 'a comparison inverted', find: /!==/, swap: () => '===' },
|
|
34
|
+
{ kind: 'a comparison inverted', find: /(?<![!=<>])!=(?!=)/, swap: () => '==' },
|
|
35
|
+
{ kind: 'an "and" that became an "or"', find: /&&/, swap: () => '||' },
|
|
36
|
+
{ kind: 'an "or" that became an "and"', find: /\|\|/, swap: () => '&&' },
|
|
37
|
+
{ kind: 'a condition inverted', find: /\bif\s*\(\s*!/, swap: (m) => m.replace('!', '') },
|
|
38
|
+
{ kind: 'a sign reversed', find: /(?<=[\w)\]]\s?)\+(?=\s?[\w(])/, swap: () => '-' },
|
|
39
|
+
{ kind: 'a sign reversed', find: /(?<=[\w)\]]\s?)-(?=\s?[\w(])/, swap: () => '+' },
|
|
40
|
+
{ kind: 'a returned flag flipped', find: /\breturn\s+true\b/, swap: () => 'return false' },
|
|
41
|
+
{ kind: 'a returned flag flipped', find: /\breturn\s+false\b/, swap: () => 'return true' },
|
|
42
|
+
{ kind: 'a returned value emptied', find: /\breturn\s+(?!null|undefined|None|nil|true|false|;|$)[\w.[\]]+\s*;?\s*$/, swap: (m) => (m.trimEnd().endsWith(';') ? 'return null;' : 'return null') },
|
|
43
|
+
];
|
|
44
|
+
/**
|
|
45
|
+
* Is this position inside a regular expression literal?
|
|
46
|
+
*
|
|
47
|
+
* Only a rough reading, and it only decides what a change is *called*, never whether it is safe: a
|
|
48
|
+
* slash that opens a pattern follows an operator, a bracket or a keyword, never a value, which is what
|
|
49
|
+
* separates `/re/` from division.
|
|
50
|
+
*/
|
|
51
|
+
function inPattern(line, at) {
|
|
52
|
+
let i = 0;
|
|
53
|
+
while (i < line.length) {
|
|
54
|
+
if (line[i] !== '/') {
|
|
55
|
+
i += 1;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const before = line.slice(0, i).trimEnd();
|
|
59
|
+
const opens = before === '' || /[=(,:[!&|?{;]$|\b(return|typeof|case|of|in|and|or|not)$/.test(before);
|
|
60
|
+
if (!opens) {
|
|
61
|
+
i += 1;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
// Find the closing slash, ignoring one escaped by a backslash and one inside a character class.
|
|
65
|
+
let cls = false;
|
|
66
|
+
for (let j = i + 1; j < line.length; j += 1) {
|
|
67
|
+
const ch = line[j];
|
|
68
|
+
if (ch === String.fromCharCode(92)) {
|
|
69
|
+
j += 1;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (ch === '[')
|
|
73
|
+
cls = true;
|
|
74
|
+
else if (ch === ']')
|
|
75
|
+
cls = false;
|
|
76
|
+
else if (ch === '/' && !cls) {
|
|
77
|
+
if (at > i && at < j)
|
|
78
|
+
return true;
|
|
79
|
+
i = j + 1;
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
if (j === line.length - 1)
|
|
83
|
+
return false; // never closed on this line
|
|
84
|
+
}
|
|
85
|
+
if (i < line.length && line[i] === '/')
|
|
86
|
+
i += 1;
|
|
87
|
+
}
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The mutants worth trying for this file, spread over the file rather than taken from the top: five
|
|
92
|
+
* changes to one function say much less about a suite than five changes in five places.
|
|
93
|
+
*/
|
|
94
|
+
export function mutantsFor(text, limit = 6) {
|
|
95
|
+
const ranges = textRanges(text);
|
|
96
|
+
const lines = text.split(String.fromCharCode(10));
|
|
97
|
+
const offsets = [];
|
|
98
|
+
let seen = 0;
|
|
99
|
+
for (const line of lines) {
|
|
100
|
+
offsets.push(seen);
|
|
101
|
+
seen += line.length + 1;
|
|
102
|
+
}
|
|
103
|
+
const found = [];
|
|
104
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
105
|
+
const line = lines[i];
|
|
106
|
+
if (line.trim().length < 3 || COULD_LOOP.test(line) || NOT_BEHAVIOUR.test(line))
|
|
107
|
+
continue;
|
|
108
|
+
for (const operator of OPERATORS) {
|
|
109
|
+
const hit = operator.find.exec(line);
|
|
110
|
+
if (!hit)
|
|
111
|
+
continue;
|
|
112
|
+
const at = offsets[i] + (hit.index ?? 0);
|
|
113
|
+
// Inside a string or a comment: not behaviour, and often not even syntax once changed.
|
|
114
|
+
if (ranges.some(([from, to]) => at > from && at < to))
|
|
115
|
+
continue;
|
|
116
|
+
const now = line.slice(0, hit.index) + operator.swap(hit[0]) + line.slice((hit.index ?? 0) + hit[0].length);
|
|
117
|
+
if (now === line)
|
|
118
|
+
continue;
|
|
119
|
+
// A change inside a regular expression is worth making - loosening a pattern is exactly how one
|
|
120
|
+
// silently stops matching what it was written for - but it is not a boundary or a comparison, and
|
|
121
|
+
// calling it one makes an otherwise good finding look like a bug in the tool.
|
|
122
|
+
const kind = inPattern(line, hit.index ?? 0) ? 'a pattern loosened' : operator.kind;
|
|
123
|
+
found.push({ line: i + 1, was: line.trim(), now: now.trim(), kind });
|
|
124
|
+
break; // one change per line keeps each answer about one thing
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return spread(found, limit);
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Takes at most `limit`, as far apart in the file as they can be. A suite that catches everything in
|
|
131
|
+
* one function and nothing in the rest of the file should not look like a suite that catches
|
|
132
|
+
* everything, and taking the first six mutants in order is how that happens.
|
|
133
|
+
*/
|
|
134
|
+
function spread(all, limit) {
|
|
135
|
+
if (all.length <= limit)
|
|
136
|
+
return all;
|
|
137
|
+
const step = all.length / limit;
|
|
138
|
+
return Array.from({ length: limit }, (unused, i) => all[Math.floor(i * step)]);
|
|
139
|
+
}
|
|
140
|
+
/** The file with this one mutant applied, ready to be written and then put back. */
|
|
141
|
+
export function applyMutant(text, mutant) {
|
|
142
|
+
const lines = text.split(String.fromCharCode(10));
|
|
143
|
+
const line = lines[mutant.line - 1];
|
|
144
|
+
if (line === undefined || line.trim() !== mutant.was)
|
|
145
|
+
throw new Error(`line ${mutant.line} is not what the mutant was made from, so nothing was written`);
|
|
146
|
+
lines[mutant.line - 1] = line.replace(mutant.was, mutant.now);
|
|
147
|
+
return lines.join(String.fromCharCode(10));
|
|
148
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// `tewip prove <file>`: what your tests would not catch, shown by breaking the code and watching.
|
|
2
|
+
//
|
|
3
|
+
// The report is written for the reader who has to decide whether to trust a suite. It leads with what
|
|
4
|
+
// went unnoticed, quotes the exact change, names the test that caught each one that was caught, and
|
|
5
|
+
// says what the answer cost in seconds - because this is the one command here that is slow, and a
|
|
6
|
+
// number somebody waited four minutes for should say how long it took.
|
|
7
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { dirname, relative, resolve } from 'node:path';
|
|
9
|
+
import { changeSummary } from '../quality/grounding.js';
|
|
10
|
+
import { chooseFramework, detectFrameworks } from '../quality/frameworks.js';
|
|
11
|
+
import { coverageFor, testFiles } from '../review/coverage.js';
|
|
12
|
+
import { importingTests, proveFile } from './prove.js';
|
|
13
|
+
import { testsReaching } from './reaches.js';
|
|
14
|
+
const SOURCE = /\.(js|jsx|mjs|cjs|ts|tsx|mts|cts|py|java|kt|go|cs|rb|php)$/i;
|
|
15
|
+
const IS_TEST = /(^|[\\/])(tests?|spec|specs|__tests__|e2e)[\\/]|[._-](test|tests|spec|specs)\.[a-z]+$|(^|[\\/])test_[^\\/]+\.py$|_test\.(go|py|rb)$|Tests?\.(java|kt|cs)$/i;
|
|
16
|
+
/** The source files to prove: the ones named, or the ones a change touched. */
|
|
17
|
+
export function filesToProve(cmd) {
|
|
18
|
+
if (cmd.files.length > 0)
|
|
19
|
+
return cmd.files.map((f) => f.split('\\').join('/'));
|
|
20
|
+
if (!cmd.diff)
|
|
21
|
+
return [];
|
|
22
|
+
const change = changeSummary(cmd.root, cmd.diff);
|
|
23
|
+
if (!change)
|
|
24
|
+
return [];
|
|
25
|
+
return change.files.map((f) => f.split('\\').join('/')).filter((f) => SOURCE.test(f) && !IS_TEST.test(f) && existsSync(resolve(cmd.root, f)));
|
|
26
|
+
}
|
|
27
|
+
export function renderProof(proofs) {
|
|
28
|
+
const tried = proofs.flatMap((p) => p.tried);
|
|
29
|
+
const unnoticed = tried.filter((t) => t.fate === 'unnoticed');
|
|
30
|
+
const noticed = tried.filter((t) => t.fate === 'noticed');
|
|
31
|
+
const invalid = tried.filter((t) => t.fate === 'did-not-compile');
|
|
32
|
+
const broke = tried.filter((t) => t.fate === 'runner-broke');
|
|
33
|
+
const seconds = proofs.reduce((total, p) => total + p.seconds, 0);
|
|
34
|
+
const lines = [
|
|
35
|
+
'# What your tests would not catch',
|
|
36
|
+
'',
|
|
37
|
+
tried.length === 0
|
|
38
|
+
? 'Nothing was tried. See below for why.'
|
|
39
|
+
: `${tried.length} change(s) were made to your code and the covering tests were run against each. **${unnoticed.length} went unnoticed.** ${noticed.length} were caught. Took ${seconds}s.`,
|
|
40
|
+
'',
|
|
41
|
+
];
|
|
42
|
+
if (unnoticed.length > 0) {
|
|
43
|
+
lines.push(`## Unnoticed (${unnoticed.length})`, '', 'Each of these is a change to your code that every test covering it still passed. The line was', 'covered; it was not checked.', '');
|
|
44
|
+
for (const proof of proofs) {
|
|
45
|
+
const gaps = proof.tried.filter((t) => t.fate === 'unnoticed');
|
|
46
|
+
if (gaps.length === 0)
|
|
47
|
+
continue;
|
|
48
|
+
lines.push(`### ${proof.file}`, '');
|
|
49
|
+
for (const gap of gaps) {
|
|
50
|
+
lines.push(`- **line ${gap.mutant.line}: ${gap.mutant.kind}**`);
|
|
51
|
+
lines.push(` - was: \`${gap.mutant.was.slice(0, 120)}\``);
|
|
52
|
+
lines.push(` - became: \`${gap.mutant.now.slice(0, 120)}\``);
|
|
53
|
+
}
|
|
54
|
+
lines.push('', `Run by: ${proof.covering.join(', ')}`, '');
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (noticed.length > 0) {
|
|
58
|
+
lines.push(`## Caught (${noticed.length})`, '', 'These your tests would have stopped.', '');
|
|
59
|
+
for (const good of noticed)
|
|
60
|
+
lines.push(`- line ${good.mutant.line}: ${good.mutant.kind} — caught by \`${good.noticedBy}\``);
|
|
61
|
+
lines.push('');
|
|
62
|
+
}
|
|
63
|
+
const blocked = proofs.filter((p) => p.blocked);
|
|
64
|
+
if (blocked.length > 0) {
|
|
65
|
+
lines.push(`## Nothing could be proved here (${blocked.length})`, '');
|
|
66
|
+
for (const p of blocked)
|
|
67
|
+
lines.push(`- **${p.file}**: ${p.blocked}`);
|
|
68
|
+
lines.push('');
|
|
69
|
+
}
|
|
70
|
+
if (invalid.length > 0 || broke.length > 0 || proofs.some((p) => p.notTried > 0 || p.setAside.length > 0)) {
|
|
71
|
+
lines.push('## Not counted', '');
|
|
72
|
+
if (invalid.length > 0)
|
|
73
|
+
lines.push(`- ${invalid.length} change(s) did not compile, so they say nothing about the tests. That is Tewip's fault, not yours.`);
|
|
74
|
+
for (const p of proofs)
|
|
75
|
+
for (const aside of p.setAside)
|
|
76
|
+
lines.push(`- \`${aside.test}\` was set aside: ${aside.why}. It covers ${p.file}, so it may well notice things the tests above did not.`);
|
|
77
|
+
if (broke.length > 0)
|
|
78
|
+
lines.push(`- ${broke.length} run(s) broke in the environment rather than the tests.`);
|
|
79
|
+
const left = proofs.reduce((total, p) => total + p.notTried, 0);
|
|
80
|
+
if (left > 0)
|
|
81
|
+
lines.push(`- ${left} change(s) were never tried: the time budget ran out. Raise it with \`--budget <seconds>\`.`);
|
|
82
|
+
lines.push('');
|
|
83
|
+
}
|
|
84
|
+
lines.push('---', '', 'Every line above was proved by running your own tests against your own code, one small change at', 'a time, and every file was put back exactly as it was. No model was called.', '', 'What this does not say: that the unnoticed changes are bugs, or that the caught ones mean the', 'suite is good. It says only which of these particular mistakes your tests would have stopped -', 'which is more than a coverage percentage can tell you, and less than a proof that the code works.');
|
|
85
|
+
return lines.join(String.fromCharCode(10));
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* One file, proved, with nothing printed: what the MCP server and any other caller use. `runProve`
|
|
89
|
+
* prints as it goes because a person waiting minutes deserves to see progress; over a protocol that
|
|
90
|
+
* speaks on stdout, printing would corrupt the stream.
|
|
91
|
+
*/
|
|
92
|
+
export function proveOne(root, file, opts = {}) {
|
|
93
|
+
const framework = chooseFramework(detectFrameworks(root), undefined, false);
|
|
94
|
+
if (!framework)
|
|
95
|
+
return 'No unit test framework was found here (jest, vitest, mocha, node --test, pytest, go test, maven, gradle, dotnet). Proving works by running your tests, so it needs one.';
|
|
96
|
+
if (!existsSync(resolve(root, file)))
|
|
97
|
+
return `No such file here: ${file}`;
|
|
98
|
+
const covering = coveringTests(root, file, testFiles(root));
|
|
99
|
+
return renderProof([proveFile(root, file, covering.tests, framework, { limit: opts.limit, budgetSeconds: opts.budgetSeconds })]);
|
|
100
|
+
}
|
|
101
|
+
/** The tests that could notice a change here: the ones that import the file, else the ones that reach it. */
|
|
102
|
+
function coveringTests(root, file, known) {
|
|
103
|
+
const direct = importingTests(root, coverageFor(root, [file], known), file);
|
|
104
|
+
if (direct.length > 0)
|
|
105
|
+
return { tests: direct, how: 'import it directly' };
|
|
106
|
+
const reached = testsReaching(root, file, known.map((t) => t.split(String.fromCharCode(92)).join('/')));
|
|
107
|
+
return { tests: reached.map((r) => r.testFile), how: reached.length > 0 ? `reach it through ${reached[0].steps} import(s), via ${reached[0].via}` : 'import it' };
|
|
108
|
+
}
|
|
109
|
+
export function runProve(cmd) {
|
|
110
|
+
const files = filesToProve(cmd);
|
|
111
|
+
if (files.length === 0) {
|
|
112
|
+
throw new Error('name a source file to prove, or pass --diff main...HEAD to prove what a change touched');
|
|
113
|
+
}
|
|
114
|
+
// A unit framework, never a browser one: proving needs dozens of runs, and a browser run costs
|
|
115
|
+
// minutes. Browser tests are where `tewip fix` and the drift lab work instead.
|
|
116
|
+
const framework = chooseFramework(detectFrameworks(cmd.root), undefined, false);
|
|
117
|
+
if (!framework) {
|
|
118
|
+
throw new Error('no unit test framework was found here (jest, vitest, mocha, pytest, go test, maven, gradle, dotnet). Proving needs one, because it works by running your tests.');
|
|
119
|
+
}
|
|
120
|
+
const known = testFiles(cmd.root);
|
|
121
|
+
const proofs = [];
|
|
122
|
+
for (const file of files) {
|
|
123
|
+
if (!existsSync(resolve(cmd.root, file))) {
|
|
124
|
+
proofs.push({ file, covering: [], tried: [], blocked: 'no such file here', setAside: [], notTried: 0, seconds: 0 });
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
// Nothing importing it by name is the normal case in a layered codebase, not the absence of
|
|
128
|
+
// tests, so the import chain is followed outwards before anything is concluded.
|
|
129
|
+
const { tests: covering, how } = coveringTests(cmd.root, file, known);
|
|
130
|
+
console.error(`${file}: ${covering.length} test file(s) ${how}`);
|
|
131
|
+
proofs.push(proveFile(cmd.root, file, covering, framework, {
|
|
132
|
+
limit: cmd.limit,
|
|
133
|
+
budgetSeconds: cmd.budgetSeconds,
|
|
134
|
+
onStep: (message) => console.error(` ${message}`),
|
|
135
|
+
}));
|
|
136
|
+
}
|
|
137
|
+
const report = renderProof(proofs);
|
|
138
|
+
console.log(report);
|
|
139
|
+
if (cmd.out) {
|
|
140
|
+
mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
|
|
141
|
+
writeFileSync(resolve(cmd.out), report + String.fromCharCode(10), 'utf8');
|
|
142
|
+
}
|
|
143
|
+
const unnoticed = proofs.flatMap((p) => p.tried).filter((t) => t.fate === 'unnoticed').length;
|
|
144
|
+
if (cmd.strict && unnoticed > 0) {
|
|
145
|
+
console.error(String.fromCharCode(10) + `${unnoticed} change(s) to your code went unnoticed by the tests covering it.`);
|
|
146
|
+
process.exitCode = 1;
|
|
147
|
+
}
|
|
148
|
+
return { proofs, unnoticed };
|
|
149
|
+
}
|
|
150
|
+
/** Kept so a caller that only wants the numbers does not have to parse the report. */
|
|
151
|
+
export const readSource = (root, file) => readFileSync(resolve(root, file), 'utf8');
|
|
152
|
+
export const shortName = (root, file) => relative(root, resolve(root, file)).split('\\').join('/');
|