probatio 0.0.0-stage → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -2
- package/dist/cli.js +499 -0
- package/dist/contract.js +29 -0
- package/dist/flags.js +87 -0
- package/dist/golden/check.js +188 -0
- package/dist/ledger/build.js +327 -0
- package/dist/matrix/report.js +256 -0
- package/dist/mcp.js +116 -0
- package/dist/memory/state.js +222 -0
- package/dist/mutate/affected.js +173 -0
- package/dist/mutate/coverage-map.js +172 -0
- package/dist/mutate/fail-reporter.mjs +44 -0
- package/dist/mutate/find.js +13 -0
- package/dist/mutate/generate.js +132 -0
- package/dist/mutate/go-coverage.js +100 -0
- package/dist/mutate/ids.js +22 -0
- package/dist/mutate/java-coverage.js +188 -0
- package/dist/mutate/kill-label.js +34 -0
- package/dist/mutate/node-coverage.mjs +245 -0
- package/dist/mutate/operators.js +122 -0
- package/dist/mutate/patch.js +135 -0
- package/dist/mutate/probatio-jacoco-run.java +81 -0
- package/dist/mutate/py-pytest-reporter.py +145 -0
- package/dist/mutate/py-reporter.py +79 -0
- package/dist/mutate/run.js +868 -0
- package/dist/mutate/suite-decision.js +129 -0
- package/dist/mutate/suites.js +1371 -0
- package/dist/mutate/tally.js +175 -0
- package/dist/mutate/text-operators.js +547 -0
- package/dist/mutate/titles.js +23 -0
- package/dist/swarm/check-kill.js +107 -0
- package/dist/swarm/queue.js +62 -0
- package/dist/verify/change.js +262 -0
- package/package.json +33 -4
package/README.md
CHANGED
|
@@ -1,3 +1,84 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Probatio
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Probatio (Latin: a testing, a proof) is a free, open-source testing toolkit built for AI agents
|
|
4
|
+
(Claude, Grok, ChatGPT, others), working alone or as swarms. Every claim it makes is checkable;
|
|
5
|
+
it remembers what was fixed so agents with small context do not redo or undo work; and it finds
|
|
6
|
+
the bugs a test suite would miss instead of asking anyone to read more tests.
|
|
7
|
+
|
|
8
|
+
Status (2026-10-05): `mutate run` discovers the project suite and scores it. Python, JavaScript, and TypeScript narrow each mutant with a line map collected on the baseline. C does the same when the binary was built with LLVM coverage. A suite with no line map, a baseline of 5 seconds or more, and more than 30 mutants stops before the first mutant. `verify-change` mutates the changed lines and uses that same map. `mutate tally` reads a finished run, names tests that killed nothing, and does not delete a test. The MCP server speaks one JSON object per line and returns the same JSON as the CLI.
|
|
9
|
+
|
|
10
|
+
Read in this order:
|
|
11
|
+
|
|
12
|
+
1. `HANDOFF.md` — why, the evidence from Auspex, the design, and the lessons that cost time.
|
|
13
|
+
2. `FOR-GROK.md` — the builder brief: build order with acceptance tests, the agent contract,
|
|
14
|
+
memory for agents, anti-gaming, swarm protocol, traps.
|
|
15
|
+
3. `seed/` — code copied from the Auspex experiment to start from (Auspex paths inside).
|
|
16
|
+
|
|
17
|
+
Roles: Grok builds most of it; Claude refines and finishes. Grok keeps `NOTES-FOR-CLAUDE.md` here
|
|
18
|
+
(decisions, guesses, shortcuts with file:line, acceptance output actually run).
|
|
19
|
+
|
|
20
|
+
## mutate
|
|
21
|
+
|
|
22
|
+
One JSON object on stdout. `schemaVersion`, `ok`, `summary`, `next`, `nextCall`. Exit 0 only when `ok` is true. Human text is `--human`. A run checks the suite out in a throwaway worktree, so the checkout you are in stays as it is.
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
npx probatio mutate generate --package . --out .probatio/generate
|
|
26
|
+
npx probatio mutate run --package . --patches .probatio/generate/mutants --out .probatio/runs
|
|
27
|
+
npx probatio mutate tally --out .probatio/runs
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`generate` writes operator mutants (conditions, `&&`/`||`, `===`/`!==`, boundaries, booleans, a dropped `!`). The same class is written for COBOL (`.cob`, `.cbl`), Rust, C, C++, Java, Go, Python, JavaScript, and C#. Strings and comments are not mutated. A COBOL `*>` comment and a fixed-format line whose column 7 is `*` or `/` are comments. Each patch is forward: apply it to introduce the bug, and the file says so. When mutants were written, `next` states the count, the suite command if one was discovered, and whether the first run collects a line map.
|
|
31
|
+
|
|
32
|
+
`run` discovers the suite: `run_tests.sh`, Cargo, Go, Swift, Maven, dotnet, pytest, unittest, node:test, Mocha, then `make test` when COBOL tests sit under that Makefile. An unknown layout stops and asks for `--suite-command`. It does not compile one file and call that the suite. `make test` that would curl or wget a missing file stops, and nothing is fetched. A worktree that has no `node_modules` uses the main checkout's. Omitting `--build` runs no build. Pass `--build` with a command when the suite needs one first.
|
|
33
|
+
|
|
34
|
+
Python, JavaScript, and TypeScript collect a line map on the baseline. C collects one when LLVM coverage was instrumented. Go and Maven/Java collect file, line, and the test names that hit that line. A later mutant runs only those tests (`go test -run`, Maven `-Dtest`). A line no test executed is `no coverage`, and the suite is not started. Rust and C# can take a test name and do not collect a line. A baseline of 5 seconds or more, with no line map and more than 30 mutants still pending, stops before the first mutant. `next` names that baseline and tells you to narrow `--src` or pass a smaller patch directory. A one-file C, C++, Java, COBOL, or assembly launcher is not a suite discovery returns.
|
|
35
|
+
|
|
36
|
+
The mutant timeout is the baseline duration times 5, and at least 20 seconds, capped by `--suite-timeout-ms`. A timeout is not a kill. `--budget-ms` stops mutant work after the baseline. The first mutant still runs, except for that no-line-map stop. A kill is the test that failed, or a compiler token when the mutant did not build. Confirm is on by default and reruns the failing names. Pytest names that `-k` cannot express are passed as node ids. A usage error, including pytest exit 4, is not a kill. `verify-change` does not confirm a second time. Add `--affected` to limit the file list to tests that can see the change. The default, once a line map exists, runs the tests on the changed line.
|
|
37
|
+
|
|
38
|
+
`tally` reads that run directory. The summary leads with the no-coverage count, then the survivors. It names tests that killed a mutant and tests that killed nothing. A gap is a survivor. An unseen line is not a gap and not a pass. A timeout is neither a kill nor a gap. `deletedTests` is 0. It does not delete a file.
|
|
39
|
+
|
|
40
|
+
## ledger
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
npx probatio ledger build --package . --commit HEAD --out .probatio/ledger
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
A fix commit has a `Fixes-bug:` trailer, or it changes both `src` and a test. `ledger build` reverts that commit's src diff onto the tree you name. A diff that applies is written as a forward patch (`source=history`): apply it to put the bug back. A diff that does not apply is reported, and left for a hand-made patch in the same directory (`source=hand`, `fix=<commit>`). A rebuild keeps those hand-made files. Nothing is fuzzy-applied. A src diff over `--max-lines` (default 300) is skipped. `--max-commits N` reads only the newest N non-merge commits and says when older history was not scanned. The default reads the whole history. A tree is checked out only when a fix inside that cap has to be applied.
|
|
47
|
+
|
|
48
|
+
## matrix
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
npx probatio matrix report --scores round2-union-scores.tsv --direct round2-direct-scores.txt --patches mutants/holdout --patches mutants/train --out .probatio/matrix
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Reads those recorded tables. It does not rerun the mutants. The headline is the gap list: `file:line`, the top 10, how many are left, and the path to the full JSON. A test that killed nothing can be named. The file is not deleted.
|
|
55
|
+
|
|
56
|
+
## golden
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
npx probatio golden check --recorded table.json --actual now.json --update wording
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`next` and `nextLead` are wording. `ok`, `reason`, `status`, `hostChanged`, and `nextCall` are the contract. `--update wording` re-records a changed `next` or `nextLead` and leaves the contract fields alone. A file whose case names are the top-level keys is written back in that shape. A changed `ok` or `reason` also needs `Golden-Change: <row>: <why>` in `--message`. A home path, or `ok` without a reason, fails and nothing is written.
|
|
63
|
+
|
|
64
|
+
## status
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
npx probatio status --state .probatio --page PROBATIO.md
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Rewrites `PROBATIO.md` from the same JSON: open gaps, recent fixes and their guards, protected guards, decisions. A sealed id is left out of the page, the JSON, and `queue seed`. If visible kills rise and sealed kills do not, the summary is `these tests fit the yardstick, not the code.`
|
|
71
|
+
|
|
72
|
+
## verify-change
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
npx probatio verify-change --package . --base HEAD~1 --out .probatio/verify
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Lists the tests that can see the diff, runs mutants on the changed lines, and reports caught, missed, mutants that were not run, lines with no coverage, and any golden contract change. The mutants are the same operator class `mutate generate` would write, including a non-TypeScript diff. A string or a comment is not a mutant. `ran` is still the direct-importer list. The tests that execute are chosen by the line map, the same way `mutate run` chooses them, including when that importer list is empty. No discovered suite leaves those mutants in `notRun` and does not start a run. The suite timeout is 10 minutes. `--max-tests` only shortens the reported importer list. `--max-mutants` (default 4) caps how many changed lines are run.
|
|
79
|
+
|
|
80
|
+
## queue
|
|
81
|
+
|
|
82
|
+
An item is `.probatio/queue/<id>.json`. `queue claim` renames it to `claimed/<agent>-<id>.json` with a lease. `queue reap` moves an expired lease back. `check-kill <id>` runs the tests that directly import the mutated file and confirms the killing test again. It is done only when that mutant dies and that suite is still green. `verify-change` is the edit-sized run: it uses the line map and does not confirm a second time.
|
|
83
|
+
|
|
84
|
+
The MCP server speaks stdio JSON-RPC. Its one tool runs the CLI and returns that command's JSON.
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { SCHEMA_VERSION, render } from "./contract.js";
|
|
5
|
+
import { bool, int, parseArgs, requireText, text, texts } from "./flags.js";
|
|
6
|
+
import { asTable, checkGolden, goldenShape, readGolden, writeGolden } from "./golden/check.js";
|
|
7
|
+
import { buildLedger } from "./ledger/build.js";
|
|
8
|
+
import { readKills, reportMatrix } from "./matrix/report.js";
|
|
9
|
+
import { appendFinding, chooseSealed, fixGap, guardAllows, lineHash, loadState, parseGuard, readSourceLine, revertGap, saveState, sealIds, statusEnvelope, } from "./memory/state.js";
|
|
10
|
+
import { git } from "./mutate/patch.js";
|
|
11
|
+
import { generateMutants } from "./mutate/generate.js";
|
|
12
|
+
import { runMutants } from "./mutate/run.js";
|
|
13
|
+
import { generateNext } from "./mutate/suite-decision.js";
|
|
14
|
+
import { discoverSuite } from "./mutate/suites.js";
|
|
15
|
+
import { tallyRun } from "./mutate/tally.js";
|
|
16
|
+
import { checkKill } from "./swarm/check-kill.js";
|
|
17
|
+
import { claimItem, reapClaims, seedQueue } from "./swarm/queue.js";
|
|
18
|
+
import { verifyChange } from "./verify/change.js";
|
|
19
|
+
const parsed = parseArgs(process.argv.slice(2));
|
|
20
|
+
const [group, action] = parsed.command;
|
|
21
|
+
try {
|
|
22
|
+
if (group === "mutate" && action === "generate")
|
|
23
|
+
finish(await generateCommand(parsed.flags), parsed.human);
|
|
24
|
+
if (group === "mutate" && action === "run")
|
|
25
|
+
finish(await runCommand(parsed.flags), parsed.human);
|
|
26
|
+
if (group === "mutate" && action === "tally")
|
|
27
|
+
finish(tallyCommand(parsed.flags), parsed.human);
|
|
28
|
+
if (group === "ledger" && action === "build")
|
|
29
|
+
finish(await ledgerCommand(parsed.flags), parsed.human);
|
|
30
|
+
if (group === "matrix" && action === "report")
|
|
31
|
+
finish(matrixCommand(parsed.flags), parsed.human);
|
|
32
|
+
if (group === "golden" && action === "check")
|
|
33
|
+
finish(goldenCommand(parsed.flags), parsed.human);
|
|
34
|
+
if (group === "gap" && action === "fix")
|
|
35
|
+
finish(gapFixCommand(parsed.flags), parsed.human);
|
|
36
|
+
if (group === "gap" && action === "revert")
|
|
37
|
+
finish(gapRevertCommand(parsed.flags), parsed.human);
|
|
38
|
+
if (group === "guard" && action === "check")
|
|
39
|
+
finish(guardCommand(parsed.flags), parsed.human);
|
|
40
|
+
if (group === "findings" && action === "add")
|
|
41
|
+
finish(findingsCommand(parsed.flags), parsed.human);
|
|
42
|
+
if (group === "status")
|
|
43
|
+
finish(statusCommand(parsed.flags), parsed.human);
|
|
44
|
+
if (group === "seal")
|
|
45
|
+
finish(sealCommand(parsed.flags), parsed.human);
|
|
46
|
+
if (group === "queue" && action === "seed")
|
|
47
|
+
finish(queueSeedCommand(parsed.flags), parsed.human);
|
|
48
|
+
if (group === "queue" && action === "claim")
|
|
49
|
+
finish(queueClaimCommand(parsed.flags), parsed.human);
|
|
50
|
+
if (group === "queue" && action === "reap")
|
|
51
|
+
finish(queueReapCommand(parsed.flags), parsed.human);
|
|
52
|
+
if (group === "check-kill")
|
|
53
|
+
finish(await checkKillCommand(parsed.flags, action), parsed.human);
|
|
54
|
+
if (group === "verify-change")
|
|
55
|
+
finish(await verifyCommand(parsed.flags), parsed.human);
|
|
56
|
+
finish(usage(false, "Use mutate, ledger, matrix, golden, gap, guard, findings, status, queue, check-kill, or verify-change."), parsed.human);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
60
|
+
finish({
|
|
61
|
+
schemaVersion: SCHEMA_VERSION,
|
|
62
|
+
ok: false,
|
|
63
|
+
command: `${group ?? "mutate"}.${action ?? "help"}`,
|
|
64
|
+
summary: message,
|
|
65
|
+
next: "Fix the command and run it again.",
|
|
66
|
+
nextCall: null,
|
|
67
|
+
}, parsed.human);
|
|
68
|
+
}
|
|
69
|
+
function usage(ok, summary) {
|
|
70
|
+
return {
|
|
71
|
+
schemaVersion: SCHEMA_VERSION,
|
|
72
|
+
ok,
|
|
73
|
+
command: "mutate.help",
|
|
74
|
+
summary,
|
|
75
|
+
next: "probatio mutate generate writes operator mutants. probatio mutate run confirms each kill twice. probatio mutate tally reads a finished run and does not delete a test. probatio ledger build reverts fix commits. probatio matrix report reads a recorded score table.",
|
|
76
|
+
nextCall: null,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
async function generateCommand(flags) {
|
|
80
|
+
const packageDir = path.resolve(requireText(flags, "package"));
|
|
81
|
+
const outDir = path.resolve(requireText(flags, "out"));
|
|
82
|
+
const perFile = int(flags, "per-file");
|
|
83
|
+
const result = generateMutants({
|
|
84
|
+
packageDir,
|
|
85
|
+
srcDir: text(flags, "src") ?? "src",
|
|
86
|
+
outDir,
|
|
87
|
+
perFile: perFile === undefined ? null : perFile,
|
|
88
|
+
skip: int(flags, "skip") ?? 0,
|
|
89
|
+
seed: int(flags, "seed") ?? 20261003,
|
|
90
|
+
maxMutants: int(flags, "max-mutants") ?? null,
|
|
91
|
+
maxMinutes: int(flags, "max-minutes") ?? null,
|
|
92
|
+
skipFiles: texts(flags, "skip-file"),
|
|
93
|
+
});
|
|
94
|
+
if (result.violations.length > 0) {
|
|
95
|
+
const first = result.violations[0];
|
|
96
|
+
return {
|
|
97
|
+
schemaVersion: SCHEMA_VERSION,
|
|
98
|
+
ok: false,
|
|
99
|
+
command: "mutate.generate",
|
|
100
|
+
summary: `Refusing to write mutants: ${first.file}:${first.line} sits inside a string or a comment.`,
|
|
101
|
+
next: "That is a generator bug. Do not use the batch.",
|
|
102
|
+
nextCall: null,
|
|
103
|
+
stringLiteralMutants: result.violations.length,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
const shown = result.mutants.slice(0, 10).map(({ id, file, line, op }) => ({ id, file, line, op }));
|
|
107
|
+
const patches = path.join(outDir, "mutants");
|
|
108
|
+
const stopped = result.budgetHit;
|
|
109
|
+
const suite = discoverSuite(packageDir, text(flags, "tests-dir") ?? "tests");
|
|
110
|
+
return {
|
|
111
|
+
schemaVersion: SCHEMA_VERSION,
|
|
112
|
+
ok: true,
|
|
113
|
+
command: "mutate.generate",
|
|
114
|
+
summary: stopped
|
|
115
|
+
? `Stopped by the time budget after ${result.filesVisited} files. ${result.mutants.length} mutants written.`
|
|
116
|
+
: `${result.mutants.length} mutants in ${result.filesVisited} files. No string-literal mutant.`,
|
|
117
|
+
next: stopped
|
|
118
|
+
? "Narrow --src or raise --max-minutes, then run the same command."
|
|
119
|
+
: generateNext(result.mutants.length, suite ? { kind: suite.kind, command: suite.command } : null),
|
|
120
|
+
nextCall: result.mutants.length === 0
|
|
121
|
+
? null
|
|
122
|
+
: { argv: ["mutate", "run", "--package", packageDir, "--patches", patches, "--out", path.join(outDir, "runs")] },
|
|
123
|
+
mutantCount: result.mutants.length,
|
|
124
|
+
stringLiteralMutants: 0,
|
|
125
|
+
filesVisited: result.filesVisited,
|
|
126
|
+
budgetHit: result.budgetHit,
|
|
127
|
+
mutants: shown,
|
|
128
|
+
rest: Math.max(result.mutants.length - shown.length, 0),
|
|
129
|
+
full: path.join(patches, "mutants.json"),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
async function runCommand(flags) {
|
|
133
|
+
const packageDir = path.resolve(requireText(flags, "package"));
|
|
134
|
+
const patchDirs = texts(flags, "patches").map((dir) => path.resolve(dir));
|
|
135
|
+
if (patchDirs.length === 0)
|
|
136
|
+
throw new Error("--patches is required");
|
|
137
|
+
const outDir = path.resolve(requireText(flags, "out"));
|
|
138
|
+
const repoDir = path.resolve(text(flags, "repo") ?? gitRoot(packageDir));
|
|
139
|
+
const workers = int(flags, "workers") ?? 1;
|
|
140
|
+
if (workers < 1)
|
|
141
|
+
throw new Error("--workers must be at least 1");
|
|
142
|
+
const direction = text(flags, "direction") ?? "auto";
|
|
143
|
+
if (direction !== "auto" && direction !== "forward" && direction !== "reverse") {
|
|
144
|
+
throw new Error("--direction must be auto, forward, or reverse");
|
|
145
|
+
}
|
|
146
|
+
const build = buildCommand(flags);
|
|
147
|
+
const report = await runMutants({
|
|
148
|
+
packageDir,
|
|
149
|
+
repoDir,
|
|
150
|
+
commit: text(flags, "commit") ?? "HEAD",
|
|
151
|
+
patchDirs,
|
|
152
|
+
outDir,
|
|
153
|
+
direction,
|
|
154
|
+
workers,
|
|
155
|
+
concurrency: int(flags, "concurrency") ?? 3,
|
|
156
|
+
maxMinutes: int(flags, "max-minutes") ?? null,
|
|
157
|
+
maxMutants: int(flags, "max-mutants") ?? null,
|
|
158
|
+
build,
|
|
159
|
+
testsDir: text(flags, "tests-dir") ?? "tests",
|
|
160
|
+
unset: texts(flags, "unset"),
|
|
161
|
+
suiteTimeoutMs: int(flags, "suite-timeout-ms") ?? 600_000,
|
|
162
|
+
testTimeoutMs: int(flags, "test-timeout-ms") ?? 60_000,
|
|
163
|
+
confirm: bool(flags, "confirm", true),
|
|
164
|
+
affected: bool(flags, "affected", false),
|
|
165
|
+
agent: text(flags, "agent") ?? null,
|
|
166
|
+
suiteCommand: text(flags, "suite-command") ?? null,
|
|
167
|
+
historyPath: text(flags, "history") ? path.resolve(text(flags, "history")) : null,
|
|
168
|
+
timeoutMultiple: int(flags, "timeout-multiple") ?? 5,
|
|
169
|
+
timeoutFloorMs: int(flags, "timeout-floor-ms") ?? 20_000,
|
|
170
|
+
budgetMs: int(flags, "budget-ms") ?? null,
|
|
171
|
+
onProgress: (line) => process.stderr.write(`${line}\n`),
|
|
172
|
+
});
|
|
173
|
+
const resume = report.budgetHit
|
|
174
|
+
? { argv: ["mutate", "run", "--package", packageDir, "--repo", repoDir, "--out", outDir, ...patchDirs.flatMap((dir) => ["--patches", dir])] }
|
|
175
|
+
: null;
|
|
176
|
+
return {
|
|
177
|
+
schemaVersion: SCHEMA_VERSION,
|
|
178
|
+
ok: report.ok,
|
|
179
|
+
command: "mutate.run",
|
|
180
|
+
summary: report.summary,
|
|
181
|
+
next: report.next,
|
|
182
|
+
nextCall: report.ok ? resume : null,
|
|
183
|
+
killed: report.killed,
|
|
184
|
+
survived: report.survived,
|
|
185
|
+
flaky: report.flaky,
|
|
186
|
+
timeouts: report.timeouts,
|
|
187
|
+
errors: report.errors,
|
|
188
|
+
noCoverage: report.noCoverage,
|
|
189
|
+
notStarted: report.notStarted,
|
|
190
|
+
suiteCommand: report.suiteCommand,
|
|
191
|
+
baselineMs: report.baselineMs,
|
|
192
|
+
commands: report.commands,
|
|
193
|
+
gaps: report.gaps,
|
|
194
|
+
rest: report.rest,
|
|
195
|
+
flakyIds: report.flakyIds,
|
|
196
|
+
budgetHit: report.budgetHit,
|
|
197
|
+
commit: report.commit,
|
|
198
|
+
runId: report.runId,
|
|
199
|
+
kills: report.kills,
|
|
200
|
+
full: report.full,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
function tallyCommand(flags) {
|
|
204
|
+
return tallyRun(path.resolve(requireText(flags, "out")));
|
|
205
|
+
}
|
|
206
|
+
async function ledgerCommand(flags) {
|
|
207
|
+
const packageDir = path.resolve(requireText(flags, "package"));
|
|
208
|
+
const outDir = path.resolve(requireText(flags, "out"));
|
|
209
|
+
const repoDir = path.resolve(text(flags, "repo") ?? gitRoot(packageDir));
|
|
210
|
+
const report = buildLedger({
|
|
211
|
+
packageDir,
|
|
212
|
+
repoDir,
|
|
213
|
+
commit: text(flags, "commit") ?? "HEAD",
|
|
214
|
+
outDir,
|
|
215
|
+
srcDir: text(flags, "src") ?? "src",
|
|
216
|
+
testsDir: text(flags, "tests-dir") ?? "tests",
|
|
217
|
+
maxLines: int(flags, "max-lines") ?? 300,
|
|
218
|
+
maxCommits: int(flags, "max-commits") ?? null,
|
|
219
|
+
});
|
|
220
|
+
return {
|
|
221
|
+
schemaVersion: SCHEMA_VERSION,
|
|
222
|
+
ok: report.ok,
|
|
223
|
+
command: "ledger.build",
|
|
224
|
+
summary: report.summary,
|
|
225
|
+
next: report.next,
|
|
226
|
+
nextCall: report.ok ? report.nextCall : null,
|
|
227
|
+
clean: report.clean,
|
|
228
|
+
hand: report.hand,
|
|
229
|
+
handmade: report.handmade,
|
|
230
|
+
skipped: report.skipped,
|
|
231
|
+
handNeeded: report.handNeeded,
|
|
232
|
+
rest: report.rest,
|
|
233
|
+
commit: report.commit,
|
|
234
|
+
full: report.full,
|
|
235
|
+
historyBudgetHit: report.historyBudgetHit,
|
|
236
|
+
commitsScanned: report.commitsScanned,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
function matrixCommand(flags) {
|
|
240
|
+
const scoresPath = requireText(flags, "scores");
|
|
241
|
+
const directPath = requireText(flags, "direct");
|
|
242
|
+
const outDir = path.resolve(requireText(flags, "out"));
|
|
243
|
+
const killsPath = text(flags, "kills");
|
|
244
|
+
return reportMatrix({
|
|
245
|
+
scores: readFileSync(scoresPath, "utf8"),
|
|
246
|
+
direct: readFileSync(directPath, "utf8"),
|
|
247
|
+
patchDirs: texts(flags, "patches").map((dir) => path.resolve(dir)),
|
|
248
|
+
outDir,
|
|
249
|
+
kills: killsPath ? readKills(killsPath) : new Map(),
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
function goldenCommand(flags) {
|
|
253
|
+
const recordedPath = requireText(flags, "recorded");
|
|
254
|
+
const recordedText = readFileSync(recordedPath, "utf8");
|
|
255
|
+
const updateRaw = text(flags, "update");
|
|
256
|
+
if (updateRaw && updateRaw !== "wording" && updateRaw !== "all")
|
|
257
|
+
throw new Error("--update must be wording or all");
|
|
258
|
+
const result = checkGolden({
|
|
259
|
+
recorded: asTable(recordedText),
|
|
260
|
+
actual: readGolden(requireText(flags, "actual")),
|
|
261
|
+
update: updateRaw === "wording" || updateRaw === "all" ? updateRaw : null,
|
|
262
|
+
message: text(flags, "message") ?? "",
|
|
263
|
+
roots: texts(flags, "root").map((dir) => path.resolve(dir)),
|
|
264
|
+
});
|
|
265
|
+
if (result.write)
|
|
266
|
+
writeGolden(recordedPath, result.table, goldenShape(recordedText));
|
|
267
|
+
return result.envelope;
|
|
268
|
+
}
|
|
269
|
+
function gapFixCommand(flags) {
|
|
270
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
271
|
+
const line = int(flags, "line");
|
|
272
|
+
if (line === undefined || line < 1)
|
|
273
|
+
throw new Error("--line must be a positive integer");
|
|
274
|
+
const state = fixGap(loadState(dir), {
|
|
275
|
+
id: requireText(flags, "id"),
|
|
276
|
+
file: requireText(flags, "file"),
|
|
277
|
+
line,
|
|
278
|
+
guard: requireText(flags, "guard"),
|
|
279
|
+
commit: requireText(flags, "commit"),
|
|
280
|
+
root: path.resolve(text(flags, "root") ?? dir),
|
|
281
|
+
});
|
|
282
|
+
saveState(dir, state);
|
|
283
|
+
const gap = state.gaps.find((item) => item.id === requireText(flags, "id"));
|
|
284
|
+
return {
|
|
285
|
+
schemaVersion: SCHEMA_VERSION,
|
|
286
|
+
ok: true,
|
|
287
|
+
command: "gap.fix",
|
|
288
|
+
summary: `${gap?.id} is fixed, guarded by ${gap?.guard}.`,
|
|
289
|
+
next: "Run status.",
|
|
290
|
+
nextCall: { argv: ["status", "--state", dir] },
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
function gapRevertCommand(flags) {
|
|
294
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
295
|
+
const reverted = revertGap(loadState(dir), requireText(flags, "id"));
|
|
296
|
+
saveState(dir, reverted.state);
|
|
297
|
+
return {
|
|
298
|
+
schemaVersion: SCHEMA_VERSION,
|
|
299
|
+
ok: true,
|
|
300
|
+
command: "gap.revert",
|
|
301
|
+
summary: reverted.text,
|
|
302
|
+
next: reverted.text,
|
|
303
|
+
nextCall: null,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
function guardCommand(flags) {
|
|
307
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
308
|
+
const guard = requireText(flags, "guard");
|
|
309
|
+
parseGuard(guard);
|
|
310
|
+
const result = guardAllows(loadState(dir), guard, text(flags, "message") ?? "", path.resolve(text(flags, "root") ?? dir));
|
|
311
|
+
return {
|
|
312
|
+
schemaVersion: SCHEMA_VERSION,
|
|
313
|
+
ok: result.ok,
|
|
314
|
+
command: "guard.check",
|
|
315
|
+
summary: result.summary,
|
|
316
|
+
next: result.ok ? "The guard check passed." : "Add a Guard-Change trailer naming the guard, or restore it.",
|
|
317
|
+
nextCall: null,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
function findingsCommand(flags) {
|
|
321
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
322
|
+
const status = text(flags, "status");
|
|
323
|
+
if (status !== "open" && status !== "fixed" && status !== "equivalent" && status !== "wont-fix") {
|
|
324
|
+
throw new Error("--status must be open, fixed, equivalent, or wont-fix");
|
|
325
|
+
}
|
|
326
|
+
const id = requireText(flags, "id");
|
|
327
|
+
const reason = requireText(flags, "reason");
|
|
328
|
+
const finding = {
|
|
329
|
+
id,
|
|
330
|
+
status,
|
|
331
|
+
reason,
|
|
332
|
+
};
|
|
333
|
+
const agent = text(flags, "agent");
|
|
334
|
+
if (agent)
|
|
335
|
+
finding.agent = agent;
|
|
336
|
+
const file = text(flags, "file");
|
|
337
|
+
const line = int(flags, "line");
|
|
338
|
+
if (file && line !== undefined) {
|
|
339
|
+
const source = readSourceLine(path.resolve(text(flags, "root") ?? dir, file), line);
|
|
340
|
+
if (source === null)
|
|
341
|
+
throw new Error("decision line does not exist");
|
|
342
|
+
finding.file = file;
|
|
343
|
+
finding.line = line;
|
|
344
|
+
finding.hash = lineHash(source);
|
|
345
|
+
const state = loadState(dir);
|
|
346
|
+
const decisions = state.decisions.filter((item) => item.id !== id);
|
|
347
|
+
decisions.push({ id, file, line, hash: finding.hash, status, reason });
|
|
348
|
+
saveState(dir, { ...state, decisions });
|
|
349
|
+
}
|
|
350
|
+
appendFinding(dir, finding);
|
|
351
|
+
return {
|
|
352
|
+
schemaVersion: SCHEMA_VERSION,
|
|
353
|
+
ok: true,
|
|
354
|
+
command: "findings.add",
|
|
355
|
+
summary: `Recorded ${id} as ${status}.`,
|
|
356
|
+
next: "Run status.",
|
|
357
|
+
nextCall: null,
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
function statusCommand(flags) {
|
|
361
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
362
|
+
const root = path.resolve(text(flags, "root") ?? dir);
|
|
363
|
+
const pagePath = path.resolve(text(flags, "page") ?? "PROBATIO.md");
|
|
364
|
+
const scoresPath = text(flags, "scores");
|
|
365
|
+
const score = scoresPath ? JSON.parse(readFileSync(scoresPath, "utf8")) : null;
|
|
366
|
+
const view = statusEnvelope(dir, root, score, text(flags, "agent") ?? null);
|
|
367
|
+
mkdirSync(path.dirname(pagePath), { recursive: true });
|
|
368
|
+
writeFileSync(pagePath, view.page);
|
|
369
|
+
return view.envelope;
|
|
370
|
+
}
|
|
371
|
+
function sealCommand(flags) {
|
|
372
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
373
|
+
const from = text(flags, "from");
|
|
374
|
+
let ids = texts(flags, "id");
|
|
375
|
+
if (from) {
|
|
376
|
+
const parsed = JSON.parse(readFileSync(from, "utf8"));
|
|
377
|
+
const list = Array.isArray(parsed) ? parsed : parsed.ids ?? [];
|
|
378
|
+
const fraction = Number(text(flags, "fraction") ?? "0.2");
|
|
379
|
+
if (!Number.isFinite(fraction) || fraction < 0 || fraction > 1)
|
|
380
|
+
throw new Error("--fraction must be between 0 and 1");
|
|
381
|
+
ids = chooseSealed(list, fraction, int(flags, "seed") ?? 20261004);
|
|
382
|
+
}
|
|
383
|
+
sealIds(dir, ids);
|
|
384
|
+
return {
|
|
385
|
+
schemaVersion: SCHEMA_VERSION,
|
|
386
|
+
ok: true,
|
|
387
|
+
command: "seal",
|
|
388
|
+
summary: `Sealed ${ids.length} ids.`,
|
|
389
|
+
next: "status, PROBATIO.md, and the queue omit them.",
|
|
390
|
+
nextCall: null,
|
|
391
|
+
sealedCount: ids.length,
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
function queueSeedCommand(flags) {
|
|
395
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
396
|
+
const written = seedQueue(dir, texts(flags, "id"));
|
|
397
|
+
return {
|
|
398
|
+
schemaVersion: SCHEMA_VERSION,
|
|
399
|
+
ok: true,
|
|
400
|
+
command: "queue.seed",
|
|
401
|
+
summary: `Seeded ${written.length} queue items.`,
|
|
402
|
+
next: "Claim one.",
|
|
403
|
+
nextCall: null,
|
|
404
|
+
seeded: written,
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function queueClaimCommand(flags) {
|
|
408
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
409
|
+
const now = int(flags, "now") ?? Date.now();
|
|
410
|
+
const lease = int(flags, "lease-ms") ?? 600_000;
|
|
411
|
+
const result = claimItem(dir, requireText(flags, "id"), requireText(flags, "agent"), now, lease);
|
|
412
|
+
return {
|
|
413
|
+
schemaVersion: SCHEMA_VERSION,
|
|
414
|
+
ok: result.ok,
|
|
415
|
+
command: "queue.claim",
|
|
416
|
+
summary: result.summary,
|
|
417
|
+
next: result.ok ? "Work the claim, then check-kill it." : "Pick another item.",
|
|
418
|
+
nextCall: null,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
function queueReapCommand(flags) {
|
|
422
|
+
const dir = path.resolve(requireText(flags, "state"));
|
|
423
|
+
const now = int(flags, "now") ?? Date.now();
|
|
424
|
+
const returned = reapClaims(dir, now);
|
|
425
|
+
return {
|
|
426
|
+
schemaVersion: SCHEMA_VERSION,
|
|
427
|
+
ok: true,
|
|
428
|
+
command: "queue.reap",
|
|
429
|
+
summary: `Returned ${returned.length} expired claims.`,
|
|
430
|
+
next: returned.length === 0 ? "Nothing expired." : "The expired claims are back in the queue.",
|
|
431
|
+
nextCall: null,
|
|
432
|
+
returned,
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
async function checkKillCommand(flags, id) {
|
|
436
|
+
if (!id)
|
|
437
|
+
throw new Error("check-kill needs a mutant id");
|
|
438
|
+
const packageDir = path.resolve(requireText(flags, "package"));
|
|
439
|
+
const patchDirs = texts(flags, "patches").map((dir) => path.resolve(dir));
|
|
440
|
+
if (patchDirs.length === 0)
|
|
441
|
+
throw new Error("--patches is required");
|
|
442
|
+
const build = buildCommand(flags);
|
|
443
|
+
return checkKill(id, {
|
|
444
|
+
packageDir,
|
|
445
|
+
repoDir: path.resolve(text(flags, "repo") ?? gitRoot(packageDir)),
|
|
446
|
+
commit: text(flags, "commit") ?? "HEAD",
|
|
447
|
+
patchDirs,
|
|
448
|
+
outDir: path.resolve(requireText(flags, "out")),
|
|
449
|
+
direction: "forward",
|
|
450
|
+
workers: 1,
|
|
451
|
+
concurrency: 1,
|
|
452
|
+
maxMinutes: int(flags, "max-minutes") ?? 3,
|
|
453
|
+
maxMutants: null,
|
|
454
|
+
build,
|
|
455
|
+
testsDir: text(flags, "tests-dir") ?? "tests",
|
|
456
|
+
unset: ["SOLARI_API_KEY", "AUSPEX_LIVE"],
|
|
457
|
+
suiteTimeoutMs: int(flags, "suite-timeout-ms") ?? 60_000,
|
|
458
|
+
testTimeoutMs: int(flags, "test-timeout-ms") ?? 30_000,
|
|
459
|
+
confirm: bool(flags, "confirm", true),
|
|
460
|
+
affected: false,
|
|
461
|
+
agent: text(flags, "agent") ?? null,
|
|
462
|
+
onProgress: (line) => process.stderr.write(`${line}\n`),
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
async function verifyCommand(flags) {
|
|
466
|
+
const packageDir = path.resolve(requireText(flags, "package"));
|
|
467
|
+
return verifyChange({
|
|
468
|
+
packageDir,
|
|
469
|
+
repoDir: path.resolve(text(flags, "repo") ?? gitRoot(packageDir)),
|
|
470
|
+
outDir: path.resolve(requireText(flags, "out")),
|
|
471
|
+
base: text(flags, "base") ?? "HEAD",
|
|
472
|
+
commit: text(flags, "commit") ?? "HEAD",
|
|
473
|
+
maxMutants: int(flags, "max-mutants") ?? 4,
|
|
474
|
+
maxMinutes: int(flags, "max-minutes") ?? 3,
|
|
475
|
+
maxTests: int(flags, "max-tests") ?? null,
|
|
476
|
+
onProgress: (line) => process.stderr.write(`${line}\n`),
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
function gitRoot(cwd) {
|
|
480
|
+
const result = git(cwd, ["rev-parse", "--show-toplevel"]);
|
|
481
|
+
if (result.status !== 0)
|
|
482
|
+
throw new Error("package is not inside a git repository. Pass --repo.");
|
|
483
|
+
return result.stdout.trim();
|
|
484
|
+
}
|
|
485
|
+
function finish(envelope, human) {
|
|
486
|
+
process.stdout.write(render(envelope, human));
|
|
487
|
+
process.exit(envelope.ok ? 0 : 1);
|
|
488
|
+
}
|
|
489
|
+
/** Omitted `--build` and `--no-build` both mean no build. `--build <cmd>` opts in. */
|
|
490
|
+
function buildCommand(flags) {
|
|
491
|
+
const values = flags.get("build");
|
|
492
|
+
if (!values || values.length === 0)
|
|
493
|
+
return null;
|
|
494
|
+
const last = values[values.length - 1];
|
|
495
|
+
if (typeof last !== "string")
|
|
496
|
+
return null;
|
|
497
|
+
const parts = last.split(" ").filter(Boolean);
|
|
498
|
+
return parts.length > 0 ? parts : null;
|
|
499
|
+
}
|
package/dist/contract.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
export const SCHEMA_VERSION = 1;
|
|
3
|
+
/** Replace the home directory so a result never carries a user's name or home path. */
|
|
4
|
+
export function scrub(value, home = homedir()) {
|
|
5
|
+
if (typeof value === "string") {
|
|
6
|
+
if (!home || !value.includes(home))
|
|
7
|
+
return value;
|
|
8
|
+
return value.split(home).join("~");
|
|
9
|
+
}
|
|
10
|
+
if (Array.isArray(value))
|
|
11
|
+
return value.map((item) => scrub(item, home));
|
|
12
|
+
if (value && typeof value === "object") {
|
|
13
|
+
const out = {};
|
|
14
|
+
for (const [key, item] of Object.entries(value))
|
|
15
|
+
out[key] = scrub(item, home);
|
|
16
|
+
return out;
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
export function render(envelope, human, home = homedir()) {
|
|
21
|
+
const clean = scrub(envelope, home);
|
|
22
|
+
if (!human)
|
|
23
|
+
return `${JSON.stringify(clean, null, 2)}\n`;
|
|
24
|
+
const lines = [clean.summary, clean.next];
|
|
25
|
+
const call = clean.nextCall;
|
|
26
|
+
if (call && Array.isArray(call.argv))
|
|
27
|
+
lines.push(`next call: probatio ${call.argv.join(" ")}`);
|
|
28
|
+
return `${lines.filter((line) => line.length > 0).join("\n")}\n`;
|
|
29
|
+
}
|