testguard-cli 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +196 -17
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +316 -0
- package/spec/README.md +27 -2
- package/spec/lib/authoring-input.mjs +40 -0
- package/spec/lib/origins.mjs +56 -0
- package/spec/lib/validate.mjs +129 -1
- package/spec/schemas/annotations.schema.json +65 -0
- package/spec/schemas/authoring-input.schema.json +67 -0
- package/spec/schemas/brief.schema.json +2 -0
- package/spec/schemas/claims.schema.json +8 -8
- package/spec/schemas/common.schema.json +73 -0
- package/spec/schemas/evidence.schema.json +41 -8
- package/spec/schemas/status.schema.json +31 -2
- package/src/brief/brief.mjs +11 -4
- package/src/claims/annotation-placement.mjs +357 -0
- package/src/claims/annotations.mjs +61 -2
- package/src/cli.mjs +40 -5
- package/src/commands/admit.mjs +21 -3
- package/src/commands/annotations.mjs +67 -0
- package/src/commands/claims.mjs +46 -6
- package/src/commands/probe.mjs +50 -7
- package/src/commands/replay.mjs +7 -0
- package/src/commands/scaffold.mjs +49 -2
- package/src/commands/sweep.mjs +7 -0
- package/src/evidence/writer.mjs +12 -4
- package/src/gate/changed.mjs +5 -1
- package/src/git.mjs +4 -4
- package/src/init/init.mjs +6 -0
- package/src/init/templates/SKILL.md +78 -0
- package/src/probe/attribution.mjs +19 -0
- package/src/probe/contention.mjs +20 -2
- package/src/probe/cost.mjs +22 -9
- package/src/probe/inject.mjs +2 -6
- package/src/probe/origin-policy.mjs +69 -0
- package/src/probe/policy-freshness.mjs +74 -0
- package/src/probe/probe.mjs +74 -45
- package/src/probe/runners/discovery.mjs +2 -0
- package/src/probe/runners/jest.mjs +2 -2
- package/src/probe/runners/lifecycle.mjs +58 -0
- package/src/probe/runners/playwright.mjs +6 -2
- package/src/probe/runners/python.mjs +10 -4
- package/src/probe/runners/shared.mjs +12 -6
- package/src/probe/runners/vitest.mjs +3 -3
- package/src/probe/universe.mjs +47 -0
- package/src/probe/worktree.mjs +3 -1
- package/src/render.mjs +17 -1
- package/src/replay/replay.mjs +8 -4
- package/src/scaffold/admission.mjs +117 -0
- package/src/scaffold/append.mjs +41 -0
- package/src/scaffold/apply-append.mjs +71 -0
- package/src/scaffold/fix-input.mjs +119 -0
- package/src/scaffold/fix-paths.mjs +90 -0
- package/src/scaffold/intent.mjs +2 -0
- package/src/scaffold/open-append.mjs +44 -0
- package/src/scaffold/plan-append.mjs +37 -0
- package/src/scaffold/plan-intent-input.mjs +38 -0
- package/src/scaffold/prepare-append.mjs +46 -0
- package/src/scaffold/request.mjs +30 -0
- package/src/scaffold/scaffold.mjs +23 -10
- package/src/status/status.mjs +42 -10
- package/src/sweep/pool.mjs +33 -0
- package/src/sweep/sweep.mjs +9 -4
|
@@ -9,8 +9,66 @@ TestGuard injects the faults a project's claims forbid and reports every one
|
|
|
9
9
|
the tests fail to detect. It is a **claim verifier**, not a test generator.
|
|
10
10
|
You operate it through one loop and one source of truth.
|
|
11
11
|
|
|
12
|
+
## Update advice (once per session, never an automatic install)
|
|
13
|
+
|
|
14
|
+
Before the operating loop, identify the CLI you will actually run: prefer the
|
|
15
|
+
project's installed `node_modules/.bin/testguard`, then the repository-root
|
|
16
|
+
install for a nested project, then `testguard` on PATH. Run that exact CLI with
|
|
17
|
+
`--version`, and use the same executable for verification. Do not use `npx` to
|
|
18
|
+
discover a version: it can fetch or install a different tool. When developing
|
|
19
|
+
TestGuard itself, use `node cli/testguard.mjs`; do not replace a source checkout
|
|
20
|
+
with a published version to make validation pass.
|
|
21
|
+
|
|
22
|
+
If the user's network policy permits registry access, make one bounded,
|
|
23
|
+
read-only check per session (not per command or project):
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Compare valid stable versions by numeric SemVer major, minor, then patch, not
|
|
30
|
+
lexicographically and not merely for inequality. Do not recommend a downgrade
|
|
31
|
+
or replacing a prerelease/development checkout. If the registry result is
|
|
32
|
+
missing, malformed, unavailable, or network access is prohibited, continue the
|
|
33
|
+
operating loop without claiming the installation is up to date. No retries or
|
|
34
|
+
repeated prompts after the user declines within this session.
|
|
35
|
+
|
|
36
|
+
For a newer stable version, say: "Using TestGuard X from <resolved executable>;
|
|
37
|
+
Y is available. Would you like to update?" Respect project pins, compatibility,
|
|
38
|
+
company policy and the original package manager; a registry's `latest` tag is
|
|
39
|
+
not proof that an upgrade is safe. Never install automatically, modify a
|
|
40
|
+
lockfile or CI pin, or switch between local and global installs without approval.
|
|
41
|
+
After an approved update, verify the same executable's version again.
|
|
42
|
+
|
|
43
|
+
Keep this advice in the AI conversation, separate from CLI JSON, evidence,
|
|
44
|
+
verdicts and exit codes. The CLI and session-start hook remain offline; the AI
|
|
45
|
+
performs this optional check, not TestGuard. Instructions cannot guarantee that
|
|
46
|
+
every AI checks for updates.
|
|
47
|
+
|
|
48
|
+
Package upgrades do not refresh an existing copied skill. Preserve customized
|
|
49
|
+
instructions: compare this file with the upgraded package's
|
|
50
|
+
`src/init/templates/SKILL.md` and merge approved changes. Use `testguard init
|
|
51
|
+
--force` only with explicit permission to replace the whole skill; review the
|
|
52
|
+
diff afterward. An ordinary `init` preserves existing skills and managed
|
|
53
|
+
sections already listing this project.
|
|
54
|
+
|
|
12
55
|
## The one source of truth
|
|
13
56
|
|
|
57
|
+
Missing source-link advisories in claims/status never replace `next`, change
|
|
58
|
+
verdicts or prove intent. Exact anchors remain checked without annotations.
|
|
59
|
+
Origin summaries are likewise informational: status counts current declared
|
|
60
|
+
claims; evidence/briefs count recorded declarations and records, including
|
|
61
|
+
mixed sources. Source kinds are not authenticated independence. Never use an
|
|
62
|
+
origin label to waive a survivor.
|
|
63
|
+
Optional source links: `testguard claims --annotate --claim <ID>` is a read-only
|
|
64
|
+
file-header preview. Only with explicit permission to edit source, rerun with
|
|
65
|
+
`--apply`; each invocation recomputes its plan. Review refusals and partial
|
|
66
|
+
failures, and retain the reported private recovery originals. Never automatically
|
|
67
|
+
remove an existing lock or roll back over user edits. Placement is file-level,
|
|
68
|
+
not symbol ownership, verified behavior or authenticated intent. Missing links
|
|
69
|
+
do not disable exact fault-anchor checks. Authoring never replaces the status
|
|
70
|
+
operating loop or writes canonical claims/evidence/baselines.
|
|
71
|
+
|
|
14
72
|
```bash
|
|
15
73
|
testguard status --json
|
|
16
74
|
```
|
|
@@ -33,6 +91,18 @@ Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) ·
|
|
|
33
91
|
|
|
34
92
|
## Every change needs a claim
|
|
35
93
|
|
|
94
|
+
Scaffold proposes mechanical faults, not intended behavior. New drafts declare
|
|
95
|
+
`source.kind: inferred` even when grouped by an annotation. Before adopting a
|
|
96
|
+
draft, ask for the intended observable behavior from a requirement, ADR, bug or
|
|
97
|
+
incident independently of the implementation: inputs, expected outcome and
|
|
98
|
+
forbidden outcome. Record the supplied source kind/ref; when unavailable, keep
|
|
99
|
+
inferred and say the intent remains unestablished. Do not promote origin because
|
|
100
|
+
an annotation exists or tests pass. References are opaque, not authenticated;
|
|
101
|
+
verification commands never fetch them. Preserve existing supplied metadata.
|
|
102
|
+
Review each proposal against that intent, then probe. Do not paraphrase the code
|
|
103
|
+
and call that independent intent, or automatically move drafts into canonical
|
|
104
|
+
claims.
|
|
105
|
+
|
|
36
106
|
`probe` verifies only the claims that exist; it is silent about unclaimed
|
|
37
107
|
code by construction. So, for every source file you create or change:
|
|
38
108
|
|
|
@@ -65,6 +135,14 @@ code by construction. So, for every source file you create or change:
|
|
|
65
135
|
|
|
66
136
|
## The fix loop
|
|
67
137
|
|
|
138
|
+
For independently supplied intent in an existing disposable draft, use
|
|
139
|
+
`testguard scaffold <source...> --into <draft.json> --claim <existing-ID> --json`
|
|
140
|
+
to preview appended faults without writes. Omit `--json` only when explicitly
|
|
141
|
+
authorized to update that draft; private recovery is retained. Never use a
|
|
142
|
+
canonical claims/evidence/baseline path. Preserve intended behavior and declared
|
|
143
|
+
origin; generated faults are not independent intent or verification. Review the
|
|
144
|
+
faults and probe the selected draft before promoting it to canonical claims.
|
|
145
|
+
|
|
68
146
|
1. `testguard status --json` → take `next.target`.
|
|
69
147
|
2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
|
|
70
148
|
3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
|
|
@@ -11,8 +11,17 @@
|
|
|
11
11
|
* worktree is. Everything it needs is passed in.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
15
|
+
|
|
14
16
|
const isKill = (r) => r.outcome === 'fail' && (r.assertionFailures ?? 0) > 0;
|
|
15
17
|
|
|
18
|
+
/** Compare only the claim metadata actually recorded in evidence. */
|
|
19
|
+
export function sameClaimMetadata(recorded, current) {
|
|
20
|
+
if (!recorded || !current) return false;
|
|
21
|
+
const metadata = ({ id, statement, severity, source, producedBy }) => ({ id, statement, severity, source, producedBy });
|
|
22
|
+
return isDeepStrictEqual(metadata(recorded), metadata(current));
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
/**
|
|
17
26
|
* Fold one escalation run into the running attribution state.
|
|
18
27
|
*
|
|
@@ -109,11 +118,21 @@ export function subjectOf(fault, sha256) {
|
|
|
109
118
|
* The cost of re-measuring is a run; the cost of the other direction is a
|
|
110
119
|
* verdict nobody measured.
|
|
111
120
|
*/
|
|
121
|
+
/** Select declarations without resolving globs or running discovery. */
|
|
122
|
+
export function defenderSelection(claim, fault) {
|
|
123
|
+
if (Object.hasOwn(fault, 'defendedBy')) return { requested: [...fault.defendedBy], selectionSource: 'fault' };
|
|
124
|
+
if (claim.defendedBy?.length) return { requested: [...claim.defendedBy], selectionSource: 'claim' };
|
|
125
|
+
return { requested: [], selectionSource: 'discovery' };
|
|
126
|
+
}
|
|
127
|
+
|
|
112
128
|
export function isReusable(prior, current) {
|
|
113
129
|
if (!prior || !prior.subject?.contentHash) return false;
|
|
130
|
+
if (current.claim && !sameClaimMetadata(prior.claim, current.claim)) return false;
|
|
114
131
|
// A record that exists because the probe threw is a statement about the run,
|
|
115
132
|
// not about the code. Reusing it would make one disturbed run permanent.
|
|
116
133
|
if (prior.detail?.reason === 'probe-error') return false;
|
|
134
|
+
const origin = prior.defenders.selectionSource ?? (prior.defenders.requested.length ? 'claim' : 'discovery');
|
|
135
|
+
if (current.selectionSource && origin !== current.selectionSource) return false;
|
|
117
136
|
const same = (x, y) => JSON.stringify(x) === JSON.stringify(y);
|
|
118
137
|
return prior.inputs.targetHash === current.inputs.targetHash
|
|
119
138
|
&& same(prior.inputs.defenderHashes, current.inputs.defenderHashes)
|
package/src/probe/contention.mjs
CHANGED
|
@@ -37,9 +37,13 @@ export function processList({ platform = process.platform, run = spawnSync } = {
|
|
|
37
37
|
try {
|
|
38
38
|
const r = platform === 'win32'
|
|
39
39
|
? run('wmic', ['process', 'get', 'ProcessId,CommandLine', '/format:csv'], { encoding: 'utf8', timeout: 5000 })
|
|
40
|
-
: run('ps', ['-Ao', 'pid=,args='], { encoding: 'utf8', timeout: 5000 });
|
|
40
|
+
: run('ps', ['-Ao', 'pid=,ppid=,stat=,args='], { encoding: 'utf8', timeout: 5000 });
|
|
41
41
|
if (r.status !== 0 || !r.stdout) return [];
|
|
42
42
|
return r.stdout.split('\n').map((line) => {
|
|
43
|
+
if (platform !== 'win32') {
|
|
44
|
+
const row = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line);
|
|
45
|
+
if (row) return { pid: Number(row[1]), ppid: Number(row[2]), stat: row[3], command: row[4].trim() };
|
|
46
|
+
}
|
|
43
47
|
const m = platform === 'win32' ? /^[^,]*,(.*),(\d+)\s*$/.exec(line) : /^\s*(\d+)\s+(.*)$/.exec(line);
|
|
44
48
|
if (!m) return null;
|
|
45
49
|
const [pid, command] = platform === 'win32' ? [m[2], m[1]] : [m[1], m[2]];
|
|
@@ -57,8 +61,22 @@ export function processList({ platform = process.platform, run = spawnSync } = {
|
|
|
57
61
|
*/
|
|
58
62
|
export function detectContention({ self = process.pid, alive = defaultAlive, ...opts } = {}) {
|
|
59
63
|
const rows = processList(opts);
|
|
64
|
+
const owned = new Set([self]);
|
|
65
|
+
let changed = true;
|
|
66
|
+
while (changed) {
|
|
67
|
+
changed = false;
|
|
68
|
+
for (const row of rows) {
|
|
69
|
+
if (owned.has(row.ppid) && !owned.has(row.pid)) { owned.add(row.pid); changed = true; }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const ancestors = new Set();
|
|
73
|
+
let parent = rows.find((row) => row.pid === self)?.ppid;
|
|
74
|
+
while (parent && !ancestors.has(parent)) {
|
|
75
|
+
ancestors.add(parent);
|
|
76
|
+
parent = rows.find((row) => row.pid === parent)?.ppid;
|
|
77
|
+
}
|
|
60
78
|
const runners = rows
|
|
61
|
-
.filter((p) => p.pid
|
|
79
|
+
.filter((p) => !owned.has(p.pid) && !ancestors.has(p.pid) && !p.stat?.startsWith('Z') && RUNNER_RE.test(p.command) && (p.ppid !== undefined || !SELF_RE.test(p.command)))
|
|
62
80
|
// `ps` lists the grep/ps itself and any shell wrapper; a runner's command
|
|
63
81
|
// line always names its own binary, so require it to look like an exec.
|
|
64
82
|
.filter((p) => !/^\s*(ps|grep|wmic)\b/.test(p.command))
|
package/src/probe/cost.mjs
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* already recorded. Nothing re-measures, nothing spawns, nothing guesses: a
|
|
13
13
|
* cost report costs the price of reading one JSON file.
|
|
14
14
|
*
|
|
15
|
-
* ATTRIBUTION, HONESTLY. Runs execute a
|
|
15
|
+
* ATTRIBUTION, HONESTLY. Runs execute a fault's selected defender set together,
|
|
16
16
|
* so the runner never says how much of a run belonged to which file. Per-file
|
|
17
17
|
* `ms` is therefore the time of every run that *included* that file: an upper
|
|
18
18
|
* bound on what removing it could save, and, when a claim names several
|
|
@@ -70,15 +70,16 @@ export function defenderCosts(records) {
|
|
|
70
70
|
for (const r of records) {
|
|
71
71
|
const c = recordCost(r);
|
|
72
72
|
for (const file of r.defenders?.resolved ?? []) {
|
|
73
|
-
const e = by.get(file) ?? { file, ms: 0, runs: 0, claims: new Set() };
|
|
73
|
+
const e = by.get(file) ?? { file, ms: 0, runs: 0, claims: new Set(), faults: new Set() };
|
|
74
74
|
e.ms += c.ms;
|
|
75
75
|
e.runs += c.runs;
|
|
76
76
|
e.claims.add(r.claim.id);
|
|
77
|
+
e.faults.add(`${r.claim.id}/${r.subject.id}`);
|
|
77
78
|
by.set(file, e);
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
81
|
return [...by.values()]
|
|
81
|
-
.map((e) => ({ ...e, claims: [...e.claims].sort() }))
|
|
82
|
+
.map((e) => ({ ...e, claims: [...e.claims].sort(), faults: [...e.faults].sort() }))
|
|
82
83
|
.sort((a, b) => b.ms - a.ms || a.file.localeCompare(b.file));
|
|
83
84
|
}
|
|
84
85
|
|
|
@@ -86,22 +87,25 @@ export function defenderCosts(records) {
|
|
|
86
87
|
* The whole cost model for one evidence document.
|
|
87
88
|
*
|
|
88
89
|
* `sharedDefenders` is the actionable part: a file named by more than one
|
|
89
|
-
*
|
|
90
|
+
* fault identity, including siblings within one claim. Splitting the
|
|
90
91
|
* behaviour it defends into a unit test that runs in milliseconds is what
|
|
91
92
|
* turns a twenty-four-minute gate into a one-minute one, and until a report
|
|
92
93
|
* names the file nobody knows which test to split.
|
|
93
94
|
*/
|
|
94
|
-
export function costReport(records) {
|
|
95
|
+
export function costReport(records, { run } = {}) {
|
|
95
96
|
const claims = claimCosts(records);
|
|
96
97
|
const defenders = defenderCosts(records);
|
|
97
98
|
return {
|
|
99
|
+
...(run?.measurements ? { measurements: { ...run.measurements } } : {}),
|
|
98
100
|
totalMs: claims.reduce((a, c) => a + c.ms, 0),
|
|
99
101
|
totalRuns: claims.reduce((a, c) => a + c.runs, 0),
|
|
100
102
|
reusedRecords: records.filter((r) => r.reusedFrom).length,
|
|
101
103
|
records: records.length,
|
|
102
104
|
claims,
|
|
105
|
+
faults: records.map((r) => ({ claimId: r.claim.id, faultId: r.subject.id, ...recordCost(r), defenders: [...(r.defenders?.resolved ?? [])], ...(r.defenders?.selectionSource ? { selectionSource: r.defenders.selectionSource } : {}) }))
|
|
106
|
+
.sort((a, b) => b.ms - a.ms || a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId)),
|
|
103
107
|
defenders,
|
|
104
|
-
sharedDefenders: defenders.filter((d) => d.
|
|
108
|
+
sharedDefenders: defenders.filter((d) => d.faults.length > 1),
|
|
105
109
|
};
|
|
106
110
|
}
|
|
107
111
|
|
|
@@ -115,6 +119,10 @@ export const formatMs = secs;
|
|
|
115
119
|
*/
|
|
116
120
|
export function renderCost(report, { limit = 10 } = {}) {
|
|
117
121
|
const out = [];
|
|
122
|
+
if (report.measurements) {
|
|
123
|
+
const m = report.measurements;
|
|
124
|
+
out.push(`actual elapsed ${secs(m.elapsedMs)}; fresh runners ${secs(m.runnerMs)} across ${m.runnerInvocations} invocations; setup/discovery/other overhead ${secs(m.overheadMs)}`);
|
|
125
|
+
}
|
|
118
126
|
if (!report.records) return 'no evidence records — run `testguard probe` first; cost is read back from the runs it records.';
|
|
119
127
|
out.push(`${report.records} fault record${report.records === 1 ? '' : 's'} cost ${secs(report.totalMs)} across ${report.totalRuns} defender runs${report.reusedRecords ? ` (${report.reusedRecords} reused; their runs are what they cost when reuse does not apply)` : ''}.`);
|
|
120
128
|
out.push('');
|
|
@@ -123,19 +131,24 @@ export function renderCost(report, { limit = 10 } = {}) {
|
|
|
123
131
|
out.push(` ${secs(c.ms).padStart(7)} ${c.claimId.padEnd(34)} ${c.faults} fault${c.faults === 1 ? ' ' : 's'} · ${c.runs} runs${c.reused ? ` · ${c.reused} reused` : ''}`);
|
|
124
132
|
}
|
|
125
133
|
if (report.claims.length > limit) out.push(` … ${report.claims.length - limit} more`);
|
|
134
|
+
if (report.faults?.length) {
|
|
135
|
+
out.push('', 'most expensive faults');
|
|
136
|
+
for (const f of report.faults.slice(0, limit)) out.push(` ${secs(f.ms).padStart(7)} ${f.claimId}/${f.faultId} · ${f.runs} runs${f.selectionSource ? ` · ${f.selectionSource}` : ''} · ${f.defenders.join(', ') || 'no defenders'}`);
|
|
137
|
+
if (report.faults.length > limit) out.push(` … ${report.faults.length - limit} more`);
|
|
138
|
+
}
|
|
126
139
|
out.push('');
|
|
127
140
|
out.push('defender files, by the time of the runs that included them');
|
|
128
|
-
out.push(' (a
|
|
141
|
+
out.push(' (a fault runs its selected defender set at once, so these overlap and do not sum to the total)');
|
|
129
142
|
for (const d of report.defenders.slice(0, limit)) {
|
|
130
143
|
out.push(` ${secs(d.ms).padStart(7)} ${d.file.padEnd(44)} ${d.claims.length} claim${d.claims.length === 1 ? '' : 's'}`);
|
|
131
144
|
}
|
|
132
145
|
if (report.defenders.length > limit) out.push(` … ${report.defenders.length - limit} more`);
|
|
133
146
|
if (report.sharedDefenders.length) {
|
|
134
147
|
out.push('');
|
|
135
|
-
out.push('shared defenders — each
|
|
148
|
+
out.push('shared defenders — each fault selects the file; durations include recorded baseline sharing');
|
|
136
149
|
for (const d of report.sharedDefenders.slice(0, limit)) {
|
|
137
150
|
out.push(` ${secs(d.ms).padStart(7)} ${d.file}`);
|
|
138
|
-
out.push(` named by ${d.
|
|
151
|
+
out.push(` named by ${d.faults.join(', ')}`);
|
|
139
152
|
}
|
|
140
153
|
out.push('');
|
|
141
154
|
out.push(' To make a probe cheaper, split the behaviour a shared defender proves into a');
|
package/src/probe/inject.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { registerCleanup } from './runners/lifecycle.mjs';
|
|
1
2
|
import { readFileSync, writeFileSync } from 'node:fs';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
|
|
@@ -19,12 +20,7 @@ function restoreAll() {
|
|
|
19
20
|
function installHandlers() {
|
|
20
21
|
if (handlersInstalled) return;
|
|
21
22
|
handlersInstalled = true;
|
|
22
|
-
|
|
23
|
-
process.on(sig, () => {
|
|
24
|
-
restoreAll();
|
|
25
|
-
process.exit(130);
|
|
26
|
-
});
|
|
27
|
-
}
|
|
23
|
+
registerCleanup(restoreAll, { priority: 1 });
|
|
28
24
|
process.on('uncaughtException', (err) => {
|
|
29
25
|
restoreAll();
|
|
30
26
|
throw err;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
+
import { validate, methodOf } from '../../spec/lib/validate.mjs';
|
|
3
|
+
import { CLAIM_SOURCE_KINDS } from '../../spec/lib/origins.mjs';
|
|
4
|
+
import { sameClaimMetadata, subjectOf } from './attribution.mjs';
|
|
5
|
+
import { sha256 } from '../util/hash.mjs';
|
|
6
|
+
|
|
7
|
+
/** Admit the planned probe option before loading claims or executing/writing. */
|
|
8
|
+
export function originPolicyRequest(values, command = 'probe') {
|
|
9
|
+
const raw = values['require-origin'];
|
|
10
|
+
if (raw === undefined) return undefined;
|
|
11
|
+
if (command !== 'probe') throw new TypeError('--require-origin is only valid on probe');
|
|
12
|
+
if (!Array.isArray(raw) || !raw.length || raw.some((v) => typeof v !== 'string')) {
|
|
13
|
+
throw new TypeError('--require-origin must name declared source kinds');
|
|
14
|
+
}
|
|
15
|
+
const kinds = raw.flatMap((v) => v.split(',').map((k) => k.trim()));
|
|
16
|
+
if (kinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) throw new TypeError('--require-origin contains an empty or unsupported kind');
|
|
17
|
+
if (values.claim !== undefined || values['allow-empty'] || values.ref !== undefined || values['ignore-dirty']) {
|
|
18
|
+
throw new TypeError('--require-origin needs a complete current-project run without --claim, --allow-empty, --ref or --ignore-dirty');
|
|
19
|
+
}
|
|
20
|
+
const n = Number(values.confirm ?? 3);
|
|
21
|
+
if (!Number.isInteger(n) || n < 3) throw new TypeError('--require-origin requires --confirm of at least 3');
|
|
22
|
+
return [...new Set(kinds)].sort();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Internal policy only: fresh must come from the existing input-binding checker. */
|
|
26
|
+
export function evaluateOriginPolicy({ claims, evidence, eligibleKinds, fresh }) {
|
|
27
|
+
if (eligibleKinds === undefined) return undefined;
|
|
28
|
+
if (!Array.isArray(eligibleKinds) || !eligibleKinds.length || eligibleKinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) {
|
|
29
|
+
throw new TypeError('origin policy needs a nonempty set of declared source kinds');
|
|
30
|
+
}
|
|
31
|
+
const eligible = [...new Set(eligibleKinds)].sort();
|
|
32
|
+
const reasons = new Set();
|
|
33
|
+
const claimsValid = validate('claims', claims).ok;
|
|
34
|
+
const evidenceValid = validate('evidence', evidence).ok;
|
|
35
|
+
if (!claimsValid) reasons.add('invalid-claims');
|
|
36
|
+
if (!evidenceValid) reasons.add('invalid-evidence');
|
|
37
|
+
if (fresh !== true) reasons.add('stale-inputs');
|
|
38
|
+
const current = claimsValid ? claims.claims : [];
|
|
39
|
+
const records = evidenceValid ? evidence.records : [];
|
|
40
|
+
const key = (claimId, faultId) => JSON.stringify([claimId, faultId]);
|
|
41
|
+
const expected = new Map(current.flatMap((claim) => claim.faults.map((fault) => [key(claim.id, fault.id), { claim, fault }])));
|
|
42
|
+
if (!current.length || !expected.size) reasons.add('empty-universe');
|
|
43
|
+
if (evidenceValid && (evidence.run.provisional === true || evidence.run.confirmRuns < 3)) reasons.add('provisional-evidence');
|
|
44
|
+
if (evidenceValid && methodOf(evidence.run) !== 'fault-injection') reasons.add('incompatible-method');
|
|
45
|
+
if (current.some((c) => c.faults.some((f) => methodOf(f) !== 'fault-injection'))) reasons.add('incompatible-method');
|
|
46
|
+
const seen = new Set();
|
|
47
|
+
for (const record of records) {
|
|
48
|
+
const identity = key(record.claim.id, record.subject.id);
|
|
49
|
+
if (seen.has(identity)) reasons.add('duplicate-record');
|
|
50
|
+
seen.add(identity);
|
|
51
|
+
const pair = expected.get(identity);
|
|
52
|
+
if (!pair) { reasons.add('extra-record'); continue; }
|
|
53
|
+
if (!sameClaimMetadata(record.claim, pair.claim)) reasons.add('claim-metadata-mismatch');
|
|
54
|
+
if (methodOf(pair.fault) !== 'fault-injection') continue;
|
|
55
|
+
const subject = subjectOf(pair.fault, sha256);
|
|
56
|
+
if (['kind', 'id', 'file', 'faultClass', 'contentHash'].some((field) => record.subject[field] !== subject[field])
|
|
57
|
+
|| !isDeepStrictEqual(record.subject.producedBy, subject.producedBy)) reasons.add('fault-mismatch');
|
|
58
|
+
}
|
|
59
|
+
if ([...expected.keys()].some((identity) => !seen.has(identity))) reasons.add('missing-record');
|
|
60
|
+
const ineligibleClaims = current.filter((c) => !eligible.includes(c.source.kind)).map((c) => c.id).sort();
|
|
61
|
+
const nonKilledFaults = [...new Map(records.filter((r) => r.verdict !== 'killed').map((r) =>
|
|
62
|
+
[key(r.claim.id, r.subject.id), { claimId: r.claim.id, faultId: r.subject.id }])).values()]
|
|
63
|
+
.sort((a, b) => a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId));
|
|
64
|
+
return {
|
|
65
|
+
state: reasons.size ? 'unavailable' : ineligibleClaims.length || nonKilledFaults.length ? 'failed' : 'passed',
|
|
66
|
+
eligibleKinds: eligible, claims: current.length, faults: expected.size,
|
|
67
|
+
ineligibleClaims, nonKilledFaults, unavailableReasons: [...reasons].sort(),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, openSync, readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, join, relative, sep } from 'node:path';
|
|
3
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
4
|
+
import { validate, methodOf } from '../../spec/lib/validate.mjs';
|
|
5
|
+
import { sha256 } from '../util/hash.mjs';
|
|
6
|
+
import { defenderSelection, isReusable, subjectOf } from './attribution.mjs';
|
|
7
|
+
import { discoverDefendersDetailed } from './discover.mjs';
|
|
8
|
+
import { resolveDefenders } from './runners/shared.mjs';
|
|
9
|
+
import { hashNativeTestUniverse } from './universe.mjs';
|
|
10
|
+
|
|
11
|
+
// Point-in-time reads, not a transaction over the whole project. Never turn a
|
|
12
|
+
// missing file into the empty-file hash that diagnostic probe records can use.
|
|
13
|
+
function currentHash(projectDir, file) {
|
|
14
|
+
if (typeof file !== 'string' || isAbsolute(file) || file.split(/[\\/]/).includes('..')) throw new Error('unsafe input path');
|
|
15
|
+
const path = join(projectDir, file);
|
|
16
|
+
const resolved = realpathSync(path);
|
|
17
|
+
const inside = relative(realpathSync(projectDir), resolved);
|
|
18
|
+
if (isAbsolute(inside) || inside === '..' || inside.startsWith(`..${sep}`)) throw new Error('input escaped project');
|
|
19
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
20
|
+
try {
|
|
21
|
+
const before = fstatSync(fd);
|
|
22
|
+
if (!before.isFile()) throw new Error('input is not a regular file');
|
|
23
|
+
const hash = sha256(readFileSync(fd));
|
|
24
|
+
const after = fstatSync(fd);
|
|
25
|
+
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || realpathSync(path) !== resolved) throw new Error('input changed during read');
|
|
26
|
+
return hash;
|
|
27
|
+
} finally { closeSync(fd); }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Internal policy boundary: native listing plus current per-fault bindings.
|
|
31
|
+
* Legacy/static universes and any uncertainty refuse; verdicts remain intact.
|
|
32
|
+
*/
|
|
33
|
+
export function originPolicyInputsFresh({ projectDir, claims, evidence, universe }) {
|
|
34
|
+
try {
|
|
35
|
+
if (!validate('claims', claims).ok || !validate('evidence', evidence).ok) return false;
|
|
36
|
+
if (methodOf(evidence.run) !== 'fault-injection' || evidence.run.provisional || evidence.run.confirmRuns < 3) return false;
|
|
37
|
+
const testUniverseHash = hashNativeTestUniverse(universe.manifests);
|
|
38
|
+
if (universe.testUniverseHash !== testUniverseHash) return false;
|
|
39
|
+
const matchesRunner = (recorded, current) => recorded?.name === current?.name && recorded?.version === current?.version;
|
|
40
|
+
if (!universe.primaryRunner || !matchesRunner(evidence.run.runner, universe.primaryRunner)) return false;
|
|
41
|
+
if ((evidence.run.runners ?? []).some((recorded) => !universe.runners?.some((current) => matchesRunner(recorded, current)))) return false;
|
|
42
|
+
const allTests = Object.freeze([...new Set([...universe.manifests.values()].flatMap((m) => m.files))].sort());
|
|
43
|
+
const expected = new Map();
|
|
44
|
+
for (const claim of claims.claims) for (const fault of claim.faults) {
|
|
45
|
+
if (methodOf(fault) !== 'fault-injection') return false;
|
|
46
|
+
expected.set(JSON.stringify([claim.id, fault.id]), { claim, fault });
|
|
47
|
+
}
|
|
48
|
+
if (!expected.size || evidence.records.length !== expected.size) return false;
|
|
49
|
+
const seen = new Set();
|
|
50
|
+
for (const record of evidence.records) {
|
|
51
|
+
const key = JSON.stringify([record.claim.id, record.subject.id]);
|
|
52
|
+
const current = expected.get(key);
|
|
53
|
+
if (!current || seen.has(key)) return false;
|
|
54
|
+
seen.add(key);
|
|
55
|
+
const { claim, fault } = current;
|
|
56
|
+
const subject = subjectOf(fault, sha256);
|
|
57
|
+
if (['kind', 'id', 'file', 'faultClass', 'contentHash', 'producedBy'].some((field) => !isDeepStrictEqual(record.subject[field], subject[field]))) return false;
|
|
58
|
+
const { requested, selectionSource } = defenderSelection(claim, fault);
|
|
59
|
+
const discovery = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, allTests);
|
|
60
|
+
if (discovery?.indeterminate.length) return false;
|
|
61
|
+
const resolved = requested.length ? resolveDefenders(projectDir, requested) : discovery.canDetect;
|
|
62
|
+
if (resolved.some((file) => !allTests.includes(file))) return false;
|
|
63
|
+
const hashes = (files) => Object.fromEntries(files.map((file) => [file, currentHash(projectDir, file)]));
|
|
64
|
+
const inputs = {
|
|
65
|
+
targetHash: currentHash(projectDir, fault.file),
|
|
66
|
+
defenderHashes: hashes(resolved),
|
|
67
|
+
testUniverseHash,
|
|
68
|
+
discoveryHashes: hashes(discovery?.dependencies ?? []),
|
|
69
|
+
};
|
|
70
|
+
if (!isReusable(record, { claim, requested, resolved, selectionSource, contentHash: subject.contentHash, inputs })) return false;
|
|
71
|
+
}
|
|
72
|
+
return true;
|
|
73
|
+
} catch { return false; }
|
|
74
|
+
}
|