testguard-cli 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/README.md +152 -13
  3. package/package.json +1 -1
  4. package/spec/GATE-SEMANTICS.md +295 -0
  5. package/spec/README.md +27 -2
  6. package/spec/lib/authoring-input.mjs +40 -0
  7. package/spec/lib/origins.mjs +56 -0
  8. package/spec/lib/validate.mjs +120 -1
  9. package/spec/schemas/annotations.schema.json +65 -0
  10. package/spec/schemas/authoring-input.schema.json +67 -0
  11. package/spec/schemas/brief.schema.json +2 -0
  12. package/spec/schemas/claims.schema.json +2 -8
  13. package/spec/schemas/common.schema.json +73 -0
  14. package/spec/schemas/evidence.schema.json +37 -8
  15. package/spec/schemas/status.schema.json +6 -2
  16. package/src/brief/brief.mjs +11 -4
  17. package/src/claims/annotation-placement.mjs +357 -0
  18. package/src/claims/annotations.mjs +61 -2
  19. package/src/cli.mjs +40 -5
  20. package/src/commands/admit.mjs +13 -0
  21. package/src/commands/annotations.mjs +67 -0
  22. package/src/commands/claims.mjs +8 -4
  23. package/src/commands/probe.mjs +50 -7
  24. package/src/commands/replay.mjs +7 -0
  25. package/src/commands/scaffold.mjs +49 -2
  26. package/src/commands/sweep.mjs +7 -0
  27. package/src/evidence/writer.mjs +12 -4
  28. package/src/git.mjs +4 -4
  29. package/src/init/templates/SKILL.md +35 -0
  30. package/src/probe/attribution.mjs +10 -0
  31. package/src/probe/contention.mjs +20 -2
  32. package/src/probe/cost.mjs +6 -1
  33. package/src/probe/inject.mjs +2 -6
  34. package/src/probe/origin-policy.mjs +69 -0
  35. package/src/probe/policy-freshness.mjs +74 -0
  36. package/src/probe/probe.mjs +48 -23
  37. package/src/probe/runners/discovery.mjs +2 -0
  38. package/src/probe/runners/jest.mjs +2 -2
  39. package/src/probe/runners/lifecycle.mjs +58 -0
  40. package/src/probe/runners/playwright.mjs +6 -2
  41. package/src/probe/runners/python.mjs +10 -4
  42. package/src/probe/runners/shared.mjs +12 -6
  43. package/src/probe/runners/vitest.mjs +3 -3
  44. package/src/probe/universe.mjs +47 -0
  45. package/src/probe/worktree.mjs +3 -1
  46. package/src/render.mjs +17 -1
  47. package/src/replay/replay.mjs +8 -4
  48. package/src/scaffold/admission.mjs +117 -0
  49. package/src/scaffold/append.mjs +41 -0
  50. package/src/scaffold/apply-append.mjs +71 -0
  51. package/src/scaffold/fix-input.mjs +119 -0
  52. package/src/scaffold/fix-paths.mjs +90 -0
  53. package/src/scaffold/intent.mjs +2 -0
  54. package/src/scaffold/open-append.mjs +44 -0
  55. package/src/scaffold/plan-append.mjs +37 -0
  56. package/src/scaffold/plan-intent-input.mjs +38 -0
  57. package/src/scaffold/prepare-append.mjs +46 -0
  58. package/src/scaffold/request.mjs +30 -0
  59. package/src/scaffold/scaffold.mjs +23 -10
  60. package/src/status/status.mjs +34 -9
  61. package/src/sweep/pool.mjs +33 -0
  62. package/src/sweep/sweep.mjs +9 -4
@@ -54,6 +54,21 @@ sections already listing this project.
54
54
 
55
55
  ## The one source of truth
56
56
 
57
+ Missing source-link advisories in claims/status never replace `next`, change
58
+ verdicts or prove intent. Exact anchors remain checked without annotations.
59
+ Origin summaries are likewise informational: status counts current declared
60
+ claims; evidence/briefs count recorded declarations and records, including
61
+ mixed sources. Source kinds are not authenticated independence. Never use an
62
+ origin label to waive a survivor.
63
+ Optional source links: `testguard claims --annotate --claim <ID>` is a read-only
64
+ file-header preview. Only with explicit permission to edit source, rerun with
65
+ `--apply`; each invocation recomputes its plan. Review refusals and partial
66
+ failures, and retain the reported private recovery originals. Never automatically
67
+ remove an existing lock or roll back over user edits. Placement is file-level,
68
+ not symbol ownership, verified behavior or authenticated intent. Missing links
69
+ do not disable exact fault-anchor checks. Authoring never replaces the status
70
+ operating loop or writes canonical claims/evidence/baselines.
71
+
57
72
  ```bash
58
73
  testguard status --json
59
74
  ```
@@ -76,6 +91,18 @@ Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) ·
76
91
 
77
92
  ## Every change needs a claim
78
93
 
94
+ Scaffold proposes mechanical faults, not intended behavior. New drafts declare
95
+ `source.kind: inferred` even when grouped by an annotation. Before adopting a
96
+ draft, ask for the intended observable behavior from a requirement, ADR, bug or
97
+ incident independently of the implementation: inputs, expected outcome and
98
+ forbidden outcome. Record the supplied source kind/ref; when unavailable, keep
99
+ inferred and say the intent remains unestablished. Do not promote origin because
100
+ an annotation exists or tests pass. References are opaque, not authenticated;
101
+ verification commands never fetch them. Preserve existing supplied metadata.
102
+ Review each proposal against that intent, then probe. Do not paraphrase the code
103
+ and call that independent intent, or automatically move drafts into canonical
104
+ claims.
105
+
79
106
  `probe` verifies only the claims that exist; it is silent about unclaimed
80
107
  code by construction. So, for every source file you create or change:
81
108
 
@@ -108,6 +135,14 @@ code by construction. So, for every source file you create or change:
108
135
 
109
136
  ## The fix loop
110
137
 
138
+ For independently supplied intent in an existing disposable draft, use
139
+ `testguard scaffold <source...> --into <draft.json> --claim <existing-ID> --json`
140
+ to preview appended faults without writes. Omit `--json` only when explicitly
141
+ authorized to update that draft; private recovery is retained. Never use a
142
+ canonical claims/evidence/baseline path. Preserve intended behavior and declared
143
+ origin; generated faults are not independent intent or verification. Review the
144
+ faults and probe the selected draft before promoting it to canonical claims.
145
+
111
146
  1. `testguard status --json` → take `next.target`.
112
147
  2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
113
148
  3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
@@ -11,8 +11,17 @@
11
11
  * worktree is. Everything it needs is passed in.
12
12
  */
13
13
 
14
+ import { isDeepStrictEqual } from 'node:util';
15
+
14
16
  const isKill = (r) => r.outcome === 'fail' && (r.assertionFailures ?? 0) > 0;
15
17
 
18
+ /** Compare only the claim metadata actually recorded in evidence. */
19
+ export function sameClaimMetadata(recorded, current) {
20
+ if (!recorded || !current) return false;
21
+ const metadata = ({ id, statement, severity, source, producedBy }) => ({ id, statement, severity, source, producedBy });
22
+ return isDeepStrictEqual(metadata(recorded), metadata(current));
23
+ }
24
+
16
25
  /**
17
26
  * Fold one escalation run into the running attribution state.
18
27
  *
@@ -118,6 +127,7 @@ export function defenderSelection(claim, fault) {
118
127
 
119
128
  export function isReusable(prior, current) {
120
129
  if (!prior || !prior.subject?.contentHash) return false;
130
+ if (current.claim && !sameClaimMetadata(prior.claim, current.claim)) return false;
121
131
  // A record that exists because the probe threw is a statement about the run,
122
132
  // not about the code. Reusing it would make one disturbed run permanent.
123
133
  if (prior.detail?.reason === 'probe-error') return false;
@@ -37,9 +37,13 @@ export function processList({ platform = process.platform, run = spawnSync } = {
37
37
  try {
38
38
  const r = platform === 'win32'
39
39
  ? run('wmic', ['process', 'get', 'ProcessId,CommandLine', '/format:csv'], { encoding: 'utf8', timeout: 5000 })
40
- : run('ps', ['-Ao', 'pid=,args='], { encoding: 'utf8', timeout: 5000 });
40
+ : run('ps', ['-Ao', 'pid=,ppid=,stat=,args='], { encoding: 'utf8', timeout: 5000 });
41
41
  if (r.status !== 0 || !r.stdout) return [];
42
42
  return r.stdout.split('\n').map((line) => {
43
+ if (platform !== 'win32') {
44
+ const row = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line);
45
+ if (row) return { pid: Number(row[1]), ppid: Number(row[2]), stat: row[3], command: row[4].trim() };
46
+ }
43
47
  const m = platform === 'win32' ? /^[^,]*,(.*),(\d+)\s*$/.exec(line) : /^\s*(\d+)\s+(.*)$/.exec(line);
44
48
  if (!m) return null;
45
49
  const [pid, command] = platform === 'win32' ? [m[2], m[1]] : [m[1], m[2]];
@@ -57,8 +61,22 @@ export function processList({ platform = process.platform, run = spawnSync } = {
57
61
  */
58
62
  export function detectContention({ self = process.pid, alive = defaultAlive, ...opts } = {}) {
59
63
  const rows = processList(opts);
64
+ const owned = new Set([self]);
65
+ let changed = true;
66
+ while (changed) {
67
+ changed = false;
68
+ for (const row of rows) {
69
+ if (owned.has(row.ppid) && !owned.has(row.pid)) { owned.add(row.pid); changed = true; }
70
+ }
71
+ }
72
+ const ancestors = new Set();
73
+ let parent = rows.find((row) => row.pid === self)?.ppid;
74
+ while (parent && !ancestors.has(parent)) {
75
+ ancestors.add(parent);
76
+ parent = rows.find((row) => row.pid === parent)?.ppid;
77
+ }
60
78
  const runners = rows
61
- .filter((p) => p.pid !== self && RUNNER_RE.test(p.command) && !SELF_RE.test(p.command))
79
+ .filter((p) => !owned.has(p.pid) && !ancestors.has(p.pid) && !p.stat?.startsWith('Z') && RUNNER_RE.test(p.command) && (p.ppid !== undefined || !SELF_RE.test(p.command)))
62
80
  // `ps` lists the grep/ps itself and any shell wrapper; a runner's command
63
81
  // line always names its own binary, so require it to look like an exec.
64
82
  .filter((p) => !/^\s*(ps|grep|wmic)\b/.test(p.command))
@@ -92,10 +92,11 @@ export function defenderCosts(records) {
92
92
  * turns a twenty-four-minute gate into a one-minute one, and until a report
93
93
  * names the file nobody knows which test to split.
94
94
  */
95
- export function costReport(records) {
95
+ export function costReport(records, { run } = {}) {
96
96
  const claims = claimCosts(records);
97
97
  const defenders = defenderCosts(records);
98
98
  return {
99
+ ...(run?.measurements ? { measurements: { ...run.measurements } } : {}),
99
100
  totalMs: claims.reduce((a, c) => a + c.ms, 0),
100
101
  totalRuns: claims.reduce((a, c) => a + c.runs, 0),
101
102
  reusedRecords: records.filter((r) => r.reusedFrom).length,
@@ -118,6 +119,10 @@ export const formatMs = secs;
118
119
  */
119
120
  export function renderCost(report, { limit = 10 } = {}) {
120
121
  const out = [];
122
+ if (report.measurements) {
123
+ const m = report.measurements;
124
+ out.push(`actual elapsed ${secs(m.elapsedMs)}; fresh runners ${secs(m.runnerMs)} across ${m.runnerInvocations} invocations; setup/discovery/other overhead ${secs(m.overheadMs)}`);
125
+ }
121
126
  if (!report.records) return 'no evidence records — run `testguard probe` first; cost is read back from the runs it records.';
122
127
  out.push(`${report.records} fault record${report.records === 1 ? '' : 's'} cost ${secs(report.totalMs)} across ${report.totalRuns} defender runs${report.reusedRecords ? ` (${report.reusedRecords} reused; their runs are what they cost when reuse does not apply)` : ''}.`);
123
128
  out.push('');
@@ -1,3 +1,4 @@
1
+ import { registerCleanup } from './runners/lifecycle.mjs';
1
2
  import { readFileSync, writeFileSync } from 'node:fs';
2
3
  import { join } from 'node:path';
3
4
 
@@ -19,12 +20,7 @@ function restoreAll() {
19
20
  function installHandlers() {
20
21
  if (handlersInstalled) return;
21
22
  handlersInstalled = true;
22
- for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
23
- process.on(sig, () => {
24
- restoreAll();
25
- process.exit(130);
26
- });
27
- }
23
+ registerCleanup(restoreAll, { priority: 1 });
28
24
  process.on('uncaughtException', (err) => {
29
25
  restoreAll();
30
26
  throw err;
@@ -0,0 +1,69 @@
1
+ import { isDeepStrictEqual } from 'node:util';
2
+ import { validate, methodOf } from '../../spec/lib/validate.mjs';
3
+ import { CLAIM_SOURCE_KINDS } from '../../spec/lib/origins.mjs';
4
+ import { sameClaimMetadata, subjectOf } from './attribution.mjs';
5
+ import { sha256 } from '../util/hash.mjs';
6
+
7
+ /** Admit the planned probe option before loading claims or executing/writing. */
8
+ export function originPolicyRequest(values, command = 'probe') {
9
+ const raw = values['require-origin'];
10
+ if (raw === undefined) return undefined;
11
+ if (command !== 'probe') throw new TypeError('--require-origin is only valid on probe');
12
+ if (!Array.isArray(raw) || !raw.length || raw.some((v) => typeof v !== 'string')) {
13
+ throw new TypeError('--require-origin must name declared source kinds');
14
+ }
15
+ const kinds = raw.flatMap((v) => v.split(',').map((k) => k.trim()));
16
+ if (kinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) throw new TypeError('--require-origin contains an empty or unsupported kind');
17
+ if (values.claim !== undefined || values['allow-empty'] || values.ref !== undefined || values['ignore-dirty']) {
18
+ throw new TypeError('--require-origin needs a complete current-project run without --claim, --allow-empty, --ref or --ignore-dirty');
19
+ }
20
+ const n = Number(values.confirm ?? 3);
21
+ if (!Number.isInteger(n) || n < 3) throw new TypeError('--require-origin requires --confirm of at least 3');
22
+ return [...new Set(kinds)].sort();
23
+ }
24
+
25
+ /** Internal policy only: fresh must come from the existing input-binding checker. */
26
+ export function evaluateOriginPolicy({ claims, evidence, eligibleKinds, fresh }) {
27
+ if (eligibleKinds === undefined) return undefined;
28
+ if (!Array.isArray(eligibleKinds) || !eligibleKinds.length || eligibleKinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) {
29
+ throw new TypeError('origin policy needs a nonempty set of declared source kinds');
30
+ }
31
+ const eligible = [...new Set(eligibleKinds)].sort();
32
+ const reasons = new Set();
33
+ const claimsValid = validate('claims', claims).ok;
34
+ const evidenceValid = validate('evidence', evidence).ok;
35
+ if (!claimsValid) reasons.add('invalid-claims');
36
+ if (!evidenceValid) reasons.add('invalid-evidence');
37
+ if (fresh !== true) reasons.add('stale-inputs');
38
+ const current = claimsValid ? claims.claims : [];
39
+ const records = evidenceValid ? evidence.records : [];
40
+ const key = (claimId, faultId) => JSON.stringify([claimId, faultId]);
41
+ const expected = new Map(current.flatMap((claim) => claim.faults.map((fault) => [key(claim.id, fault.id), { claim, fault }])));
42
+ if (!current.length || !expected.size) reasons.add('empty-universe');
43
+ if (evidenceValid && (evidence.run.provisional === true || evidence.run.confirmRuns < 3)) reasons.add('provisional-evidence');
44
+ if (evidenceValid && methodOf(evidence.run) !== 'fault-injection') reasons.add('incompatible-method');
45
+ if (current.some((c) => c.faults.some((f) => methodOf(f) !== 'fault-injection'))) reasons.add('incompatible-method');
46
+ const seen = new Set();
47
+ for (const record of records) {
48
+ const identity = key(record.claim.id, record.subject.id);
49
+ if (seen.has(identity)) reasons.add('duplicate-record');
50
+ seen.add(identity);
51
+ const pair = expected.get(identity);
52
+ if (!pair) { reasons.add('extra-record'); continue; }
53
+ if (!sameClaimMetadata(record.claim, pair.claim)) reasons.add('claim-metadata-mismatch');
54
+ if (methodOf(pair.fault) !== 'fault-injection') continue;
55
+ const subject = subjectOf(pair.fault, sha256);
56
+ if (['kind', 'id', 'file', 'faultClass', 'contentHash'].some((field) => record.subject[field] !== subject[field])
57
+ || !isDeepStrictEqual(record.subject.producedBy, subject.producedBy)) reasons.add('fault-mismatch');
58
+ }
59
+ if ([...expected.keys()].some((identity) => !seen.has(identity))) reasons.add('missing-record');
60
+ const ineligibleClaims = current.filter((c) => !eligible.includes(c.source.kind)).map((c) => c.id).sort();
61
+ const nonKilledFaults = [...new Map(records.filter((r) => r.verdict !== 'killed').map((r) =>
62
+ [key(r.claim.id, r.subject.id), { claimId: r.claim.id, faultId: r.subject.id }])).values()]
63
+ .sort((a, b) => a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId));
64
+ return {
65
+ state: reasons.size ? 'unavailable' : ineligibleClaims.length || nonKilledFaults.length ? 'failed' : 'passed',
66
+ eligibleKinds: eligible, claims: current.length, faults: expected.size,
67
+ ineligibleClaims, nonKilledFaults, unavailableReasons: [...reasons].sort(),
68
+ };
69
+ }
@@ -0,0 +1,74 @@
1
+ import { closeSync, constants, fstatSync, openSync, readFileSync, realpathSync } from 'node:fs';
2
+ import { isAbsolute, join, relative, sep } from 'node:path';
3
+ import { isDeepStrictEqual } from 'node:util';
4
+ import { validate, methodOf } from '../../spec/lib/validate.mjs';
5
+ import { sha256 } from '../util/hash.mjs';
6
+ import { defenderSelection, isReusable, subjectOf } from './attribution.mjs';
7
+ import { discoverDefendersDetailed } from './discover.mjs';
8
+ import { resolveDefenders } from './runners/shared.mjs';
9
+ import { hashNativeTestUniverse } from './universe.mjs';
10
+
11
+ // Point-in-time reads, not a transaction over the whole project. Never turn a
12
+ // missing file into the empty-file hash that diagnostic probe records can use.
13
+ function currentHash(projectDir, file) {
14
+ if (typeof file !== 'string' || isAbsolute(file) || file.split(/[\\/]/).includes('..')) throw new Error('unsafe input path');
15
+ const path = join(projectDir, file);
16
+ const resolved = realpathSync(path);
17
+ const inside = relative(realpathSync(projectDir), resolved);
18
+ if (isAbsolute(inside) || inside === '..' || inside.startsWith(`..${sep}`)) throw new Error('input escaped project');
19
+ const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
20
+ try {
21
+ const before = fstatSync(fd);
22
+ if (!before.isFile()) throw new Error('input is not a regular file');
23
+ const hash = sha256(readFileSync(fd));
24
+ const after = fstatSync(fd);
25
+ if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || realpathSync(path) !== resolved) throw new Error('input changed during read');
26
+ return hash;
27
+ } finally { closeSync(fd); }
28
+ }
29
+
30
+ /** Internal policy boundary: native listing plus current per-fault bindings.
31
+ * Legacy/static universes and any uncertainty refuse; verdicts remain intact.
32
+ */
33
+ export function originPolicyInputsFresh({ projectDir, claims, evidence, universe }) {
34
+ try {
35
+ if (!validate('claims', claims).ok || !validate('evidence', evidence).ok) return false;
36
+ if (methodOf(evidence.run) !== 'fault-injection' || evidence.run.provisional || evidence.run.confirmRuns < 3) return false;
37
+ const testUniverseHash = hashNativeTestUniverse(universe.manifests);
38
+ if (universe.testUniverseHash !== testUniverseHash) return false;
39
+ const matchesRunner = (recorded, current) => recorded?.name === current?.name && recorded?.version === current?.version;
40
+ if (!universe.primaryRunner || !matchesRunner(evidence.run.runner, universe.primaryRunner)) return false;
41
+ if ((evidence.run.runners ?? []).some((recorded) => !universe.runners?.some((current) => matchesRunner(recorded, current)))) return false;
42
+ const allTests = Object.freeze([...new Set([...universe.manifests.values()].flatMap((m) => m.files))].sort());
43
+ const expected = new Map();
44
+ for (const claim of claims.claims) for (const fault of claim.faults) {
45
+ if (methodOf(fault) !== 'fault-injection') return false;
46
+ expected.set(JSON.stringify([claim.id, fault.id]), { claim, fault });
47
+ }
48
+ if (!expected.size || evidence.records.length !== expected.size) return false;
49
+ const seen = new Set();
50
+ for (const record of evidence.records) {
51
+ const key = JSON.stringify([record.claim.id, record.subject.id]);
52
+ const current = expected.get(key);
53
+ if (!current || seen.has(key)) return false;
54
+ seen.add(key);
55
+ const { claim, fault } = current;
56
+ const subject = subjectOf(fault, sha256);
57
+ if (['kind', 'id', 'file', 'faultClass', 'contentHash', 'producedBy'].some((field) => !isDeepStrictEqual(record.subject[field], subject[field]))) return false;
58
+ const { requested, selectionSource } = defenderSelection(claim, fault);
59
+ const discovery = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, allTests);
60
+ if (discovery?.indeterminate.length) return false;
61
+ const resolved = requested.length ? resolveDefenders(projectDir, requested) : discovery.canDetect;
62
+ if (resolved.some((file) => !allTests.includes(file))) return false;
63
+ const hashes = (files) => Object.fromEntries(files.map((file) => [file, currentHash(projectDir, file)]));
64
+ const inputs = {
65
+ targetHash: currentHash(projectDir, fault.file),
66
+ defenderHashes: hashes(resolved),
67
+ testUniverseHash,
68
+ discoveryHashes: hashes(discovery?.dependencies ?? []),
69
+ };
70
+ if (!isReusable(record, { claim, requested, resolved, selectionSource, contentHash: subject.contentHash, inputs })) return false;
71
+ }
72
+ return true;
73
+ } catch { return false; }
74
+ }
@@ -1,3 +1,6 @@
1
+ import { resetInterpreterCache } from './runners/python.mjs';
2
+ import { assertNotCancelled } from './runners/lifecycle.mjs';
3
+ import { performance } from 'node:perf_hooks';
1
4
  import { existsSync, readFileSync, realpathSync as fsRealpathSync } from 'node:fs';
2
5
  import { gitRaw } from '../git.mjs';
3
6
  import { createRequire } from 'node:module';
@@ -16,7 +19,9 @@ import { classifyIndependence } from './independence.mjs';
16
19
  import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable, defenderSelection } from './attribution.mjs';
17
20
  import { hashFile, sha256 } from '../util/hash.mjs';
18
21
  import { fingerprint } from '../../spec/lib/fingerprint.mjs';
22
+ import { recordedOriginSummary } from '../../spec/lib/origins.mjs';
19
23
  import { persistenceSignalsFor } from '../supply/persistence.mjs';
24
+ import { collectOwnedManifests, hashNativeTestUniverse } from './universe.mjs';
20
25
 
21
26
 
22
27
  /** Is `child` the same file as `root`, or under it? Both must already be real paths. */
@@ -51,6 +56,12 @@ export function selectClaims(claims, only) {
51
56
  return only.map((id) => byId.get(id));
52
57
  }
53
58
 
59
+ /** Reuse cannot stand in for a measurement under a different worker policy. */
60
+ export function sameWorkerPolicy(previousRun, { workers = 1, serial = workers === 1, runnerCommand } = {}) {
61
+ if (runnerCommand) return previousRun?.workers === undefined && !previousRun?.serial;
62
+ return previousRun?.workers === (serial ? 1 : workers) && Boolean(previousRun?.serial) === serial;
63
+ }
64
+
54
65
  /**
55
66
  * Probe every fault of every claim. Returns a spec-conformant evidence
56
67
  * document; writing it is the caller's job.
@@ -63,7 +74,8 @@ export async function probe({
63
74
  ref = 'HEAD',
64
75
  refExplicit = false,
65
76
  ignoreDirty = false,
66
- serial = false,
77
+ workers = 1,
78
+ serial = workers === 1,
67
79
  onWarn = () => {},
68
80
  budgetMs = 120_000,
69
81
  commandBudget,
@@ -80,6 +92,18 @@ export async function probe({
80
92
  previous,
81
93
  onProgress = () => {},
82
94
  }) {
95
+ if (!Number.isSafeInteger(workers) || workers < 1) throw new PreconditionError('workers must be a positive safe integer');
96
+ resetInterpreterCache(); // Cache only within this measurement, never across commands.
97
+ const measuredStart = performance.now();
98
+ const startedAt = new Date().toISOString();
99
+ const measurements = { elapsedMs: 0, runnerMs: 0, overheadMs: 0, runnerInvocations: 0 };
100
+ const timedRun = async (runner, opts) => {
101
+ assertNotCancelled();
102
+ const start = performance.now();
103
+ measurements.runnerInvocations++;
104
+ try { return await runner.run(opts); }
105
+ finally { measurements.runnerMs += performance.now() - start; }
106
+ };
83
107
  commandBudget?.assertOpen();
84
108
  // realpath: git reports the repository root by its real path (/private/var
85
109
  // on macOS, not /var); every relative() below must start from the same place.
@@ -183,7 +207,6 @@ export async function probe({
183
207
  const contention = detectContention();
184
208
  if (contention.detected) onWarn(contentionWarning(contention));
185
209
 
186
- const startedAt = new Date().toISOString();
187
210
  const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
188
211
  const isoReal = realpathSync(iso.projectDir);
189
212
  const records = [];
@@ -218,20 +241,16 @@ export async function probe({
218
241
  // project runner is; it must resolve before the first such defender runs.
219
242
  const owned = OWNED_RUNNERS.filter((r) => r !== runner
220
243
  && !(r.name === 'python' && ['python', 'pytest', 'unittest'].includes(runner.name)));
221
- const ownedChecked = new Map();
222
- const manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
244
+ let ownedChecked = new Map();
245
+ let manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
223
246
  if (!commandTemplate) {
224
- for (const r of owned) {
225
- const check = await r.check({ projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs });
226
- commandBudget?.assertOpen();
227
- ownedChecked.set(r, check);
228
- if (!check.ok) continue;
229
- try {
230
- manifests.set(r, await discoverRunnerManifest(r, check, iso.projectDir, commandBudget?.runBudget(budgetMs) ?? budgetMs));
231
- } catch (error) {
232
- throw new PreconditionError(`${r.name}: test discovery failed: ${error.message}`);
233
- }
234
- }
247
+ try {
248
+ ({ ownedChecked, manifests } = await collectOwnedManifests(runner, primaryManifest, {
249
+ projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs,
250
+ budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
251
+ assertOpen: () => commandBudget?.assertOpen(),
252
+ }));
253
+ } catch (error) { throw new PreconditionError(`test discovery failed: ${error.message}`); }
235
254
  }
236
255
  const ensureOwned = async (r, file) => {
237
256
  if (!ownedChecked.has(r)) {
@@ -265,9 +284,7 @@ export async function probe({
265
284
  : [...ownerByFile.keys()]).sort());
266
285
  const testUniverseHash = commandTemplate
267
286
  ? sha256(JSON.stringify({ schemaVersion: 1, source: 'custom-command-static', files: allTests }))
268
- : sha256(JSON.stringify([...manifests]
269
- .map(([r, manifest]) => ({ runner: r.name, hash: manifest.testUniverseHash }))
270
- .sort((a, b) => a.runner.localeCompare(b.runner))));
287
+ : hashNativeTestUniverse(manifests);
271
288
  const baselineCache = new Map();
272
289
  const discoveryHashCache = new Map();
273
290
  // The negative control is charged per (target file, defender set): two
@@ -277,6 +294,7 @@ export async function probe({
277
294
  // runner — a `.py` source file is Python's whatever the project runs.
278
295
  const fatalEditFor = (file) => runnerFor(iso.projectDir, file, runner).fatalEdit?.();
279
296
  const prior = previous && previous.run.confirmRuns === confirmRuns
297
+ && sameWorkerPolicy(previous.run, { workers, serial, runnerCommand })
280
298
  ? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
281
299
  : new Map();
282
300
  const partitionConfigured = (files) => {
@@ -295,7 +313,7 @@ export async function probe({
295
313
  // file the interpreter actually loaded (Python) reports it back.
296
314
  const runDefenders = async (files, targets = []) => {
297
315
  if (commandTemplate) {
298
- const result = await runner.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, targets });
316
+ const result = await timedRun(runner, { projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, workers, targets });
299
317
  commandBudget?.assertOpen();
300
318
  return result;
301
319
  }
@@ -304,7 +322,7 @@ export async function probe({
304
322
  for (const [r, group] of groups) {
305
323
  commandBudget?.assertOpen();
306
324
  if (r !== runner) await ensureOwned(r, group[0]);
307
- parts.push(await r.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, targets }));
325
+ parts.push(await timedRun(r, { projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, workers, targets }));
308
326
  commandBudget?.assertOpen();
309
327
  }
310
328
  return mergeRuns(parts);
@@ -373,6 +391,10 @@ export async function probe({
373
391
 
374
392
  commandBudget?.assertOpen();
375
393
 
394
+ assertNotCancelled();
395
+ const dirty = isDirty(root);
396
+ measurements.elapsedMs = performance.now() - measuredStart;
397
+ measurements.overheadMs = measurements.elapsedMs - measurements.runnerMs;
376
398
  return {
377
399
  schemaVersion: 1,
378
400
  tool: { name: 'testguard', version: toolVersion },
@@ -380,11 +402,13 @@ export async function probe({
380
402
  id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
381
403
  startedAt,
382
404
  finishedAt: new Date().toISOString(),
383
- repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
405
+ repo: { head, dirty, ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
384
406
  runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
385
407
  ...(runnersUsed.size > 1 ? { runners: [...runnersUsed].map(([n, v]) => ({ name: n, ...(v ? { version: v } : {}) })) } : {}),
386
408
  confirmRuns,
387
- ...(serial ? { serial: true } : {}),
409
+ measurements,
410
+ ...(!runnerCommand ? { workers: serial ? 1 : workers } : {}),
411
+ ...(serial && !runnerCommand ? { serial: true } : {}),
388
412
  ...(contention.detected ? { contention } : {}),
389
413
  ...(confirmRuns < 3 ? { provisional: true } : {}),
390
414
  // The isolation that was actually created, not the mode that was asked
@@ -394,6 +418,7 @@ export async function probe({
394
418
  mode: iso.mode,
395
419
  },
396
420
  records,
421
+ origins: recordedOriginSummary(records),
397
422
  };
398
423
  }
399
424
 
@@ -520,7 +545,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
520
545
  const subject = subjectOf(fault, sha256);
521
546
 
522
547
  // Same source, same defenders, same N, same fault: the verdict cannot have changed.
523
- if (isReusable(prior, { inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
548
+ if (isReusable(prior, { claim, inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
524
549
  return { ...prior, defenders: { ...prior.defenders, ...defenderSelection(claim, fault) }, reusedFrom: prior.reusedFrom ?? priorRunId };
525
550
  }
526
551
 
@@ -1,3 +1,4 @@
1
+ import { registerChild } from './lifecycle.mjs';
1
2
  import { spawn } from 'node:child_process';
2
3
  import { randomBytes } from 'node:crypto';
3
4
  import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, realpathSync, rmSync, statSync } from 'node:fs';
@@ -177,6 +178,7 @@ export function runDiscoveryProcess({
177
178
  return;
178
179
  }
179
180
 
181
+ registerChild(child, () => terminateProcessTree(child));
180
182
  const started = Date.now();
181
183
  const stdout = [];
182
184
  const stderr = [];
@@ -38,8 +38,8 @@ export async function discoverTests({ projectDir, version, timeoutMs, maxOutputB
38
38
  }
39
39
  // --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
40
40
  /** The command line, as data. Exported so it is falsifiable without spawning anything. */
41
- export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
42
- [...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial ? ['--runInBand'] : []), '--runTestsByPath', ...files];
41
+ export const argvFor = (projectDir, files, outFile, { serial = false, workers = 1 } = {}) =>
42
+ [...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial || workers === 1 ? ['--runInBand'] : [`--maxWorkers=${workers}`]), '--runTestsByPath', ...files];
43
43
  export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
44
44
 
45
45
  /**
@@ -0,0 +1,58 @@
1
+ /** Own only children and cleanups registered by this TestGuard process. */
2
+ const children = new Map();
3
+ const cleanups = new Map();
4
+ let installed = false;
5
+ let cancelling = false;
6
+
7
+ function cleanup() {
8
+ // Restore mutations before removing isolation, even across successive probes.
9
+ for (const [fn] of [...cleanups].reverse().sort((a, b) => b[1] - a[1])) {
10
+ try { fn(); } catch (error) { process.stderr.write(`cleanup failed: ${error.message}\n`); }
11
+ }
12
+ cleanups.clear();
13
+ }
14
+
15
+ function install() {
16
+ if (installed) return;
17
+ installed = true;
18
+ for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
19
+ process.on(signal, async () => {
20
+ if (cancelling) return;
21
+ cancelling = true;
22
+ const waits = [...children].map(([child, stop]) => {
23
+ const closed = new Promise((resolve) => child.once('close', resolve));
24
+ stop();
25
+ return closed;
26
+ });
27
+ let timer;
28
+ await Promise.race([
29
+ Promise.allSettled(waits),
30
+ new Promise((resolve) => { timer = setTimeout(resolve, 1000); }),
31
+ ]);
32
+ clearTimeout(timer);
33
+ cleanup();
34
+ process.exit(130);
35
+ });
36
+ }
37
+ process.on('exit', () => {
38
+ for (const stop of children.values()) stop();
39
+ cleanup();
40
+ });
41
+ }
42
+
43
+ export function registerChild(child, stop) {
44
+ install();
45
+ children.set(child, stop);
46
+ child.once('close', () => children.delete(child));
47
+ if (cancelling) stop();
48
+ }
49
+
50
+ export function registerCleanup(fn, { priority = 0 } = {}) {
51
+ install();
52
+ cleanups.set(fn, priority);
53
+ return () => cleanups.delete(fn);
54
+ }
55
+
56
+ export function assertNotCancelled() {
57
+ if (cancelling) throw new Error('TestGuard command cancelled; no partial result may be published');
58
+ }
@@ -1,7 +1,7 @@
1
1
  import { existsSync, readFileSync } from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import { isAbsolute, join, relative, resolve, sep } from 'node:path';
4
- import { npx, runProcess, listTestFiles, firstInformativeLine, runnerArgv } from './shared.mjs';
4
+ import { runProcess, listTestFiles, firstInformativeLine, runnerArgv } from './shared.mjs';
5
5
  import { createDiscoveryManifest, DiscoveryError, hashDiscoveryConfigs, normalizeDiscoveredFiles, runDiscoveryReportProcess } from './discovery.mjs';
6
6
  import { globToRegExp } from '../../util/glob.mjs';
7
7
 
@@ -185,7 +185,7 @@ export function parseReport(report, durationMs) {
185
185
 
186
186
  export const run = (opts) => {
187
187
  const env = { PLAYWRIGHT_JSON_OUTPUT_FILE: '{out}', PLAYWRIGHT_JSON_OUTPUT_NAME: '{out}' };
188
- return runProcess({ ...opts, env, parse: parseReport, argv: (files) => [npx, 'playwright', 'test', '--reporter=json', ...(opts.serial ? ['--workers=1'] : []), ...files] });
188
+ return runProcess({ ...opts, env, parse: parseReport, argv: (files) => argvFor(opts.projectDir, files, opts) });
189
189
  };
190
190
 
191
191
  /**
@@ -196,3 +196,7 @@ export const run = (opts) => {
196
196
  * contained before.
197
197
  */
198
198
  export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';
199
+
200
+ /** Run the project-pinned CLI directly, avoiding an npm subprocess per confirmation. */
201
+ export const argvFor = (projectDir, files, { serial = false, workers = 1 } = {}) =>
202
+ [...runnerArgv(projectDir, '@playwright/test', 'playwright'), 'test', '--reporter=json', `--workers=${serial ? 1 : workers}`, ...files];
@@ -1,4 +1,6 @@
1
+ import { performance } from 'node:perf_hooks';
1
2
  import { existsSync, readFileSync } from 'node:fs';
3
+ import { registerChild } from './lifecycle.mjs';
2
4
  import { spawn } from 'node:child_process';
3
5
  import { dirname, join } from 'node:path';
4
6
  import { fileURLToPath } from 'node:url';
@@ -120,9 +122,10 @@ function ask(path, budgetMs) {
120
122
  } catch {
121
123
  return resolve(null);
122
124
  }
125
+ registerChild(child, () => terminateProcessTree(child));
123
126
  let out = '';
124
127
  child.on('error', () => resolve(null));
125
- child.stdout.on('data', (d) => (out += d));
128
+ child.stdout.on('data', (d) => (out = (out + d).slice(-65536)));
126
129
  child.stderr.on('data', () => {});
127
130
  const timer = setTimeout(() => terminateProcessTree(child), budgetMs);
128
131
  child.on('close', (code) => {
@@ -276,10 +279,10 @@ export function environment(targets) {
276
279
  * the configured value; the rest of the project's `addopts` is left alone,
277
280
  * because TestGuard runs a project's tests the way the project runs them.
278
281
  */
279
- export function argvFor({ engine, interpreter, files, serial }) {
282
+ export function argvFor({ engine, interpreter, files, serial = true }) {
280
283
  if (engine === 'pytest') {
281
284
  return [interpreter, '-m', 'pytest', '-p', '_testguard_pytest_plugin', '-p', 'no:cacheprovider',
282
- '--maxfail=0', '-q', ...(serial ? ['-p', 'no:xdist'] : []), ...files];
285
+ '--maxfail=0', '-q', '-p', 'no:xdist', ...files];
283
286
  }
284
287
  return [interpreter, '-m', '_testguard_unittest_main', ...files];
285
288
  }
@@ -287,12 +290,15 @@ export function argvFor({ engine, interpreter, files, serial }) {
287
290
  export function makeRun(pinned) {
288
291
  const check = makeCheck(pinned);
289
292
  return async (opts) => {
290
- const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python });
293
+ const deadline = performance.now() + (opts.budgetMs ?? 120_000);
294
+ const remaining = () => Math.max(1, Math.floor(deadline - performance.now()));
295
+ const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python, budgetMs: remaining(), budgetFor: remaining });
291
296
  if (!resolved.ok) {
292
297
  return { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs: 0 }, timeouts: 0, loadMessage: resolved.message, failedTests: [], provenance: {} };
293
298
  }
294
299
  return runProcess({
295
300
  ...opts,
301
+ budgetMs: remaining(),
296
302
  env: environment(opts.targets),
297
303
  parse: parseReport,
298
304
  argv: (files) => argvFor({ engine: resolved.engine, interpreter: resolved.interpreter, files, serial: opts.serial }),