testguard-cli 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +82 -0
  2. package/README.md +196 -17
  3. package/package.json +1 -1
  4. package/spec/GATE-SEMANTICS.md +316 -0
  5. package/spec/README.md +27 -2
  6. package/spec/lib/authoring-input.mjs +40 -0
  7. package/spec/lib/origins.mjs +56 -0
  8. package/spec/lib/validate.mjs +129 -1
  9. package/spec/schemas/annotations.schema.json +65 -0
  10. package/spec/schemas/authoring-input.schema.json +67 -0
  11. package/spec/schemas/brief.schema.json +2 -0
  12. package/spec/schemas/claims.schema.json +8 -8
  13. package/spec/schemas/common.schema.json +73 -0
  14. package/spec/schemas/evidence.schema.json +41 -8
  15. package/spec/schemas/status.schema.json +31 -2
  16. package/src/brief/brief.mjs +11 -4
  17. package/src/claims/annotation-placement.mjs +357 -0
  18. package/src/claims/annotations.mjs +61 -2
  19. package/src/cli.mjs +40 -5
  20. package/src/commands/admit.mjs +21 -3
  21. package/src/commands/annotations.mjs +67 -0
  22. package/src/commands/claims.mjs +46 -6
  23. package/src/commands/probe.mjs +50 -7
  24. package/src/commands/replay.mjs +7 -0
  25. package/src/commands/scaffold.mjs +49 -2
  26. package/src/commands/sweep.mjs +7 -0
  27. package/src/evidence/writer.mjs +12 -4
  28. package/src/gate/changed.mjs +5 -1
  29. package/src/git.mjs +4 -4
  30. package/src/init/init.mjs +6 -0
  31. package/src/init/templates/SKILL.md +78 -0
  32. package/src/probe/attribution.mjs +19 -0
  33. package/src/probe/contention.mjs +20 -2
  34. package/src/probe/cost.mjs +22 -9
  35. package/src/probe/inject.mjs +2 -6
  36. package/src/probe/origin-policy.mjs +69 -0
  37. package/src/probe/policy-freshness.mjs +74 -0
  38. package/src/probe/probe.mjs +74 -45
  39. package/src/probe/runners/discovery.mjs +2 -0
  40. package/src/probe/runners/jest.mjs +2 -2
  41. package/src/probe/runners/lifecycle.mjs +58 -0
  42. package/src/probe/runners/playwright.mjs +6 -2
  43. package/src/probe/runners/python.mjs +10 -4
  44. package/src/probe/runners/shared.mjs +12 -6
  45. package/src/probe/runners/vitest.mjs +3 -3
  46. package/src/probe/universe.mjs +47 -0
  47. package/src/probe/worktree.mjs +3 -1
  48. package/src/render.mjs +17 -1
  49. package/src/replay/replay.mjs +8 -4
  50. package/src/scaffold/admission.mjs +117 -0
  51. package/src/scaffold/append.mjs +41 -0
  52. package/src/scaffold/apply-append.mjs +71 -0
  53. package/src/scaffold/fix-input.mjs +119 -0
  54. package/src/scaffold/fix-paths.mjs +90 -0
  55. package/src/scaffold/intent.mjs +2 -0
  56. package/src/scaffold/open-append.mjs +44 -0
  57. package/src/scaffold/plan-append.mjs +37 -0
  58. package/src/scaffold/plan-intent-input.mjs +38 -0
  59. package/src/scaffold/prepare-append.mjs +46 -0
  60. package/src/scaffold/request.mjs +30 -0
  61. package/src/scaffold/scaffold.mjs +23 -10
  62. package/src/status/status.mjs +42 -10
  63. package/src/sweep/pool.mjs +33 -0
  64. package/src/sweep/sweep.mjs +9 -4
@@ -1,3 +1,6 @@
1
+ import { resetInterpreterCache } from './runners/python.mjs';
2
+ import { assertNotCancelled } from './runners/lifecycle.mjs';
3
+ import { performance } from 'node:perf_hooks';
1
4
  import { existsSync, readFileSync, realpathSync as fsRealpathSync } from 'node:fs';
2
5
  import { gitRaw } from '../git.mjs';
3
6
  import { createRequire } from 'node:module';
@@ -13,10 +16,12 @@ import { selectRunner, discoverRunnerManifest, RUNNERS, OWNED_RUNNERS, partition
13
16
  import { classify, shouldStopEarly } from './classify.mjs';
14
17
  import { blastRadius, rank } from './rank.mjs';
15
18
  import { classifyIndependence } from './independence.mjs';
16
- import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable } from './attribution.mjs';
19
+ import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable, defenderSelection } from './attribution.mjs';
17
20
  import { hashFile, sha256 } from '../util/hash.mjs';
18
21
  import { fingerprint } from '../../spec/lib/fingerprint.mjs';
22
+ import { recordedOriginSummary } from '../../spec/lib/origins.mjs';
19
23
  import { persistenceSignalsFor } from '../supply/persistence.mjs';
24
+ import { collectOwnedManifests, hashNativeTestUniverse } from './universe.mjs';
20
25
 
21
26
 
22
27
  /** Is `child` the same file as `root`, or under it? Both must already be real paths. */
@@ -51,6 +56,12 @@ export function selectClaims(claims, only) {
51
56
  return only.map((id) => byId.get(id));
52
57
  }
53
58
 
59
+ /** Reuse cannot stand in for a measurement under a different worker policy. */
60
+ export function sameWorkerPolicy(previousRun, { workers = 1, serial = workers === 1, runnerCommand } = {}) {
61
+ if (runnerCommand) return previousRun?.workers === undefined && !previousRun?.serial;
62
+ return previousRun?.workers === (serial ? 1 : workers) && Boolean(previousRun?.serial) === serial;
63
+ }
64
+
54
65
  /**
55
66
  * Probe every fault of every claim. Returns a spec-conformant evidence
56
67
  * document; writing it is the caller's job.
@@ -63,7 +74,8 @@ export async function probe({
63
74
  ref = 'HEAD',
64
75
  refExplicit = false,
65
76
  ignoreDirty = false,
66
- serial = false,
77
+ workers = 1,
78
+ serial = workers === 1,
67
79
  onWarn = () => {},
68
80
  budgetMs = 120_000,
69
81
  commandBudget,
@@ -80,6 +92,18 @@ export async function probe({
80
92
  previous,
81
93
  onProgress = () => {},
82
94
  }) {
95
+ if (!Number.isSafeInteger(workers) || workers < 1) throw new PreconditionError('workers must be a positive safe integer');
96
+ resetInterpreterCache(); // Cache only within this measurement, never across commands.
97
+ const measuredStart = performance.now();
98
+ const startedAt = new Date().toISOString();
99
+ const measurements = { elapsedMs: 0, runnerMs: 0, overheadMs: 0, runnerInvocations: 0 };
100
+ const timedRun = async (runner, opts) => {
101
+ assertNotCancelled();
102
+ const start = performance.now();
103
+ measurements.runnerInvocations++;
104
+ try { return await runner.run(opts); }
105
+ finally { measurements.runnerMs += performance.now() - start; }
106
+ };
83
107
  commandBudget?.assertOpen();
84
108
  // realpath: git reports the repository root by its real path (/private/var
85
109
  // on macOS, not /var); every relative() below must start from the same place.
@@ -154,11 +178,14 @@ export async function probe({
154
178
  }
155
179
  }
156
180
  for (const claim of scopedClaims) {
157
- const discovered = claim.defendedBy?.length ? null : claim.faults.map((fault) => discoverDefendersDetailed(projectDir, fault.file, configuredFiles));
158
- const declared = claim.defendedBy?.length ? resolveDefenders(projectDir, claim.defendedBy) : discovered.flatMap((detail) => detail.canDetect);
159
- for (const d of declared) watched.add(relative(root, join(projectDir, d)));
160
- for (const dependency of discovered?.flatMap((detail) => detail.dependencies) ?? []) watched.add(relative(root, join(projectDir, dependency)));
161
- for (const g of claim.defendedBy ?? []) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
181
+ for (const fault of claim.faults) {
182
+ const { requested } = defenderSelection(claim, fault);
183
+ const discovered = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, configuredFiles);
184
+ const declared = requested.length ? resolveDefenders(projectDir, requested) : discovered.canDetect;
185
+ for (const d of declared) watched.add(relative(root, join(projectDir, d)));
186
+ for (const dependency of discovered?.dependencies ?? []) watched.add(relative(root, join(projectDir, dependency)));
187
+ for (const g of requested) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
188
+ }
162
189
  }
163
190
  for (const config of [
164
191
  'package.json', 'pnpm-workspace.yaml', 'pnpm-workspace.yml', 'lerna.json', 'nx.json', 'turbo.json', 'rush.json', 'workspace.json',
@@ -180,7 +207,6 @@ export async function probe({
180
207
  const contention = detectContention();
181
208
  if (contention.detected) onWarn(contentionWarning(contention));
182
209
 
183
- const startedAt = new Date().toISOString();
184
210
  const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
185
211
  const isoReal = realpathSync(iso.projectDir);
186
212
  const records = [];
@@ -215,20 +241,16 @@ export async function probe({
215
241
  // project runner is; it must resolve before the first such defender runs.
216
242
  const owned = OWNED_RUNNERS.filter((r) => r !== runner
217
243
  && !(r.name === 'python' && ['python', 'pytest', 'unittest'].includes(runner.name)));
218
- const ownedChecked = new Map();
219
- const manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
244
+ let ownedChecked = new Map();
245
+ let manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
220
246
  if (!commandTemplate) {
221
- for (const r of owned) {
222
- const check = await r.check({ projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs });
223
- commandBudget?.assertOpen();
224
- ownedChecked.set(r, check);
225
- if (!check.ok) continue;
226
- try {
227
- manifests.set(r, await discoverRunnerManifest(r, check, iso.projectDir, commandBudget?.runBudget(budgetMs) ?? budgetMs));
228
- } catch (error) {
229
- throw new PreconditionError(`${r.name}: test discovery failed: ${error.message}`);
230
- }
231
- }
247
+ try {
248
+ ({ ownedChecked, manifests } = await collectOwnedManifests(runner, primaryManifest, {
249
+ projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs,
250
+ budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
251
+ assertOpen: () => commandBudget?.assertOpen(),
252
+ }));
253
+ } catch (error) { throw new PreconditionError(`test discovery failed: ${error.message}`); }
232
254
  }
233
255
  const ensureOwned = async (r, file) => {
234
256
  if (!ownedChecked.has(r)) {
@@ -262,9 +284,7 @@ export async function probe({
262
284
  : [...ownerByFile.keys()]).sort());
263
285
  const testUniverseHash = commandTemplate
264
286
  ? sha256(JSON.stringify({ schemaVersion: 1, source: 'custom-command-static', files: allTests }))
265
- : sha256(JSON.stringify([...manifests]
266
- .map(([r, manifest]) => ({ runner: r.name, hash: manifest.testUniverseHash }))
267
- .sort((a, b) => a.runner.localeCompare(b.runner))));
287
+ : hashNativeTestUniverse(manifests);
268
288
  const baselineCache = new Map();
269
289
  const discoveryHashCache = new Map();
270
290
  // The negative control is charged per (target file, defender set): two
@@ -274,6 +294,7 @@ export async function probe({
274
294
  // runner — a `.py` source file is Python's whatever the project runs.
275
295
  const fatalEditFor = (file) => runnerFor(iso.projectDir, file, runner).fatalEdit?.();
276
296
  const prior = previous && previous.run.confirmRuns === confirmRuns
297
+ && sameWorkerPolicy(previous.run, { workers, serial, runnerCommand })
277
298
  ? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
278
299
  : new Map();
279
300
  const partitionConfigured = (files) => {
@@ -292,7 +313,7 @@ export async function probe({
292
313
  // file the interpreter actually loaded (Python) reports it back.
293
314
  const runDefenders = async (files, targets = []) => {
294
315
  if (commandTemplate) {
295
- const result = await runner.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, targets });
316
+ const result = await timedRun(runner, { projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, workers, targets });
296
317
  commandBudget?.assertOpen();
297
318
  return result;
298
319
  }
@@ -301,7 +322,7 @@ export async function probe({
301
322
  for (const [r, group] of groups) {
302
323
  commandBudget?.assertOpen();
303
324
  if (r !== runner) await ensureOwned(r, group[0]);
304
- parts.push(await r.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, targets }));
325
+ parts.push(await timedRun(r, { projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, workers, targets }));
305
326
  commandBudget?.assertOpen();
306
327
  }
307
328
  return mergeRuns(parts);
@@ -315,20 +336,21 @@ export async function probe({
315
336
 
316
337
  for (const claim of scopedClaims) {
317
338
  commandBudget?.assertOpen();
318
- const declared = claim.defendedBy?.length ? resolveDefenders(iso.projectDir, claim.defendedBy) : null;
319
- if (declared && !commandTemplate) {
320
- const excluded = declared.filter((file) => !ownerByFile.has(file));
321
- for (const file of excluded) {
322
- const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
323
- const check = expectedOwner && ownedChecked.get(expectedOwner);
324
- if (expectedOwner && check && !check.ok) {
325
- throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
326
- }
327
- }
328
- if (excluded.length) throw new PreconditionError(`${claim.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
329
- }
330
339
  for (const fault of claim.faults) {
331
340
  commandBudget?.assertOpen();
341
+ const { requested } = defenderSelection(claim, fault);
342
+ const declared = requested.length ? resolveDefenders(iso.projectDir, requested) : null;
343
+ if (declared && !commandTemplate) {
344
+ const excluded = declared.filter((file) => !ownerByFile.has(file));
345
+ for (const file of excluded) {
346
+ const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
347
+ const check = expectedOwner && ownedChecked.get(expectedOwner);
348
+ if (expectedOwner && check && !check.ok) {
349
+ throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
350
+ }
351
+ }
352
+ if (excluded.length) throw new PreconditionError(`${claim.id}/${fault.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
353
+ }
332
354
  // Mock-awareness: a discovered file that mocks the target is not a
333
355
  // defender; a declared one that mocks it stays (the author named it)
334
356
  // but is listed, because a declared defender that mocks the subject
@@ -369,6 +391,10 @@ export async function probe({
369
391
 
370
392
  commandBudget?.assertOpen();
371
393
 
394
+ assertNotCancelled();
395
+ const dirty = isDirty(root);
396
+ measurements.elapsedMs = performance.now() - measuredStart;
397
+ measurements.overheadMs = measurements.elapsedMs - measurements.runnerMs;
372
398
  return {
373
399
  schemaVersion: 1,
374
400
  tool: { name: 'testguard', version: toolVersion },
@@ -376,11 +402,13 @@ export async function probe({
376
402
  id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
377
403
  startedAt,
378
404
  finishedAt: new Date().toISOString(),
379
- repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
405
+ repo: { head, dirty, ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
380
406
  runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
381
407
  ...(runnersUsed.size > 1 ? { runners: [...runnersUsed].map(([n, v]) => ({ name: n, ...(v ? { version: v } : {}) })) } : {}),
382
408
  confirmRuns,
383
- ...(serial ? { serial: true } : {}),
409
+ measurements,
410
+ ...(!runnerCommand ? { workers: serial ? 1 : workers } : {}),
411
+ ...(serial && !runnerCommand ? { serial: true } : {}),
384
412
  ...(contention.detected ? { contention } : {}),
385
413
  ...(confirmRuns < 3 ? { provisional: true } : {}),
386
414
  // The isolation that was actually created, not the mode that was asked
@@ -390,6 +418,7 @@ export async function probe({
390
418
  mode: iso.mode,
391
419
  },
392
420
  records,
421
+ origins: recordedOriginSummary(records),
393
422
  };
394
423
  }
395
424
 
@@ -413,7 +442,7 @@ export function errorRecord({ claim, fault, defenders, discovered, byRunner, moc
413
442
  subject: subjectOf(fault, sha256),
414
443
  verdict: 'unverifiable',
415
444
  detail: { baselineRuns: [], probeRuns: [], reason: 'probe-error', message },
416
- defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
445
+ defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
417
446
  inputs,
418
447
  };
419
448
  }
@@ -428,7 +457,7 @@ export function discoveryIndeterminateRecord({ claim, fault, defenders, discover
428
457
  subject: subjectOf(fault, sha256),
429
458
  verdict: 'unverifiable',
430
459
  detail: { baselineRuns: [], probeRuns: [], reason: 'defender-discovery-indeterminate', message },
431
- defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: false, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
460
+ defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: false, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
432
461
  inputs,
433
462
  };
434
463
  }
@@ -516,8 +545,8 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
516
545
  const subject = subjectOf(fault, sha256);
517
546
 
518
547
  // Same source, same defenders, same N, same fault: the verdict cannot have changed.
519
- if (isReusable(prior, { inputs, requested: claim.defendedBy ?? [], resolved: defenders, contentHash: subject.contentHash })) {
520
- return { ...prior, reusedFrom: prior.reusedFrom ?? priorRunId };
548
+ if (isReusable(prior, { claim, inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
549
+ return { ...prior, defenders: { ...prior.defenders, ...defenderSelection(claim, fault) }, reusedFrom: prior.reusedFrom ?? priorRunId };
521
550
  }
522
551
 
523
552
  const detail = { baselineRuns: [], probeRuns: [] };
@@ -655,7 +684,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
655
684
  subject,
656
685
  verdict,
657
686
  detail,
658
- defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(allSignals.length ? { signals: allSignals } : {}) },
687
+ defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(allSignals.length ? { signals: allSignals } : {}) },
659
688
  inputs,
660
689
  rank: rank({ severity: claim.severity, sourceKind: claim.source.kind, blast, independence: detail.independence?.class }),
661
690
  };
@@ -1,3 +1,4 @@
1
+ import { registerChild } from './lifecycle.mjs';
1
2
  import { spawn } from 'node:child_process';
2
3
  import { randomBytes } from 'node:crypto';
3
4
  import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, realpathSync, rmSync, statSync } from 'node:fs';
@@ -177,6 +178,7 @@ export function runDiscoveryProcess({
177
178
  return;
178
179
  }
179
180
 
181
+ registerChild(child, () => terminateProcessTree(child));
180
182
  const started = Date.now();
181
183
  const stdout = [];
182
184
  const stderr = [];
@@ -38,8 +38,8 @@ export async function discoverTests({ projectDir, version, timeoutMs, maxOutputB
38
38
  }
39
39
  // --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
40
40
  /** The command line, as data. Exported so it is falsifiable without spawning anything. */
41
- export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
42
- [...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial ? ['--runInBand'] : []), '--runTestsByPath', ...files];
41
+ export const argvFor = (projectDir, files, outFile, { serial = false, workers = 1 } = {}) =>
42
+ [...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial || workers === 1 ? ['--runInBand'] : [`--maxWorkers=${workers}`]), '--runTestsByPath', ...files];
43
43
  export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
44
44
 
45
45
  /**
@@ -0,0 +1,58 @@
1
+ /** Own only children and cleanups registered by this TestGuard process. */
2
+ const children = new Map();
3
+ const cleanups = new Map();
4
+ let installed = false;
5
+ let cancelling = false;
6
+
7
+ function cleanup() {
8
+ // Restore mutations before removing isolation, even across successive probes.
9
+ for (const [fn] of [...cleanups].reverse().sort((a, b) => b[1] - a[1])) {
10
+ try { fn(); } catch (error) { process.stderr.write(`cleanup failed: ${error.message}\n`); }
11
+ }
12
+ cleanups.clear();
13
+ }
14
+
15
+ function install() {
16
+ if (installed) return;
17
+ installed = true;
18
+ for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
19
+ process.on(signal, async () => {
20
+ if (cancelling) return;
21
+ cancelling = true;
22
+ const waits = [...children].map(([child, stop]) => {
23
+ const closed = new Promise((resolve) => child.once('close', resolve));
24
+ stop();
25
+ return closed;
26
+ });
27
+ let timer;
28
+ await Promise.race([
29
+ Promise.allSettled(waits),
30
+ new Promise((resolve) => { timer = setTimeout(resolve, 1000); }),
31
+ ]);
32
+ clearTimeout(timer);
33
+ cleanup();
34
+ process.exit(130);
35
+ });
36
+ }
37
+ process.on('exit', () => {
38
+ for (const stop of children.values()) stop();
39
+ cleanup();
40
+ });
41
+ }
42
+
43
+ export function registerChild(child, stop) {
44
+ install();
45
+ children.set(child, stop);
46
+ child.once('close', () => children.delete(child));
47
+ if (cancelling) stop();
48
+ }
49
+
50
+ export function registerCleanup(fn, { priority = 0 } = {}) {
51
+ install();
52
+ cleanups.set(fn, priority);
53
+ return () => cleanups.delete(fn);
54
+ }
55
+
56
+ export function assertNotCancelled() {
57
+ if (cancelling) throw new Error('TestGuard command cancelled; no partial result may be published');
58
+ }
@@ -1,7 +1,7 @@
1
1
  import { existsSync, readFileSync } from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import { isAbsolute, join, relative, resolve, sep } from 'node:path';
4
- import { npx, runProcess, listTestFiles, firstInformativeLine, runnerArgv } from './shared.mjs';
4
+ import { runProcess, listTestFiles, firstInformativeLine, runnerArgv } from './shared.mjs';
5
5
  import { createDiscoveryManifest, DiscoveryError, hashDiscoveryConfigs, normalizeDiscoveredFiles, runDiscoveryReportProcess } from './discovery.mjs';
6
6
  import { globToRegExp } from '../../util/glob.mjs';
7
7
 
@@ -185,7 +185,7 @@ export function parseReport(report, durationMs) {
185
185
 
186
186
  export const run = (opts) => {
187
187
  const env = { PLAYWRIGHT_JSON_OUTPUT_FILE: '{out}', PLAYWRIGHT_JSON_OUTPUT_NAME: '{out}' };
188
- return runProcess({ ...opts, env, parse: parseReport, argv: (files) => [npx, 'playwright', 'test', '--reporter=json', ...(opts.serial ? ['--workers=1'] : []), ...files] });
188
+ return runProcess({ ...opts, env, parse: parseReport, argv: (files) => argvFor(opts.projectDir, files, opts) });
189
189
  };
190
190
 
191
191
  /**
@@ -196,3 +196,7 @@ export const run = (opts) => {
196
196
  * contained before.
197
197
  */
198
198
  export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';
199
+
200
+ /** Run the project-pinned CLI directly, avoiding an npm subprocess per confirmation. */
201
+ export const argvFor = (projectDir, files, { serial = false, workers = 1 } = {}) =>
202
+ [...runnerArgv(projectDir, '@playwright/test', 'playwright'), 'test', '--reporter=json', `--workers=${serial ? 1 : workers}`, ...files];
@@ -1,4 +1,6 @@
1
+ import { performance } from 'node:perf_hooks';
1
2
  import { existsSync, readFileSync } from 'node:fs';
3
+ import { registerChild } from './lifecycle.mjs';
2
4
  import { spawn } from 'node:child_process';
3
5
  import { dirname, join } from 'node:path';
4
6
  import { fileURLToPath } from 'node:url';
@@ -120,9 +122,10 @@ function ask(path, budgetMs) {
120
122
  } catch {
121
123
  return resolve(null);
122
124
  }
125
+ registerChild(child, () => terminateProcessTree(child));
123
126
  let out = '';
124
127
  child.on('error', () => resolve(null));
125
- child.stdout.on('data', (d) => (out += d));
128
+ child.stdout.on('data', (d) => (out = (out + d).slice(-65536)));
126
129
  child.stderr.on('data', () => {});
127
130
  const timer = setTimeout(() => terminateProcessTree(child), budgetMs);
128
131
  child.on('close', (code) => {
@@ -276,10 +279,10 @@ export function environment(targets) {
276
279
  * the configured value; the rest of the project's `addopts` is left alone,
277
280
  * because TestGuard runs a project's tests the way the project runs them.
278
281
  */
279
- export function argvFor({ engine, interpreter, files, serial }) {
282
+ export function argvFor({ engine, interpreter, files, serial = true }) {
280
283
  if (engine === 'pytest') {
281
284
  return [interpreter, '-m', 'pytest', '-p', '_testguard_pytest_plugin', '-p', 'no:cacheprovider',
282
- '--maxfail=0', '-q', ...(serial ? ['-p', 'no:xdist'] : []), ...files];
285
+ '--maxfail=0', '-q', '-p', 'no:xdist', ...files];
283
286
  }
284
287
  return [interpreter, '-m', '_testguard_unittest_main', ...files];
285
288
  }
@@ -287,12 +290,15 @@ export function argvFor({ engine, interpreter, files, serial }) {
287
290
  export function makeRun(pinned) {
288
291
  const check = makeCheck(pinned);
289
292
  return async (opts) => {
290
- const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python });
293
+ const deadline = performance.now() + (opts.budgetMs ?? 120_000);
294
+ const remaining = () => Math.max(1, Math.floor(deadline - performance.now()));
295
+ const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python, budgetMs: remaining(), budgetFor: remaining });
291
296
  if (!resolved.ok) {
292
297
  return { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs: 0 }, timeouts: 0, loadMessage: resolved.message, failedTests: [], provenance: {} };
293
298
  }
294
299
  return runProcess({
295
300
  ...opts,
301
+ budgetMs: remaining(),
296
302
  env: environment(opts.targets),
297
303
  parse: parseReport,
298
304
  argv: (files) => argvFor({ engine: resolved.engine, interpreter: resolved.interpreter, files, serial: opts.serial }),
@@ -1,4 +1,6 @@
1
1
  import { spawn, spawnSync } from 'node:child_process';
2
+ import { registerChild, assertNotCancelled } from './lifecycle.mjs';
3
+ import { readBoundedJsonFile } from './discovery.mjs';
2
4
  import { existsSync, readFileSync, rmSync } from 'node:fs';
3
5
  import { createRequire } from 'node:module';
4
6
  import { dirname } from 'node:path';
@@ -16,7 +18,7 @@ export const NODE_NOISE = /MODULE_TYPELESS_PACKAGE_JSON|ExperimentalWarning|--tr
16
18
 
17
19
  function descendantPids(rootPid) {
18
20
  if (process.platform === 'win32') return [];
19
- const result = spawnSync('ps', ['-axo', 'pid=,ppid='], { encoding: 'utf8' });
21
+ const result = spawnSync('ps', ['-axo', 'pid=,ppid='], { encoding: 'utf8', timeout: 1000, maxBuffer: 4 * 1024 * 1024 });
20
22
  if (result.status !== 0) return [];
21
23
  const children = new Map();
22
24
  for (const line of result.stdout.split('\n')) {
@@ -43,7 +45,7 @@ export const TIMEOUT_CLEANUP_WARNING = 'cleanup-unverified: TestGuard hard-kille
43
45
  export function terminateProcessTree(child) {
44
46
  if (!child.pid) return;
45
47
  if (process.platform === 'win32') {
46
- spawnSync('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' });
48
+ spawnSync('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore', timeout: 1000 });
47
49
  try { child.kill('SIGKILL'); } catch {}
48
50
  return;
49
51
  }
@@ -147,11 +149,12 @@ export function checkRunner({ projectDir, pkg, bin, budgetMs = 60_000, env = pro
147
149
  } catch (e) {
148
150
  return resolve({ ok: false, message: e.message });
149
151
  }
152
+ registerChild(child, () => terminateProcessTree(child));
150
153
  let out = '';
151
154
  let err = '';
152
155
  child.on('error', () => resolve({ ok: false, message: `${pkg} is not installed in the project and \`${bin}\` is not on PATH (npm i -D ${pkg})` }));
153
- child.stdout.on('data', (d) => (out += d));
154
- child.stderr.on('data', (d) => (err += d));
156
+ child.stdout.on('data', (d) => (out = (out + d).slice(-65536)));
157
+ child.stderr.on('data', (d) => (err = (err + d).slice(-65536)));
155
158
  const timer = setTimeout(() => terminateProcessTree(child), budgetMs);
156
159
  child.on('close', (code) => {
157
160
  clearTimeout(timer);
@@ -239,6 +242,7 @@ export function parseReport(report, durationMs) {
239
242
  * override it.
240
243
  */
241
244
  export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv, env = {}, parse = parseReport }) {
245
+ assertNotCancelled();
242
246
  const outFile = join(tmpdir(), `testguard-run-${randomBytes(6).toString('hex')}.json`);
243
247
  const [cmd, ...args] = command
244
248
  ?? (commandTemplate ? expandCommand(commandTemplate, files, outFile) : argv(files, outFile));
@@ -248,11 +252,13 @@ export function runProcess({ projectDir, files, budgetMs = 120_000, command, com
248
252
 
249
253
  return new Promise((resolve) => {
250
254
  const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], detached: process.platform !== 'win32', env: { ...process.env, CI: '1', FORCE_COLOR: '0', ...extraEnv } });
255
+ registerChild(child, () => terminateProcessTree(child));
256
+ child.stdout.resume(); // Reports are read from the file; drain logs to prevent pipe backpressure.
251
257
  let stderr = '';
252
258
  // Node prints MODULE_TYPELESS_PACKAGE_JSON (and friends) for the PROBED
253
259
  // project's config, not for anything the fault did. Keep it out of the
254
260
  // stream and out of `loadMessage`, which names the cause of a load error.
255
- child.stderr.on('data', (d) => (stderr += String(d).split('\n').filter((l) => !NODE_NOISE.test(l)).join('\n')));
261
+ child.stderr.on('data', (d) => (stderr = (stderr + String(d).split('\n').filter((l) => !NODE_NOISE.test(l)).join('\n')).slice(-65536)));
256
262
  child.on('error', (e) => (stderr += e.message));
257
263
  let killed = false;
258
264
  const timer = setTimeout(() => {
@@ -270,7 +276,7 @@ export function runProcess({ projectDir, files, budgetMs = 120_000, command, com
270
276
  result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: stderr.trim().split('\n').filter(Boolean).slice(-1)[0] ?? 'runner produced no report', failedTests: [] };
271
277
  } else {
272
278
  try {
273
- result = parse(JSON.parse(readFileSync(outFile, 'utf8')), durationMs);
279
+ result = parse(readBoundedJsonFile(outFile), durationMs);
274
280
  } catch (e) {
275
281
  result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: `unreadable report: ${e.message}`, failedTests: [] };
276
282
  }
@@ -10,7 +10,7 @@ const DISCOVERY_CONFIG_FILES = [
10
10
  'vitest.workspace.json',
11
11
  ];
12
12
  /** Ask Vitest itself to load config and enumerate the files it would collect. */
13
- export const discoveryArgvFor = (projectDir) => [...runnerArgv(projectDir, 'vitest', 'vitest'), 'list', '--filesOnly', '--passWithNoTests'];
13
+ export const discoveryArgvFor = (projectDir) => [...runnerArgv(projectDir, 'vitest', 'vitest'), 'list', '--filesOnly', '--passWithNoTests', '--maxWorkers=1'];
14
14
 
15
15
  export function parseDiscoveryOutput(stdout) {
16
16
  if (typeof stdout !== 'string') throw new TypeError('Vitest discovery output must be text');
@@ -27,8 +27,8 @@ export async function discoverTests({ projectDir, version, timeoutMs, maxOutputB
27
27
  }
28
28
  // --serial: one file at a time in one process, so a contended machine cannot turn a slow suite into a TIMEOUT verdict.
29
29
  /** The command line, as data. Exported so it is falsifiable without spawning anything. */
30
- export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
31
- [...runnerArgv(projectDir, 'vitest', 'vitest'), 'run', ...files, '--reporter=json', `--outputFile=${outFile}`, ...(serial ? ['--no-file-parallelism'] : [])];
30
+ export const argvFor = (projectDir, files, outFile, { serial = false, workers = 1 } = {}) =>
31
+ [...runnerArgv(projectDir, 'vitest', 'vitest'), 'run', ...files, '--reporter=json', `--outputFile=${outFile}`, `--maxWorkers=${serial ? 1 : workers}`, ...(serial || workers === 1 ? ['--no-file-parallelism'] : [])];
32
32
  export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
33
33
 
34
34
  /**
@@ -0,0 +1,47 @@
1
+ import { sha256 } from '../util/hash.mjs';
2
+ import { selectRunner, discoverRunnerManifest, OWNED_RUNNERS } from './runners/index.mjs';
3
+ import { createDiscoveryManifest, DiscoveryError } from './runners/discovery.mjs';
4
+
5
+ /** The same adapter/configuration universe used by probing, never a file-glob guess. */
6
+ export function hashNativeTestUniverse(manifests) {
7
+ const entries = [...manifests];
8
+ if (!entries.length) throw new DiscoveryError('a native test universe requires runner manifests');
9
+ if (!entries.some(([, manifest]) => manifest?.files?.length)) throw new DiscoveryError('a native test universe must contain test files');
10
+ const names = new Set();
11
+ const bound = entries.map(([runner, manifest]) => {
12
+ if (!runner?.name || names.has(runner.name) || !manifest?.runner) throw new DiscoveryError('native universe runner identities must be unique');
13
+ names.add(runner.name);
14
+ const canonical = createDiscoveryManifest({ runner: manifest.runner.name, version: manifest.runner.version, files: manifest.files, configFiles: manifest.configFiles, source: manifest.source });
15
+ if (canonical.testUniverseHash !== manifest.testUniverseHash || runner.name !== manifest.runner.name) throw new DiscoveryError('native universe manifest binding is inconsistent');
16
+ return { runner: runner.name, hash: manifest.testUniverseHash };
17
+ }).sort((a, b) => a.runner.localeCompare(b.runner));
18
+ return sha256(JSON.stringify(bound));
19
+ }
20
+
21
+ /** Shared owning-adapter collection; check failures retain existing probe semantics. */
22
+ export async function collectOwnedManifests(runner, primaryManifest, { projectDir, sourceDir, python, budgetMs, budgetFor, assertOpen = () => {} }) {
23
+ const owned = OWNED_RUNNERS.filter((r) => r !== runner
24
+ && !(r.name === 'python' && ['python', 'pytest', 'unittest'].includes(runner.name)));
25
+ const ownedChecked = new Map();
26
+ const manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
27
+ for (const r of owned) {
28
+ const check = await r.check({ projectDir, sourceDir, python, budgetMs: budgetFor ? budgetFor() : budgetMs, budgetFor });
29
+ assertOpen();
30
+ ownedChecked.set(r, check);
31
+ if (!check.ok) continue;
32
+ manifests.set(r, await discoverRunnerManifest(r, check, projectDir, budgetFor ? budgetFor() : budgetMs));
33
+ assertOpen();
34
+ }
35
+ return { owned, ownedChecked, manifests };
36
+ }
37
+
38
+ /** Fresh current-project collection for an explicitly requested policy boundary. */
39
+ export async function readNativeTestUniverse(options) {
40
+ const selected = await selectRunner({ ...options, name: options.runnerName ?? 'auto' });
41
+ options.assertOpen?.();
42
+ if (selected.error) throw new DiscoveryError(selected.error);
43
+ const collection = await collectOwnedManifests(selected.runner, selected.manifest, options);
44
+ const primaryRunner = { name: selected.engine ?? selected.runner.name, version: selected.version };
45
+ const runners = [primaryRunner, ...[...collection.ownedChecked].filter(([, check]) => check.ok).map(([runner, check]) => ({ name: check.engine ?? runner.name, version: check.version }))];
46
+ return { ...collection, primary: selected.runner, primaryRunner, runners, testUniverseHash: hashNativeTestUniverse(collection.manifests) };
47
+ }
@@ -1,3 +1,4 @@
1
+ import { registerCleanup } from './runners/lifecycle.mjs';
1
2
  import { existsSync, mkdtempSync, readdirSync, symlinkSync, mkdirSync, statSync, realpathSync } from 'node:fs';
2
3
  import { tmpdir } from 'node:os';
3
4
  import { join, relative, dirname } from 'node:path';
@@ -53,6 +54,7 @@ export function createScratch({ repoRoot, projectDir, ref = 'HEAD', scratchBase
53
54
  throw new PreconditionError(ref === 'HEAD' ? 'repository has no commits; commit first, or run with --in-place' : `ref ${ref} does not resolve to a commit`);
54
55
  }
55
56
  const dest = mkdtempSync(join(scratchBase, 'testguard-'));
57
+ const unregister = registerCleanup(() => removeWorktree(repoRoot, dest));
56
58
  addWorktree(repoRoot, dest, sha);
57
59
  const links = findNodeModules(repoRoot);
58
60
  if (nodeModules) {
@@ -71,7 +73,7 @@ export function createScratch({ repoRoot, projectDir, ref = 'HEAD', scratchBase
71
73
  sha,
72
74
  root: dest,
73
75
  projectDir: join(dest, relative(repoRoot, projectDir)),
74
- cleanup: () => removeWorktree(repoRoot, dest),
76
+ cleanup: () => { removeWorktree(repoRoot, dest); unregister(); },
75
77
  };
76
78
  }
77
79
 
package/src/render.mjs CHANGED
@@ -1,5 +1,15 @@
1
+ import { recordedOriginSummary } from '../spec/lib/origins.mjs';
2
+
1
3
  const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender', 'killed'];
2
4
 
5
+ /** Informational declarations only; never a verdict or an eligibility gate. */
6
+ export function renderOrigins(origins) {
7
+ const split = (counts) => Object.entries(counts).filter(([, n]) => n > 0).map(([kind, n]) => `${kind}: ${n}`).join(', ') || 'none';
8
+ return `origins (${origins.basis}): ${origins.claims.total} distinct claims (${split(origins.claims.byKind)}; mixed: ${origins.claims.mixed})` +
9
+ (origins.records ? `; ${origins.records.total} records (${split(origins.records.byKind)})` : '') +
10
+ ' — declared origins, not authenticated independence';
11
+ }
12
+
3
13
  /** Non-passing verdicts shout; the one pass does not. */
4
14
  export const formatVerdict = (v, provisional = false) => (v === 'killed' ? 'killed' : v.toUpperCase()) + (provisional ? '?' : '');
5
15
 
@@ -37,7 +47,13 @@ export function renderSummary(records, run) {
37
47
  const kills = records.filter((r) => r.verdict === 'killed');
38
48
  const coAuthored = kills.filter((r) => r.detail.independence?.class === 'co-authored').length;
39
49
  const quality = coAuthored ? `\n${coAuthorshipWarning(coAuthored, kills.length)}` : '';
40
- return `${run?.provisional ? 'PROVISIONAL: ' : ''}${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}${quality}`;
50
+ return `${run?.provisional ? 'PROVISIONAL: ' : ''}${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}${quality}\n${renderOrigins(recordedOriginSummary(records))}`;
51
+ }
52
+
53
+ /** Distinguish current admission, offline audit and recorded-run policy. */
54
+ export function renderOriginPolicy(policy, { basis = 'recorded' } = {}) {
55
+ const label = basis === 'offline' ? 'current declarations; offline freshness unavailable' : basis === 'current' ? 'current inputs' : 'recorded run';
56
+ return `origin policy (${label}): ${policy.state}; eligible declarations: ${policy.eligibleKinds.join(', ')}; ${policy.claims} claims / ${policy.faults} faults; ${policy.ineligibleClaims.length} ineligible claims; ${policy.nonKilledFaults.length} non-killed faults${policy.unavailableReasons.length ? `; unavailable: ${policy.unavailableReasons.join(', ')}` : ''} — declared origins, not authenticated independence${basis === 'recorded' ? '; not current freshness' : ''}`;
41
57
  }
42
58
 
43
59
  /** Survivors first, then by rank score; killed last. */