tickmarkr 2.6.6 → 2.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/adapters/claude-code.js +8 -2
  2. package/dist/adapters/codex-commit-check.d.ts +3 -2
  3. package/dist/adapters/codex-commit-check.js +37 -10
  4. package/dist/adapters/codex.js +10 -4
  5. package/dist/cli/commands/beat.d.ts +7 -2
  6. package/dist/cli/commands/beat.js +144 -77
  7. package/dist/cli/commands/fleet.js +66 -21
  8. package/dist/cli/commands/init.js +18 -7
  9. package/dist/cli/commands/plan.js +16 -18
  10. package/dist/cli/commands/verify.js +4 -2
  11. package/dist/cli/help.d.ts +1 -0
  12. package/dist/cli/help.js +14 -6
  13. package/dist/cli/index.d.ts +1 -1
  14. package/dist/cli/index.js +1 -1
  15. package/dist/drivers/herdr.js +2 -2
  16. package/dist/drivers/orca.d.ts +12 -4
  17. package/dist/drivers/orca.js +200 -39
  18. package/dist/gates/baseline.d.ts +9 -0
  19. package/dist/gates/baseline.js +14 -3
  20. package/dist/gates/cache.js +1 -1
  21. package/dist/gates/evidence.js +3 -3
  22. package/dist/gates/review.d.ts +1 -1
  23. package/dist/gates/review.js +77 -13
  24. package/dist/gates/run-gates.d.ts +21 -6
  25. package/dist/gates/run-gates.js +439 -224
  26. package/dist/gates/test-manifest.d.ts +42 -0
  27. package/dist/gates/test-manifest.js +203 -118
  28. package/dist/run/beat-lifecycle.d.ts +239 -0
  29. package/dist/run/beat-lifecycle.js +1782 -0
  30. package/dist/run/daemon.d.ts +4 -0
  31. package/dist/run/daemon.js +129 -40
  32. package/dist/run/git-trust.d.ts +102 -0
  33. package/dist/run/git-trust.js +334 -0
  34. package/dist/run/git.d.ts +24 -3
  35. package/dist/run/git.js +169 -30
  36. package/dist/run/journal.js +15 -3
  37. package/dist/run/lease.d.ts +23 -0
  38. package/dist/run/lease.js +232 -38
  39. package/dist/run/merge.js +4 -4
  40. package/dist/run/stall.js +20 -1
  41. package/dist/run/supervision.d.ts +8 -0
  42. package/dist/run/supervision.js +24 -4
  43. package/dist/run/verification-job.d.ts +53 -0
  44. package/dist/run/verification-job.js +159 -0
  45. package/dist/tui/ink/fleet-app.d.ts +16 -2
  46. package/dist/tui/ink/fleet-app.js +265 -141
  47. package/package.json +1 -1
  48. package/skills/tickmarkr-auto/SKILL.md +80 -27
  49. package/skills/tickmarkr-loop/SKILL.md +87 -27
  50. package/skills/tickmarkr-overseer/SKILL.md +125 -47
  51. package/skills/tickmarkr-overseer/scripts/watch-journal.sh +9 -7
@@ -87,6 +87,17 @@ export declare function verifyManifestReport(opts: {
87
87
  /** How much longer than its baseline measurement one file may legitimately run before it is a hang. */
88
88
  export declare const FILE_HANG_SLACK = 3;
89
89
  export declare const DEFAULT_FILE_HANG_BUDGET_MS = 60000;
90
+ /**
91
+ * No measured-green file is a hang before this much active time, whatever its baseline says (the battery ceiling still wins
92
+ * below it). A fast suite's per-file timings are a few ms, so 3 × its own killed a slower file that would have passed.
93
+ */
94
+ export declare const MIN_FILE_HANG_BUDGET_MS = 10000;
95
+ /** Tests of the baseline-derived budget's kill mechanics opt out of the floor; production never does. */
96
+ export declare const setHangBudgetFloorForTests: (ms: number) => void;
97
+ export declare const resetHangBudgetFloorForTests: () => void;
98
+ /** v2.6.8 P4 (prospectively superseding v2.5.5's "untimed = 3 × longest"): only the file's OWN usable green duration K
99
+ * buys measured slack, max(3 × K, L) with L the longest usable entry (red included) or legacy longestFile. An untimed,
100
+ * baseline-red or legacy longest-only file has no measure of its runtime and gets the finite battery ceiling. */
90
101
  export declare function fileHangBudgetMs(file: string, baselineDurations?: readonly BaselineFileDuration[] | null, ceilingMs?: number, longestFile?: BaselineFileDuration | null): number;
91
102
  /**
92
103
  * OBS-953 (+add): a per-file hang budget counted in raw wall time charged a lid-close to the file that
@@ -122,6 +133,37 @@ export declare function classifyPollGap(prev: {
122
133
  wall: number;
123
134
  mono: number;
124
135
  }, pollMs: number): HostInterruption | undefined;
136
+ /**
137
+ * v2.6.8 P4: an overdue poll on Darwin asks the kernel when it last slept and woke (`sysctl -n kern.sleeptime
138
+ * kern.waketime`), synchronously and bounded, before that poll's hang decision; the segment's overlap with the gap is
139
+ * credited as `host-suspend`. Unreadable evidence is unknown and credits 0. NAMED RESIDUAL: the kernel keeps only the
140
+ * MOST RECENT segment, so a gap holding several sleeps or maintenance DarkWakes credits only the last one — that
141
+ * under-credit fails closed: the file can still be killed as a hang, as on 2.6.7.
142
+ */
143
+ export interface SleepSegment {
144
+ sleep: number;
145
+ wake: number;
146
+ }
147
+ export interface SleepEvidence {
148
+ platform: () => NodeJS.Platform;
149
+ /** The synchronous OS-command boundary (spawnSync's shape). */
150
+ command: (file: string, args: string[], opts: {
151
+ encoding: "utf8";
152
+ timeout: number;
153
+ maxBuffer: number;
154
+ killSignal: "SIGKILL";
155
+ }) => {
156
+ status: number | null;
157
+ signal: NodeJS.Signals | null;
158
+ stdout?: string;
159
+ stderr?: string;
160
+ error?: Error;
161
+ };
162
+ /** Replaces the default sysctl reader; production supplies none. */
163
+ reader?: () => SleepSegment | undefined;
164
+ }
165
+ export declare const setSleepEvidenceForTests: (evidence: Partial<SleepEvidence>) => void;
166
+ export declare const resetSleepEvidenceForTests: () => void;
125
167
  export declare const runWithInterruptionSink: <T>(sink: (interruption: HostInterruption) => void, run: () => Promise<T>) => Promise<T>;
126
168
  export interface ManifestRunResult {
127
169
  evidenceReceipt: GateEvidenceReceipt;
@@ -1,9 +1,11 @@
1
1
  import { AsyncLocalStorage } from "node:async_hooks";
2
+ import { spawnSync } from "node:child_process";
2
3
  import { createHash, randomBytes } from "node:crypto";
3
4
  import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
4
5
  import { tmpdir } from "node:os";
5
6
  import { isAbsolute, join, relative, resolve, sep } from "node:path";
6
7
  import { performance } from "node:perf_hooks";
8
+ import { verificationPhase, withVerificationJob } from "../run/verification-job.js";
7
9
  import { TEST_REPORTER_SOURCE } from "./test-reporter.js";
8
10
  import { shq } from "../adapters/types.js";
9
11
  import { beginGateEvidence, redactGateOutput } from "./baseline.js";
@@ -329,15 +331,30 @@ export function verifyManifestReport(opts) {
329
331
  /** How much longer than its baseline measurement one file may legitimately run before it is a hang. */
330
332
  export const FILE_HANG_SLACK = 3;
331
333
  export const DEFAULT_FILE_HANG_BUDGET_MS = 60_000;
334
+ /**
335
+ * No measured-green file is a hang before this much active time, whatever its baseline says (the battery ceiling still wins
336
+ * below it). A fast suite's per-file timings are a few ms, so 3 × its own killed a slower file that would have passed.
337
+ */
338
+ export const MIN_FILE_HANG_BUDGET_MS = 10_000;
339
+ let hangBudgetFloorMs = MIN_FILE_HANG_BUDGET_MS;
340
+ /** Tests of the baseline-derived budget's kill mechanics opt out of the floor; production never does. */
341
+ export const setHangBudgetFloorForTests = (ms) => { hangBudgetFloorMs = ms; };
342
+ export const resetHangBudgetFloorForTests = () => { hangBudgetFloorMs = MIN_FILE_HANG_BUDGET_MS; };
332
343
  const usable = (n) => n !== undefined && Number.isFinite(n) && n > 0;
344
+ /** v2.6.8 P4 (prospectively superseding v2.5.5's "untimed = 3 × longest"): only the file's OWN usable green duration K
345
+ * buys measured slack, max(3 × K, L) with L the longest usable entry (red included) or legacy longestFile. An untimed,
346
+ * baseline-red or legacy longest-only file has no measure of its runtime and gets the finite battery ceiling. */
333
347
  export function fileHangBudgetMs(file, baselineDurations, ceilingMs = DEFAULT_FILE_HANG_BUDGET_MS, longestFile) {
334
348
  const ceiling = usable(ceilingMs) ? ceilingMs : DEFAULT_FILE_HANG_BUDGET_MS;
335
349
  const timings = baselineDurations?.filter((d) => usable(d.durationMs)) ?? [];
336
- const longest = Math.max(0, ...timings.map((d) => d.durationMs), usable(longestFile?.durationMs) ? longestFile.durationMs : 0);
337
- if (!longest)
350
+ // A file that was RED in the baseline ended early there: its duration is no measure of its green runtime. The
351
+ // failure is read from EVERY entry, before the usable-duration filter can drop a 0 ms or non-finite red one.
352
+ const red = baselineDurations?.some((d) => d.file === file && d.failed) ?? false;
353
+ const known = red ? undefined : timings.find((d) => d.file === file)?.durationMs;
354
+ if (known === undefined)
338
355
  return ceiling;
339
- const known = timings.find((d) => d.file === file)?.durationMs;
340
- return Math.min(ceiling, known === undefined ? FILE_HANG_SLACK * longest : Math.max(FILE_HANG_SLACK * known, longest));
356
+ const longest = Math.max(...timings.map((d) => d.durationMs), usable(longestFile?.durationMs) ? longestFile.durationMs : 0);
357
+ return Math.min(ceiling, Math.max(FILE_HANG_SLACK * known, longest));
341
358
  }
342
359
  /**
343
360
  * OBS-953 (+add): a per-file hang budget counted in raw wall time charged a lid-close to the file that
@@ -364,8 +381,29 @@ export function classifyPollGap(prev, now, pollMs) {
364
381
  return { kind: "unknown", ...gap, subtractedMs: 0 };
365
382
  return undefined;
366
383
  }
367
- /** The detected suspend a file started at `startedAt` sat through. */
368
- const suspendedSince = (interruptions, startedAt) => interruptions.reduce((sum, i) => sum + Math.min(i.subtractedMs, Math.max(0, i.to - Math.max(i.from, startedAt))), 0);
384
+ const systemSleepEvidence = { platform: () => process.platform, command: (file, args, opts) => spawnSync(file, args, opts) };
385
+ let sleepEvidence = systemSleepEvidence;
386
+ export const setSleepEvidenceForTests = (evidence) => { sleepEvidence = { ...systemSleepEvidence, ...evidence }; };
387
+ export const resetSleepEvidenceForTests = () => { sleepEvidence = systemSleepEvidence; };
388
+ const SYSCTL_TIMEOUT_MS = 1_000, SYSCTL_MAX_BYTES = 4_096;
389
+ const SYSCTL_STAMP = /^\{ sec = (\d{1,15}), usec = (\d{1,6}) \} [A-Z][a-z]{2} [A-Z][a-z]{2} +\d{1,2} \d{2}:\d{2}:\d{2} \d{4}$/;
390
+ /** Lazy: runs only from an overdue poll, never at import. A timeout or overrun of 1000 ms, a signal, nonzero exit, more
391
+ * than 4096 combined output bytes, or a truncated, unparseable or reversed pair is unknown. spawnSync SIGKILLs the child
392
+ * at either bound and joins it: a child ignoring SIGTERM cannot hold the poll, so hang enforcement still runs. */
393
+ function readDarwinSleepSegment() {
394
+ const startedMs = hangClocks.mono();
395
+ const r = sleepEvidence.command("sysctl", ["-n", "kern.sleeptime", "kern.waketime"], { encoding: "utf8", timeout: SYSCTL_TIMEOUT_MS, maxBuffer: SYSCTL_MAX_BYTES, killSignal: "SIGKILL" });
396
+ const out = r.stdout ?? "", err = r.stderr ?? "";
397
+ if (hangClocks.mono() - startedMs >= SYSCTL_TIMEOUT_MS || r.error || r.signal || r.status !== 0
398
+ || Buffer.byteLength(out) + Buffer.byteLength(err) > SYSCTL_MAX_BYTES)
399
+ return undefined;
400
+ const [sleep, wake, ...rest] = out.replace(/\n$/, "").split("\n").map((line) => SYSCTL_STAMP.exec(line))
401
+ .map((m) => m ? Number(m[1]) * 1000 + Number(m[2]) / 1000 : NaN);
402
+ return !rest.length && sleep < wake ? { sleep: sleep, wake: wake } : undefined;
403
+ }
404
+ /** Per gap, a file started at `startedAt` is credited the larger of the offset it sat through and its own overlap with
405
+ * the kernel segment: the same suspend is never counted twice, and nothing before the file started is credited. */
406
+ const suspendedSince = (credits, startedAt) => credits.reduce((sum, c) => sum + Math.max(Math.min(c.offsetMs, Math.max(0, c.to - Math.max(c.from, startedAt))), Math.max(0, c.wake - Math.max(c.sleep, startedAt))), 0);
369
407
  // The daemon journals each interruption where it ran (task gate, tip verify); no sink, no journal row.
370
408
  const interruptionSink = new AsyncLocalStorage();
371
409
  export const runWithInterruptionSink = (sink, run) => interruptionSink.run(sink, run);
@@ -384,13 +422,33 @@ export function runManifestedTest(cmd, cwd, opts) {
384
422
  let hangWallMs;
385
423
  let poll;
386
424
  const interruptions = [];
425
+ const credits = [];
426
+ let creditedUntil = -Infinity;
387
427
  const sink = interruptionSink.getStore();
388
428
  let last = { wall: hangClocks.wall(), mono: hangClocks.mono() };
389
429
  const checkHang = (atCeiling = false) => {
390
430
  const at = { wall: hangClocks.wall(), mono: hangClocks.mono() };
391
- const gap = classifyPollGap(last, at, pollMs);
431
+ let gap = classifyPollGap(last, at, pollMs);
392
432
  last = at;
393
433
  if (gap) {
434
+ let os;
435
+ if (sleepEvidence.platform() === "darwin") {
436
+ try {
437
+ os = (sleepEvidence.reader ?? readDarwinSleepSegment)();
438
+ }
439
+ catch {
440
+ os = undefined;
441
+ }
442
+ last = { wall: hangClocks.wall(), mono: hangClocks.mono() }; // the bounded read itself is no poll gap
443
+ }
444
+ // Clipped to this gap and past any segment already credited, so a repeated pair never credits twice.
445
+ const sleep = Math.max(os?.sleep ?? Infinity, gap.from, creditedUntil), wake = Math.min(os?.wake ?? -Infinity, gap.to);
446
+ const osMs = wake > sleep ? wake - sleep : 0;
447
+ if (osMs)
448
+ creditedUntil = wake;
449
+ credits.push({ from: gap.from, to: gap.to, offsetMs: gap.subtractedMs, ...(osMs ? { sleep, wake } : { sleep: 0, wake: 0 }) });
450
+ if (osMs > gap.subtractedMs)
451
+ gap = { ...gap, kind: "host-suspend", subtractedMs: osMs };
394
452
  interruptions.push(gap);
395
453
  sink?.(gap);
396
454
  }
@@ -401,9 +459,9 @@ export function runManifestedTest(cmd, cwd, opts) {
401
459
  const startedAt = report.started[file];
402
460
  if (startedAt === undefined || file in report.completed)
403
461
  continue;
404
- const budget = fileHangBudgetMs(file, opts.baselineDurations, overallCeilingMs, opts.longestFile);
462
+ const budget = Math.min(overallCeilingMs, Math.max(hangBudgetFloorMs, fileHangBudgetMs(file, opts.baselineDurations, overallCeilingMs, opts.longestFile)));
405
463
  const wallMs = at.wall - startedAt;
406
- const activeMs = wallMs - suspendedSince(interruptions, startedAt);
464
+ const activeMs = wallMs - suspendedSince(credits, startedAt);
407
465
  if (activeMs >= budget || (atCeiling && budget === overallCeilingMs)) {
408
466
  killedFile = file;
409
467
  hangBudgetMs = budget;
@@ -532,9 +590,14 @@ function strandedSingleForkFiles(files, nonce, run) {
532
590
  const parallel = files.filter(f => !scheduling[f].singleFork);
533
591
  if (!single.length || !parallel.length || single.some(f => Object.hasOwn(r.started, f) || Object.hasOwn(r.completed, f)))
534
592
  return;
535
- if (parallel.some(f => !Object.hasOwn(r.started, f)
536
- || !["passed", "skipped"].includes(r.completed[f]?.status ?? "")
537
- || (r.completed[f]?.tests?.failed ?? 0) !== 0))
593
+ if (parallel.some(f => {
594
+ const completion = r.completed[f];
595
+ if (!Object.hasOwn(r.started, f) || !completion)
596
+ return true;
597
+ if (completion.status === "failed")
598
+ return !completion.failures?.length || !(completion.tests && completion.tests.failed > 0);
599
+ return !["passed", "skipped"].includes(completion.status) || (completion.tests?.failed ?? 0) !== 0;
600
+ }))
538
601
  return;
539
602
  return single;
540
603
  }
@@ -561,115 +624,137 @@ export async function evaluateManifestedTest(cmd, cwd, opts) {
561
624
  const dir = opts.artifactDir ?? mkdtempSync(join(tmpdir(), "tickmarkr-test-report-"));
562
625
  let nonce = randomBytes(16).toString("hex");
563
626
  let reportPath = join(dir, `test-manifest-report-${nonce}.json`);
564
- const reporterPath = join(dir, `test-reporter-${nonce}.mjs`);
565
- const evidenceReceipts = [];
566
- let spawnedCommand = cmd;
567
- let manifestPath;
568
- let recovery;
569
- const { env, verification } = manifestEnvironment(cwd);
570
627
  try {
571
- const invocation = await discoverTestManifest(cmd, cwd, { dir, nonce, env, overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } });
572
- evidenceReceipts.push(...invocation.evidenceReceipts);
573
- const files = invocation.files;
574
- // R41: the EXPECTED manifest is evidence in its own right — persisted beside the report with the
575
- // exact discovery invocation, so a later reader can tell what this invocation was asked to prove
576
- // without reconstructing it from the runner's own claims.
577
- manifestPath = join(dir, `test-manifest-expected-${nonce}.json`);
578
- writeFileSync(manifestPath, JSON.stringify({
579
- nonce, verification, listingCommand: invocation.listing, listingExit: invocation.listingExit,
580
- listingStdoutSha256: createHash("sha256").update(invocation.listingStdout).digest("hex"),
581
- discoveredAt: Date.now(), files,
582
- }, null, 2) + "\n");
583
- writeFileSync(reporterPath, TEST_REPORTER_SOURCE);
584
- spawnedCommand = `${cmd}${invocation.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
585
- let invoked = await runManifestedTest(spawnedCommand, cwd, {
586
- evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: files, nonce, reportPath, env,
587
- baselineDurations: opts.baselineDurations?.map((d) => ({ ...d, file: toManifestPath(d.file, cwd) })),
588
- longestFile: opts.longestFile,
589
- overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS,
590
- pollMs: 20,
591
- });
592
- let verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode,
593
- report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
594
- hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
595
- evidenceReceipts.push(...invoked.evidenceReceipts);
596
- const interruptions = [...invoked.interruptions];
597
- const stranded = opts.allowStrandedRecovery === false ? undefined : strandedSingleForkFiles(files, nonce, invoked);
598
- if (stranded) {
599
- const first = invoked.report;
600
- const retryNonce = randomBytes(16).toString("hex");
601
- recovery = { firstNonce: nonce, firstReportPath: reportPath, retryNonce, files: stranded };
602
- nonce = retryNonce;
603
- reportPath = join(dir, `test-manifest-report-${nonce}.json`);
604
- const retryCommand = singleForkRetryCommand(opts.retryBaseCommand ?? cmd, cwd, stranded, files.filter(f => !stranded.includes(f)));
605
- const listed = await discoverTestManifest(retryCommand, cwd, { dir, nonce, env,
606
- overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } });
607
- evidenceReceipts.push(...listed.evidenceReceipts);
608
- if (listed.files.length !== stranded.length || listed.files.some(f => !stranded.includes(f)))
609
- throw new Error("single fork retry discovery does not match the stranded set");
610
- writeFileSync(join(dir, `test-manifest-expected-${nonce}.json`), JSON.stringify({ nonce, files: stranded,
611
- firstNonce: recovery.firstNonce, listingCommand: listed.listing, verification }, null, 2) + "\n");
612
- spawnedCommand = `${retryCommand}${listed.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
613
- invoked = await runManifestedTest(spawnedCommand, cwd, {
614
- evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: stranded, nonce, reportPath, env,
615
- baselineDurations: opts.baselineDurations?.map(d => ({ ...d, file: toManifestPath(d.file, cwd) })),
616
- longestFile: opts.longestFile, overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS, pollMs: 20,
617
- });
618
- evidenceReceipts.push(...invoked.evidenceReceipts);
619
- interruptions.push(...invoked.interruptions);
620
- verdict = verifyManifestReport({ manifest: stranded, nonce, exitCode: invoked.exitCode,
621
- report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
622
- hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
623
- // An all-skipped retry may contribute lifecycle accounting, but only the combined manifest
624
- // can establish executed success. Never rewrite either invocation's persisted certificate.
625
- if (verdict.pass || verdict.meta.noExecutedModules) {
626
- const retry = invoked.report;
627
- if (retry.certificate?.errors !== 0 || !Array.isArray(retry.certificate.diagnostics) || retry.certificate.diagnostics.length)
628
- verdict = { kind: "fail-closed", pass: false, details: "single fork retry has unknown or nonempty runner diagnostics", meta: { classification: "infra", infra: true } };
629
- else
630
- verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode, report: {
631
- ...retry, requested: files, started: { ...first.started, ...retry.started }, completed: { ...first.completed, ...retry.completed },
632
- } });
628
+ return await withVerificationJob(cmd, cwd, reportPath, opts.evidence, async () => {
629
+ const reporterPath = join(dir, `test-reporter-${nonce}.mjs`);
630
+ const evidenceReceipts = [];
631
+ let spawnedCommand = cmd;
632
+ let manifestPath;
633
+ let retainedFirstVerdict;
634
+ let recovery;
635
+ const { env, verification } = manifestEnvironment(cwd);
636
+ try {
637
+ const invocation = await verificationPhase("discovery", `${nonce}-listing`, join(dir, `listing-${nonce}.json`), () => discoverTestManifest(cmd, cwd, { dir, nonce, env, overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } }));
638
+ evidenceReceipts.push(...invocation.evidenceReceipts);
639
+ const files = invocation.files;
640
+ // R41: the EXPECTED manifest is evidence in its own right — persisted beside the report with the
641
+ // exact discovery invocation, so a later reader can tell what this invocation was asked to prove
642
+ // without reconstructing it from the runner's own claims.
643
+ manifestPath = join(dir, `test-manifest-expected-${nonce}.json`);
644
+ writeFileSync(manifestPath, JSON.stringify({
645
+ nonce, verification, listingCommand: invocation.listing, listingExit: invocation.listingExit,
646
+ listingStdoutSha256: createHash("sha256").update(invocation.listingStdout).digest("hex"),
647
+ discoveredAt: Date.now(), files,
648
+ }, null, 2) + "\n");
649
+ writeFileSync(reporterPath, TEST_REPORTER_SOURCE);
650
+ spawnedCommand = `${cmd}${invocation.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
651
+ let invoked = await verificationPhase("first-pass", nonce, reportPath, () => runManifestedTest(spawnedCommand, cwd, {
652
+ evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: files, nonce, reportPath, env,
653
+ baselineDurations: opts.baselineDurations?.map((d) => ({ ...d, file: toManifestPath(d.file, cwd) })),
654
+ longestFile: opts.longestFile,
655
+ overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS,
656
+ pollMs: 20,
657
+ }));
658
+ let verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode,
659
+ report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
660
+ hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
661
+ evidenceReceipts.push(...invoked.evidenceReceipts);
662
+ const interruptions = [...invoked.interruptions];
663
+ const stranded = opts.allowStrandedRecovery === false ? undefined : strandedSingleForkFiles(files, nonce, invoked);
664
+ if (stranded) {
665
+ const first = invoked.report;
666
+ if (verdict.kind === "work")
667
+ retainedFirstVerdict = verdict;
668
+ const retryNonce = randomBytes(16).toString("hex");
669
+ recovery = { firstNonce: nonce, firstReportPath: reportPath, retryNonce, files: stranded };
670
+ nonce = retryNonce;
671
+ reportPath = join(dir, `test-manifest-report-${nonce}.json`);
672
+ const retryCommand = singleForkRetryCommand(opts.retryBaseCommand ?? cmd, cwd, stranded, files.filter(f => !stranded.includes(f)));
673
+ const listed = await verificationPhase("continuation-discovery", `${nonce}-listing`, join(dir, `listing-${nonce}.json`), () => discoverTestManifest(retryCommand, cwd, { dir, nonce, env,
674
+ overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } }), recovery.firstNonce);
675
+ evidenceReceipts.push(...listed.evidenceReceipts);
676
+ if (listed.files.length !== stranded.length || listed.files.some(f => !stranded.includes(f)))
677
+ throw new Error("single fork retry discovery does not match the stranded set");
678
+ writeFileSync(join(dir, `test-manifest-expected-${nonce}.json`), JSON.stringify({ nonce, files: stranded,
679
+ firstNonce: recovery.firstNonce, listingCommand: listed.listing, verification }, null, 2) + "\n");
680
+ spawnedCommand = `${retryCommand}${listed.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
681
+ invoked = await verificationPhase("continuation", nonce, reportPath, () => runManifestedTest(spawnedCommand, cwd, {
682
+ evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: stranded, nonce, reportPath, env,
683
+ baselineDurations: opts.baselineDurations?.map(d => ({ ...d, file: toManifestPath(d.file, cwd) })),
684
+ longestFile: opts.longestFile, overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS, pollMs: 20,
685
+ }), recovery.firstNonce);
686
+ evidenceReceipts.push(...invoked.evidenceReceipts);
687
+ interruptions.push(...invoked.interruptions);
688
+ verdict = verifyManifestReport({ manifest: stranded, nonce, exitCode: invoked.exitCode,
689
+ report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
690
+ hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
691
+ // An all-skipped retry may contribute lifecycle accounting, but only the combined manifest
692
+ // can establish executed success. Never rewrite either invocation's persisted certificate.
693
+ if (verdict.pass || verdict.meta.noExecutedModules) {
694
+ const retry = invoked.report;
695
+ if (retry.certificate?.errors !== 0 || !Array.isArray(retry.certificate.diagnostics) || retry.certificate.diagnostics.length)
696
+ verdict = { kind: "fail-closed", pass: false, details: "single fork retry has unknown or nonempty runner diagnostics", meta: { classification: "infra", infra: true } };
697
+ else
698
+ verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode, report: {
699
+ ...retry, requested: files, started: { ...first.started, ...retry.started }, completed: { ...first.completed, ...retry.completed },
700
+ } });
701
+ }
702
+ const firstFailures = verifyManifestReport({ manifest: files, nonce: recovery.firstNonce, exitCode: 1, report: first });
703
+ if (firstFailures.kind === "work")
704
+ verdict = { ...firstFailures,
705
+ details: `${firstFailures.details}\ncontinuation: ${verdict.details}`,
706
+ meta: { ...firstFailures.meta, continuation: verdict.meta,
707
+ failingTests: [...new Set([...firstFailures.meta.failingTests, ...(verdict.meta.failingTests ?? [])])],
708
+ failingFiles: [...new Set([...firstFailures.meta.failingFiles, ...(verdict.meta.failingFiles ?? [])])],
709
+ failureEvidence: [...firstFailures.meta.failureEvidence, ...(verdict.meta.failureEvidence ?? [])] } };
710
+ verdict.details = `single fork retry ${nonce} after worker RPC timeout in ${recovery.firstNonce}: ${verdict.details}`;
711
+ }
712
+ // Preserve the validator's verdict and classification; runner evidence only explains it.
713
+ const evidenceReceipt = invoked.evidenceReceipt;
714
+ const evidenceRoot = opts.evidence?.artifactDir ?? dir;
715
+ const stdoutPath = join(evidenceRoot, evidenceReceipt.stdout.path);
716
+ const stderrPath = join(evidenceRoot, evidenceReceipt.stderr.path);
717
+ const stdoutTail = Buffer.from(redactGateOutput(invoked.stdout, env)).subarray(-16 * 1024);
718
+ const stderrTail = Buffer.from(redactGateOutput(invoked.stderr, env)).subarray(-16 * 1024);
719
+ const report = invoked.report?.nonce === nonce ? invoked.report : undefined;
720
+ const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
721
+ const errors = report?.certificate?.errors;
722
+ const reporterErrors = typeof errors === "number" && Number.isInteger(errors) && errors >= 0 ? errors : "unknown";
723
+ const reportedDiagnostics = report?.certificate?.diagnostics;
724
+ const runnerErrors = Array.isArray(reportedDiagnostics) ? reportedDiagnostics.filter(error => typeof error === "string") : [];
725
+ const diagnostics = !verdict.pass
726
+ ? `\nclassification: ${verdict.meta.classification ?? "unknown"}; runner-level diagnostic: never-started ${neverStarted}; reporter errors ${reporterErrors}; runner vitest`
727
+ + [...runnerErrors,
728
+ stdoutTail.toString(), stderrTail.toString()].filter(Boolean).map(text => `\n${text}`).join("")
729
+ : "";
730
+ // The interruption is reported beside the verdict, never folded into it or into raw wall service.
731
+ const interrupted = interruptions.length
732
+ ? `\nhost interruptions: ${interruptions.map(i => `${i.kind} ${Math.round(i.wallMs)}ms wall / ${Math.round(i.monoMs)}ms monotonic, ${Math.round(i.subtractedMs)}ms subtracted`).join("; ")}`
733
+ : "";
734
+ return { evidenceReceipt, evidenceReceipts, pass: verdict.pass, kind: verdict.kind,
735
+ details: verdict.details + interrupted + diagnostics,
736
+ classification: verdict.meta.classification,
737
+ meta: { ...verdict.meta, nonce, manifest: files, manifestPath, listingCommand: invocation.listing, verification,
738
+ ...(recovery ? { recovery, retryable: false } : {}), ...(interruptions.length ? { interruptions } : {}),
739
+ spawnedCommand, processExit: invoked.exitCode, pid: invoked.pid, stdoutPath, stderrPath },
740
+ exitCode: invoked.exitCode ?? -1, reportPath };
633
741
  }
634
- verdict.details = `single fork retry ${nonce} after worker RPC timeout in ${recovery.firstNonce}: ${verdict.details}`;
635
- }
636
- // Preserve the validator's verdict and classification; runner evidence only explains it.
637
- const evidenceReceipt = invoked.evidenceReceipt;
638
- const evidenceRoot = opts.evidence?.artifactDir ?? dir;
639
- const stdoutPath = join(evidenceRoot, evidenceReceipt.stdout.path);
640
- const stderrPath = join(evidenceRoot, evidenceReceipt.stderr.path);
641
- const stdoutTail = Buffer.from(redactGateOutput(invoked.stdout, env)).subarray(-16 * 1024);
642
- const stderrTail = Buffer.from(redactGateOutput(invoked.stderr, env)).subarray(-16 * 1024);
643
- const report = invoked.report?.nonce === nonce ? invoked.report : undefined;
644
- const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
645
- const errors = report?.certificate?.errors;
646
- const reporterErrors = typeof errors === "number" && Number.isInteger(errors) && errors >= 0 ? errors : "unknown";
647
- const reportedDiagnostics = report?.certificate?.diagnostics;
648
- const runnerErrors = Array.isArray(reportedDiagnostics) ? reportedDiagnostics.filter(error => typeof error === "string") : [];
649
- const diagnostics = !verdict.pass
650
- ? `\nclassification: ${verdict.meta.classification ?? "unknown"}; runner-level diagnostic: never-started ${neverStarted}; reporter errors ${reporterErrors}; runner vitest`
651
- + [...runnerErrors,
652
- stdoutTail.toString(), stderrTail.toString()].filter(Boolean).map(text => `\n${text}`).join("")
653
- : "";
654
- // The interruption is reported beside the verdict, never folded into it or into raw wall service.
655
- const interrupted = interruptions.length
656
- ? `\nhost interruptions: ${interruptions.map(i => `${i.kind} ${Math.round(i.wallMs)}ms wall / ${Math.round(i.monoMs)}ms monotonic, ${Math.round(i.subtractedMs)}ms subtracted`).join("; ")}`
657
- : "";
658
- return { evidenceReceipt, evidenceReceipts, pass: verdict.pass, kind: verdict.kind,
659
- details: verdict.details + interrupted + diagnostics,
660
- classification: verdict.meta.classification,
661
- meta: { ...verdict.meta, nonce, manifest: files, manifestPath, listingCommand: invocation.listing, verification,
662
- ...(recovery ? { recovery, retryable: false } : {}), ...(interruptions.length ? { interruptions } : {}),
663
- spawnedCommand, processExit: invoked.exitCode, pid: invoked.pid, stdoutPath, stderrPath },
664
- exitCode: invoked.exitCode ?? -1, reportPath };
742
+ catch (error) {
743
+ const evidenceReceipt = error?.evidenceReceipt;
744
+ if (evidenceReceipt)
745
+ evidenceReceipts.push(...(error.evidenceReceipts ?? [evidenceReceipt]));
746
+ return { evidenceReceipt, evidenceReceipts, pass: false, kind: retainedFirstVerdict ? "work" : "infra", classification: retainedFirstVerdict ? "regression" : "infra", exitCode: -1, reportPath,
747
+ details: (retainedFirstVerdict ? `${retainedFirstVerdict.details}\n` : "") + (recovery ? `single fork retry ${recovery.retryNonce} after ${recovery.firstNonce}: ` : "") + (error instanceof Error ? error.message : String(error)),
748
+ meta: { ...(retainedFirstVerdict?.meta ?? { classification: "infra", infra: true }), ...(retainedFirstVerdict ? { firstPass: retainedFirstVerdict.meta } : {}), manifestDiscoveryFailed: true, spawnedCommand, verification,
749
+ ...(recovery ? { recovery, retryable: false } : {}), ...(manifestPath ? { manifestPath } : {}) } };
750
+ }
751
+ });
665
752
  }
666
753
  catch (error) {
667
- const evidenceReceipt = error?.evidenceReceipt;
668
- if (evidenceReceipt)
669
- evidenceReceipts.push(...(error.evidenceReceipts ?? [evidenceReceipt]));
670
- return { evidenceReceipt, evidenceReceipts, pass: false, kind: "infra", classification: "infra", exitCode: -1, reportPath,
671
- details: (recovery ? `single fork retry ${recovery.retryNonce} after ${recovery.firstNonce}: ` : "") + (error instanceof Error ? error.message : String(error)),
672
- meta: { classification: "infra", infra: true, manifestDiscoveryFailed: true, spawnedCommand, verification,
673
- ...(recovery ? { recovery, retryable: false } : {}), ...(manifestPath ? { manifestPath } : {}) } };
754
+ const job = error;
755
+ return { pass: false, kind: "infra", classification: "infra", exitCode: -1, reportPath,
756
+ details: error instanceof Error ? error.message : String(error),
757
+ meta: { infra: true, classification: "infra", retryable: false,
758
+ verificationJobPath: job.verificationJobPath, verificationJob: job.verificationJob } };
674
759
  }
675
760
  }