tickmarkr 2.6.6 → 2.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code.js +8 -2
- package/dist/adapters/codex-commit-check.d.ts +3 -2
- package/dist/adapters/codex-commit-check.js +37 -10
- package/dist/adapters/codex.js +10 -4
- package/dist/cli/commands/beat.d.ts +7 -2
- package/dist/cli/commands/beat.js +144 -77
- package/dist/cli/commands/fleet.js +66 -21
- package/dist/cli/commands/init.js +18 -7
- package/dist/cli/commands/plan.js +16 -18
- package/dist/cli/commands/verify.js +4 -2
- package/dist/cli/help.d.ts +1 -0
- package/dist/cli/help.js +14 -6
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/drivers/herdr.js +2 -2
- package/dist/drivers/orca.d.ts +12 -4
- package/dist/drivers/orca.js +200 -39
- package/dist/gates/baseline.d.ts +9 -0
- package/dist/gates/baseline.js +14 -3
- package/dist/gates/cache.js +1 -1
- package/dist/gates/evidence.js +3 -3
- package/dist/gates/review.d.ts +1 -1
- package/dist/gates/review.js +77 -13
- package/dist/gates/run-gates.d.ts +21 -6
- package/dist/gates/run-gates.js +439 -224
- package/dist/gates/test-manifest.d.ts +42 -0
- package/dist/gates/test-manifest.js +203 -118
- package/dist/run/beat-lifecycle.d.ts +239 -0
- package/dist/run/beat-lifecycle.js +1782 -0
- package/dist/run/daemon.d.ts +4 -0
- package/dist/run/daemon.js +129 -40
- package/dist/run/git-trust.d.ts +102 -0
- package/dist/run/git-trust.js +334 -0
- package/dist/run/git.d.ts +24 -3
- package/dist/run/git.js +169 -30
- package/dist/run/journal.js +15 -3
- package/dist/run/lease.d.ts +23 -0
- package/dist/run/lease.js +232 -38
- package/dist/run/merge.js +4 -4
- package/dist/run/stall.js +20 -1
- package/dist/run/supervision.d.ts +8 -0
- package/dist/run/supervision.js +24 -4
- package/dist/run/verification-job.d.ts +53 -0
- package/dist/run/verification-job.js +159 -0
- package/dist/tui/ink/fleet-app.d.ts +16 -2
- package/dist/tui/ink/fleet-app.js +265 -141
- package/package.json +1 -1
- package/skills/tickmarkr-auto/SKILL.md +80 -27
- package/skills/tickmarkr-loop/SKILL.md +87 -27
- package/skills/tickmarkr-overseer/SKILL.md +125 -47
- package/skills/tickmarkr-overseer/scripts/watch-journal.sh +9 -7
|
@@ -87,6 +87,17 @@ export declare function verifyManifestReport(opts: {
|
|
|
87
87
|
/** How much longer than its baseline measurement one file may legitimately run before it is a hang. */
|
|
88
88
|
export declare const FILE_HANG_SLACK = 3;
|
|
89
89
|
export declare const DEFAULT_FILE_HANG_BUDGET_MS = 60000;
|
|
90
|
+
/**
|
|
91
|
+
* No measured-green file is a hang before this much active time, whatever its baseline says (the battery ceiling still wins
|
|
92
|
+
* below it). A fast suite's per-file timings are a few ms, so 3 × its own killed a slower file that would have passed.
|
|
93
|
+
*/
|
|
94
|
+
export declare const MIN_FILE_HANG_BUDGET_MS = 10000;
|
|
95
|
+
/** Tests of the baseline-derived budget's kill mechanics opt out of the floor; production never does. */
|
|
96
|
+
export declare const setHangBudgetFloorForTests: (ms: number) => void;
|
|
97
|
+
export declare const resetHangBudgetFloorForTests: () => void;
|
|
98
|
+
/** v2.6.8 P4 (prospectively superseding v2.5.5's "untimed = 3 × longest"): only the file's OWN usable green duration K
|
|
99
|
+
* buys measured slack, max(3 × K, L) with L the longest usable entry (red included) or legacy longestFile. An untimed,
|
|
100
|
+
* baseline-red or legacy longest-only file has no measure of its runtime and gets the finite battery ceiling. */
|
|
90
101
|
export declare function fileHangBudgetMs(file: string, baselineDurations?: readonly BaselineFileDuration[] | null, ceilingMs?: number, longestFile?: BaselineFileDuration | null): number;
|
|
91
102
|
/**
|
|
92
103
|
* OBS-953 (+add): a per-file hang budget counted in raw wall time charged a lid-close to the file that
|
|
@@ -122,6 +133,37 @@ export declare function classifyPollGap(prev: {
|
|
|
122
133
|
wall: number;
|
|
123
134
|
mono: number;
|
|
124
135
|
}, pollMs: number): HostInterruption | undefined;
|
|
136
|
+
/**
|
|
137
|
+
* v2.6.8 P4: an overdue poll on Darwin asks the kernel when it last slept and woke (`sysctl -n kern.sleeptime
|
|
138
|
+
* kern.waketime`), synchronously and bounded, before that poll's hang decision; the segment's overlap with the gap is
|
|
139
|
+
* credited as `host-suspend`. Unreadable evidence is unknown and credits 0. NAMED RESIDUAL: the kernel keeps only the
|
|
140
|
+
* MOST RECENT segment, so a gap holding several sleeps or maintenance DarkWakes credits only the last one — that
|
|
141
|
+
* under-credit fails closed: the file can still be killed as a hang, as on 2.6.7.
|
|
142
|
+
*/
|
|
143
|
+
export interface SleepSegment {
|
|
144
|
+
sleep: number;
|
|
145
|
+
wake: number;
|
|
146
|
+
}
|
|
147
|
+
export interface SleepEvidence {
|
|
148
|
+
platform: () => NodeJS.Platform;
|
|
149
|
+
/** The synchronous OS-command boundary (spawnSync's shape). */
|
|
150
|
+
command: (file: string, args: string[], opts: {
|
|
151
|
+
encoding: "utf8";
|
|
152
|
+
timeout: number;
|
|
153
|
+
maxBuffer: number;
|
|
154
|
+
killSignal: "SIGKILL";
|
|
155
|
+
}) => {
|
|
156
|
+
status: number | null;
|
|
157
|
+
signal: NodeJS.Signals | null;
|
|
158
|
+
stdout?: string;
|
|
159
|
+
stderr?: string;
|
|
160
|
+
error?: Error;
|
|
161
|
+
};
|
|
162
|
+
/** Replaces the default sysctl reader; production supplies none. */
|
|
163
|
+
reader?: () => SleepSegment | undefined;
|
|
164
|
+
}
|
|
165
|
+
export declare const setSleepEvidenceForTests: (evidence: Partial<SleepEvidence>) => void;
|
|
166
|
+
export declare const resetSleepEvidenceForTests: () => void;
|
|
125
167
|
export declare const runWithInterruptionSink: <T>(sink: (interruption: HostInterruption) => void, run: () => Promise<T>) => Promise<T>;
|
|
126
168
|
export interface ManifestRunResult {
|
|
127
169
|
evidenceReceipt: GateEvidenceReceipt;
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
2
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
4
|
import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
4
5
|
import { tmpdir } from "node:os";
|
|
5
6
|
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
7
|
import { performance } from "node:perf_hooks";
|
|
8
|
+
import { verificationPhase, withVerificationJob } from "../run/verification-job.js";
|
|
7
9
|
import { TEST_REPORTER_SOURCE } from "./test-reporter.js";
|
|
8
10
|
import { shq } from "../adapters/types.js";
|
|
9
11
|
import { beginGateEvidence, redactGateOutput } from "./baseline.js";
|
|
@@ -329,15 +331,30 @@ export function verifyManifestReport(opts) {
|
|
|
329
331
|
/** How much longer than its baseline measurement one file may legitimately run before it is a hang. */
|
|
330
332
|
export const FILE_HANG_SLACK = 3;
|
|
331
333
|
export const DEFAULT_FILE_HANG_BUDGET_MS = 60_000;
|
|
334
|
+
/**
|
|
335
|
+
* No measured-green file is a hang before this much active time, whatever its baseline says (the battery ceiling still wins
|
|
336
|
+
* below it). A fast suite's per-file timings are a few ms, so 3 × its own killed a slower file that would have passed.
|
|
337
|
+
*/
|
|
338
|
+
export const MIN_FILE_HANG_BUDGET_MS = 10_000;
|
|
339
|
+
let hangBudgetFloorMs = MIN_FILE_HANG_BUDGET_MS;
|
|
340
|
+
/** Tests of the baseline-derived budget's kill mechanics opt out of the floor; production never does. */
|
|
341
|
+
export const setHangBudgetFloorForTests = (ms) => { hangBudgetFloorMs = ms; };
|
|
342
|
+
export const resetHangBudgetFloorForTests = () => { hangBudgetFloorMs = MIN_FILE_HANG_BUDGET_MS; };
|
|
332
343
|
const usable = (n) => n !== undefined && Number.isFinite(n) && n > 0;
|
|
344
|
+
/** v2.6.8 P4 (prospectively superseding v2.5.5's "untimed = 3 × longest"): only the file's OWN usable green duration K
|
|
345
|
+
* buys measured slack, max(3 × K, L) with L the longest usable entry (red included) or legacy longestFile. An untimed,
|
|
346
|
+
* baseline-red or legacy longest-only file has no measure of its runtime and gets the finite battery ceiling. */
|
|
333
347
|
export function fileHangBudgetMs(file, baselineDurations, ceilingMs = DEFAULT_FILE_HANG_BUDGET_MS, longestFile) {
|
|
334
348
|
const ceiling = usable(ceilingMs) ? ceilingMs : DEFAULT_FILE_HANG_BUDGET_MS;
|
|
335
349
|
const timings = baselineDurations?.filter((d) => usable(d.durationMs)) ?? [];
|
|
336
|
-
|
|
337
|
-
|
|
350
|
+
// A file that was RED in the baseline ended early there: its duration is no measure of its green runtime. The
|
|
351
|
+
// failure is read from EVERY entry, before the usable-duration filter can drop a 0 ms or non-finite red one.
|
|
352
|
+
const red = baselineDurations?.some((d) => d.file === file && d.failed) ?? false;
|
|
353
|
+
const known = red ? undefined : timings.find((d) => d.file === file)?.durationMs;
|
|
354
|
+
if (known === undefined)
|
|
338
355
|
return ceiling;
|
|
339
|
-
const
|
|
340
|
-
return Math.min(ceiling,
|
|
356
|
+
const longest = Math.max(...timings.map((d) => d.durationMs), usable(longestFile?.durationMs) ? longestFile.durationMs : 0);
|
|
357
|
+
return Math.min(ceiling, Math.max(FILE_HANG_SLACK * known, longest));
|
|
341
358
|
}
|
|
342
359
|
/**
|
|
343
360
|
* OBS-953 (+add): a per-file hang budget counted in raw wall time charged a lid-close to the file that
|
|
@@ -364,8 +381,29 @@ export function classifyPollGap(prev, now, pollMs) {
|
|
|
364
381
|
return { kind: "unknown", ...gap, subtractedMs: 0 };
|
|
365
382
|
return undefined;
|
|
366
383
|
}
|
|
367
|
-
|
|
368
|
-
|
|
384
|
+
const systemSleepEvidence = { platform: () => process.platform, command: (file, args, opts) => spawnSync(file, args, opts) };
|
|
385
|
+
let sleepEvidence = systemSleepEvidence;
|
|
386
|
+
export const setSleepEvidenceForTests = (evidence) => { sleepEvidence = { ...systemSleepEvidence, ...evidence }; };
|
|
387
|
+
export const resetSleepEvidenceForTests = () => { sleepEvidence = systemSleepEvidence; };
|
|
388
|
+
const SYSCTL_TIMEOUT_MS = 1_000, SYSCTL_MAX_BYTES = 4_096;
|
|
389
|
+
const SYSCTL_STAMP = /^\{ sec = (\d{1,15}), usec = (\d{1,6}) \} [A-Z][a-z]{2} [A-Z][a-z]{2} +\d{1,2} \d{2}:\d{2}:\d{2} \d{4}$/;
|
|
390
|
+
/** Lazy: runs only from an overdue poll, never at import. A timeout or overrun of 1000 ms, a signal, nonzero exit, more
|
|
391
|
+
* than 4096 combined output bytes, or a truncated, unparseable or reversed pair is unknown. spawnSync SIGKILLs the child
|
|
392
|
+
* at either bound and joins it: a child ignoring SIGTERM cannot hold the poll, so hang enforcement still runs. */
|
|
393
|
+
function readDarwinSleepSegment() {
|
|
394
|
+
const startedMs = hangClocks.mono();
|
|
395
|
+
const r = sleepEvidence.command("sysctl", ["-n", "kern.sleeptime", "kern.waketime"], { encoding: "utf8", timeout: SYSCTL_TIMEOUT_MS, maxBuffer: SYSCTL_MAX_BYTES, killSignal: "SIGKILL" });
|
|
396
|
+
const out = r.stdout ?? "", err = r.stderr ?? "";
|
|
397
|
+
if (hangClocks.mono() - startedMs >= SYSCTL_TIMEOUT_MS || r.error || r.signal || r.status !== 0
|
|
398
|
+
|| Buffer.byteLength(out) + Buffer.byteLength(err) > SYSCTL_MAX_BYTES)
|
|
399
|
+
return undefined;
|
|
400
|
+
const [sleep, wake, ...rest] = out.replace(/\n$/, "").split("\n").map((line) => SYSCTL_STAMP.exec(line))
|
|
401
|
+
.map((m) => m ? Number(m[1]) * 1000 + Number(m[2]) / 1000 : NaN);
|
|
402
|
+
return !rest.length && sleep < wake ? { sleep: sleep, wake: wake } : undefined;
|
|
403
|
+
}
|
|
404
|
+
/** Per gap, a file started at `startedAt` is credited the larger of the offset it sat through and its own overlap with
|
|
405
|
+
* the kernel segment: the same suspend is never counted twice, and nothing before the file started is credited. */
|
|
406
|
+
const suspendedSince = (credits, startedAt) => credits.reduce((sum, c) => sum + Math.max(Math.min(c.offsetMs, Math.max(0, c.to - Math.max(c.from, startedAt))), Math.max(0, c.wake - Math.max(c.sleep, startedAt))), 0);
|
|
369
407
|
// The daemon journals each interruption where it ran (task gate, tip verify); no sink, no journal row.
|
|
370
408
|
const interruptionSink = new AsyncLocalStorage();
|
|
371
409
|
export const runWithInterruptionSink = (sink, run) => interruptionSink.run(sink, run);
|
|
@@ -384,13 +422,33 @@ export function runManifestedTest(cmd, cwd, opts) {
|
|
|
384
422
|
let hangWallMs;
|
|
385
423
|
let poll;
|
|
386
424
|
const interruptions = [];
|
|
425
|
+
const credits = [];
|
|
426
|
+
let creditedUntil = -Infinity;
|
|
387
427
|
const sink = interruptionSink.getStore();
|
|
388
428
|
let last = { wall: hangClocks.wall(), mono: hangClocks.mono() };
|
|
389
429
|
const checkHang = (atCeiling = false) => {
|
|
390
430
|
const at = { wall: hangClocks.wall(), mono: hangClocks.mono() };
|
|
391
|
-
|
|
431
|
+
let gap = classifyPollGap(last, at, pollMs);
|
|
392
432
|
last = at;
|
|
393
433
|
if (gap) {
|
|
434
|
+
let os;
|
|
435
|
+
if (sleepEvidence.platform() === "darwin") {
|
|
436
|
+
try {
|
|
437
|
+
os = (sleepEvidence.reader ?? readDarwinSleepSegment)();
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
os = undefined;
|
|
441
|
+
}
|
|
442
|
+
last = { wall: hangClocks.wall(), mono: hangClocks.mono() }; // the bounded read itself is no poll gap
|
|
443
|
+
}
|
|
444
|
+
// Clipped to this gap and past any segment already credited, so a repeated pair never credits twice.
|
|
445
|
+
const sleep = Math.max(os?.sleep ?? Infinity, gap.from, creditedUntil), wake = Math.min(os?.wake ?? -Infinity, gap.to);
|
|
446
|
+
const osMs = wake > sleep ? wake - sleep : 0;
|
|
447
|
+
if (osMs)
|
|
448
|
+
creditedUntil = wake;
|
|
449
|
+
credits.push({ from: gap.from, to: gap.to, offsetMs: gap.subtractedMs, ...(osMs ? { sleep, wake } : { sleep: 0, wake: 0 }) });
|
|
450
|
+
if (osMs > gap.subtractedMs)
|
|
451
|
+
gap = { ...gap, kind: "host-suspend", subtractedMs: osMs };
|
|
394
452
|
interruptions.push(gap);
|
|
395
453
|
sink?.(gap);
|
|
396
454
|
}
|
|
@@ -401,9 +459,9 @@ export function runManifestedTest(cmd, cwd, opts) {
|
|
|
401
459
|
const startedAt = report.started[file];
|
|
402
460
|
if (startedAt === undefined || file in report.completed)
|
|
403
461
|
continue;
|
|
404
|
-
const budget = fileHangBudgetMs(file, opts.baselineDurations, overallCeilingMs, opts.longestFile);
|
|
462
|
+
const budget = Math.min(overallCeilingMs, Math.max(hangBudgetFloorMs, fileHangBudgetMs(file, opts.baselineDurations, overallCeilingMs, opts.longestFile)));
|
|
405
463
|
const wallMs = at.wall - startedAt;
|
|
406
|
-
const activeMs = wallMs - suspendedSince(
|
|
464
|
+
const activeMs = wallMs - suspendedSince(credits, startedAt);
|
|
407
465
|
if (activeMs >= budget || (atCeiling && budget === overallCeilingMs)) {
|
|
408
466
|
killedFile = file;
|
|
409
467
|
hangBudgetMs = budget;
|
|
@@ -532,9 +590,14 @@ function strandedSingleForkFiles(files, nonce, run) {
|
|
|
532
590
|
const parallel = files.filter(f => !scheduling[f].singleFork);
|
|
533
591
|
if (!single.length || !parallel.length || single.some(f => Object.hasOwn(r.started, f) || Object.hasOwn(r.completed, f)))
|
|
534
592
|
return;
|
|
535
|
-
if (parallel.some(f =>
|
|
536
|
-
|
|
537
|
-
|
|
593
|
+
if (parallel.some(f => {
|
|
594
|
+
const completion = r.completed[f];
|
|
595
|
+
if (!Object.hasOwn(r.started, f) || !completion)
|
|
596
|
+
return true;
|
|
597
|
+
if (completion.status === "failed")
|
|
598
|
+
return !completion.failures?.length || !(completion.tests && completion.tests.failed > 0);
|
|
599
|
+
return !["passed", "skipped"].includes(completion.status) || (completion.tests?.failed ?? 0) !== 0;
|
|
600
|
+
}))
|
|
538
601
|
return;
|
|
539
602
|
return single;
|
|
540
603
|
}
|
|
@@ -561,115 +624,137 @@ export async function evaluateManifestedTest(cmd, cwd, opts) {
|
|
|
561
624
|
const dir = opts.artifactDir ?? mkdtempSync(join(tmpdir(), "tickmarkr-test-report-"));
|
|
562
625
|
let nonce = randomBytes(16).toString("hex");
|
|
563
626
|
let reportPath = join(dir, `test-manifest-report-${nonce}.json`);
|
|
564
|
-
const reporterPath = join(dir, `test-reporter-${nonce}.mjs`);
|
|
565
|
-
const evidenceReceipts = [];
|
|
566
|
-
let spawnedCommand = cmd;
|
|
567
|
-
let manifestPath;
|
|
568
|
-
let recovery;
|
|
569
|
-
const { env, verification } = manifestEnvironment(cwd);
|
|
570
627
|
try {
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
628
|
+
return await withVerificationJob(cmd, cwd, reportPath, opts.evidence, async () => {
|
|
629
|
+
const reporterPath = join(dir, `test-reporter-${nonce}.mjs`);
|
|
630
|
+
const evidenceReceipts = [];
|
|
631
|
+
let spawnedCommand = cmd;
|
|
632
|
+
let manifestPath;
|
|
633
|
+
let retainedFirstVerdict;
|
|
634
|
+
let recovery;
|
|
635
|
+
const { env, verification } = manifestEnvironment(cwd);
|
|
636
|
+
try {
|
|
637
|
+
const invocation = await verificationPhase("discovery", `${nonce}-listing`, join(dir, `listing-${nonce}.json`), () => discoverTestManifest(cmd, cwd, { dir, nonce, env, overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } }));
|
|
638
|
+
evidenceReceipts.push(...invocation.evidenceReceipts);
|
|
639
|
+
const files = invocation.files;
|
|
640
|
+
// R41: the EXPECTED manifest is evidence in its own right — persisted beside the report with the
|
|
641
|
+
// exact discovery invocation, so a later reader can tell what this invocation was asked to prove
|
|
642
|
+
// without reconstructing it from the runner's own claims.
|
|
643
|
+
manifestPath = join(dir, `test-manifest-expected-${nonce}.json`);
|
|
644
|
+
writeFileSync(manifestPath, JSON.stringify({
|
|
645
|
+
nonce, verification, listingCommand: invocation.listing, listingExit: invocation.listingExit,
|
|
646
|
+
listingStdoutSha256: createHash("sha256").update(invocation.listingStdout).digest("hex"),
|
|
647
|
+
discoveredAt: Date.now(), files,
|
|
648
|
+
}, null, 2) + "\n");
|
|
649
|
+
writeFileSync(reporterPath, TEST_REPORTER_SOURCE);
|
|
650
|
+
spawnedCommand = `${cmd}${invocation.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
|
|
651
|
+
let invoked = await verificationPhase("first-pass", nonce, reportPath, () => runManifestedTest(spawnedCommand, cwd, {
|
|
652
|
+
evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: files, nonce, reportPath, env,
|
|
653
|
+
baselineDurations: opts.baselineDurations?.map((d) => ({ ...d, file: toManifestPath(d.file, cwd) })),
|
|
654
|
+
longestFile: opts.longestFile,
|
|
655
|
+
overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS,
|
|
656
|
+
pollMs: 20,
|
|
657
|
+
}));
|
|
658
|
+
let verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode,
|
|
659
|
+
report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
|
|
660
|
+
hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
|
|
661
|
+
evidenceReceipts.push(...invoked.evidenceReceipts);
|
|
662
|
+
const interruptions = [...invoked.interruptions];
|
|
663
|
+
const stranded = opts.allowStrandedRecovery === false ? undefined : strandedSingleForkFiles(files, nonce, invoked);
|
|
664
|
+
if (stranded) {
|
|
665
|
+
const first = invoked.report;
|
|
666
|
+
if (verdict.kind === "work")
|
|
667
|
+
retainedFirstVerdict = verdict;
|
|
668
|
+
const retryNonce = randomBytes(16).toString("hex");
|
|
669
|
+
recovery = { firstNonce: nonce, firstReportPath: reportPath, retryNonce, files: stranded };
|
|
670
|
+
nonce = retryNonce;
|
|
671
|
+
reportPath = join(dir, `test-manifest-report-${nonce}.json`);
|
|
672
|
+
const retryCommand = singleForkRetryCommand(opts.retryBaseCommand ?? cmd, cwd, stranded, files.filter(f => !stranded.includes(f)));
|
|
673
|
+
const listed = await verificationPhase("continuation-discovery", `${nonce}-listing`, join(dir, `listing-${nonce}.json`), () => discoverTestManifest(retryCommand, cwd, { dir, nonce, env,
|
|
674
|
+
overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } }), recovery.firstNonce);
|
|
675
|
+
evidenceReceipts.push(...listed.evidenceReceipts);
|
|
676
|
+
if (listed.files.length !== stranded.length || listed.files.some(f => !stranded.includes(f)))
|
|
677
|
+
throw new Error("single fork retry discovery does not match the stranded set");
|
|
678
|
+
writeFileSync(join(dir, `test-manifest-expected-${nonce}.json`), JSON.stringify({ nonce, files: stranded,
|
|
679
|
+
firstNonce: recovery.firstNonce, listingCommand: listed.listing, verification }, null, 2) + "\n");
|
|
680
|
+
spawnedCommand = `${retryCommand}${listed.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
|
|
681
|
+
invoked = await verificationPhase("continuation", nonce, reportPath, () => runManifestedTest(spawnedCommand, cwd, {
|
|
682
|
+
evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: stranded, nonce, reportPath, env,
|
|
683
|
+
baselineDurations: opts.baselineDurations?.map(d => ({ ...d, file: toManifestPath(d.file, cwd) })),
|
|
684
|
+
longestFile: opts.longestFile, overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS, pollMs: 20,
|
|
685
|
+
}), recovery.firstNonce);
|
|
686
|
+
evidenceReceipts.push(...invoked.evidenceReceipts);
|
|
687
|
+
interruptions.push(...invoked.interruptions);
|
|
688
|
+
verdict = verifyManifestReport({ manifest: stranded, nonce, exitCode: invoked.exitCode,
|
|
689
|
+
report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
|
|
690
|
+
hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
|
|
691
|
+
// An all-skipped retry may contribute lifecycle accounting, but only the combined manifest
|
|
692
|
+
// can establish executed success. Never rewrite either invocation's persisted certificate.
|
|
693
|
+
if (verdict.pass || verdict.meta.noExecutedModules) {
|
|
694
|
+
const retry = invoked.report;
|
|
695
|
+
if (retry.certificate?.errors !== 0 || !Array.isArray(retry.certificate.diagnostics) || retry.certificate.diagnostics.length)
|
|
696
|
+
verdict = { kind: "fail-closed", pass: false, details: "single fork retry has unknown or nonempty runner diagnostics", meta: { classification: "infra", infra: true } };
|
|
697
|
+
else
|
|
698
|
+
verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode, report: {
|
|
699
|
+
...retry, requested: files, started: { ...first.started, ...retry.started }, completed: { ...first.completed, ...retry.completed },
|
|
700
|
+
} });
|
|
701
|
+
}
|
|
702
|
+
const firstFailures = verifyManifestReport({ manifest: files, nonce: recovery.firstNonce, exitCode: 1, report: first });
|
|
703
|
+
if (firstFailures.kind === "work")
|
|
704
|
+
verdict = { ...firstFailures,
|
|
705
|
+
details: `${firstFailures.details}\ncontinuation: ${verdict.details}`,
|
|
706
|
+
meta: { ...firstFailures.meta, continuation: verdict.meta,
|
|
707
|
+
failingTests: [...new Set([...firstFailures.meta.failingTests, ...(verdict.meta.failingTests ?? [])])],
|
|
708
|
+
failingFiles: [...new Set([...firstFailures.meta.failingFiles, ...(verdict.meta.failingFiles ?? [])])],
|
|
709
|
+
failureEvidence: [...firstFailures.meta.failureEvidence, ...(verdict.meta.failureEvidence ?? [])] } };
|
|
710
|
+
verdict.details = `single fork retry ${nonce} after worker RPC timeout in ${recovery.firstNonce}: ${verdict.details}`;
|
|
711
|
+
}
|
|
712
|
+
// Preserve the validator's verdict and classification; runner evidence only explains it.
|
|
713
|
+
const evidenceReceipt = invoked.evidenceReceipt;
|
|
714
|
+
const evidenceRoot = opts.evidence?.artifactDir ?? dir;
|
|
715
|
+
const stdoutPath = join(evidenceRoot, evidenceReceipt.stdout.path);
|
|
716
|
+
const stderrPath = join(evidenceRoot, evidenceReceipt.stderr.path);
|
|
717
|
+
const stdoutTail = Buffer.from(redactGateOutput(invoked.stdout, env)).subarray(-16 * 1024);
|
|
718
|
+
const stderrTail = Buffer.from(redactGateOutput(invoked.stderr, env)).subarray(-16 * 1024);
|
|
719
|
+
const report = invoked.report?.nonce === nonce ? invoked.report : undefined;
|
|
720
|
+
const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
|
|
721
|
+
const errors = report?.certificate?.errors;
|
|
722
|
+
const reporterErrors = typeof errors === "number" && Number.isInteger(errors) && errors >= 0 ? errors : "unknown";
|
|
723
|
+
const reportedDiagnostics = report?.certificate?.diagnostics;
|
|
724
|
+
const runnerErrors = Array.isArray(reportedDiagnostics) ? reportedDiagnostics.filter(error => typeof error === "string") : [];
|
|
725
|
+
const diagnostics = !verdict.pass
|
|
726
|
+
? `\nclassification: ${verdict.meta.classification ?? "unknown"}; runner-level diagnostic: never-started ${neverStarted}; reporter errors ${reporterErrors}; runner vitest`
|
|
727
|
+
+ [...runnerErrors,
|
|
728
|
+
stdoutTail.toString(), stderrTail.toString()].filter(Boolean).map(text => `\n${text}`).join("")
|
|
729
|
+
: "";
|
|
730
|
+
// The interruption is reported beside the verdict, never folded into it or into raw wall service.
|
|
731
|
+
const interrupted = interruptions.length
|
|
732
|
+
? `\nhost interruptions: ${interruptions.map(i => `${i.kind} ${Math.round(i.wallMs)}ms wall / ${Math.round(i.monoMs)}ms monotonic, ${Math.round(i.subtractedMs)}ms subtracted`).join("; ")}`
|
|
733
|
+
: "";
|
|
734
|
+
return { evidenceReceipt, evidenceReceipts, pass: verdict.pass, kind: verdict.kind,
|
|
735
|
+
details: verdict.details + interrupted + diagnostics,
|
|
736
|
+
classification: verdict.meta.classification,
|
|
737
|
+
meta: { ...verdict.meta, nonce, manifest: files, manifestPath, listingCommand: invocation.listing, verification,
|
|
738
|
+
...(recovery ? { recovery, retryable: false } : {}), ...(interruptions.length ? { interruptions } : {}),
|
|
739
|
+
spawnedCommand, processExit: invoked.exitCode, pid: invoked.pid, stdoutPath, stderrPath },
|
|
740
|
+
exitCode: invoked.exitCode ?? -1, reportPath };
|
|
633
741
|
}
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
|
|
645
|
-
const errors = report?.certificate?.errors;
|
|
646
|
-
const reporterErrors = typeof errors === "number" && Number.isInteger(errors) && errors >= 0 ? errors : "unknown";
|
|
647
|
-
const reportedDiagnostics = report?.certificate?.diagnostics;
|
|
648
|
-
const runnerErrors = Array.isArray(reportedDiagnostics) ? reportedDiagnostics.filter(error => typeof error === "string") : [];
|
|
649
|
-
const diagnostics = !verdict.pass
|
|
650
|
-
? `\nclassification: ${verdict.meta.classification ?? "unknown"}; runner-level diagnostic: never-started ${neverStarted}; reporter errors ${reporterErrors}; runner vitest`
|
|
651
|
-
+ [...runnerErrors,
|
|
652
|
-
stdoutTail.toString(), stderrTail.toString()].filter(Boolean).map(text => `\n${text}`).join("")
|
|
653
|
-
: "";
|
|
654
|
-
// The interruption is reported beside the verdict, never folded into it or into raw wall service.
|
|
655
|
-
const interrupted = interruptions.length
|
|
656
|
-
? `\nhost interruptions: ${interruptions.map(i => `${i.kind} ${Math.round(i.wallMs)}ms wall / ${Math.round(i.monoMs)}ms monotonic, ${Math.round(i.subtractedMs)}ms subtracted`).join("; ")}`
|
|
657
|
-
: "";
|
|
658
|
-
return { evidenceReceipt, evidenceReceipts, pass: verdict.pass, kind: verdict.kind,
|
|
659
|
-
details: verdict.details + interrupted + diagnostics,
|
|
660
|
-
classification: verdict.meta.classification,
|
|
661
|
-
meta: { ...verdict.meta, nonce, manifest: files, manifestPath, listingCommand: invocation.listing, verification,
|
|
662
|
-
...(recovery ? { recovery, retryable: false } : {}), ...(interruptions.length ? { interruptions } : {}),
|
|
663
|
-
spawnedCommand, processExit: invoked.exitCode, pid: invoked.pid, stdoutPath, stderrPath },
|
|
664
|
-
exitCode: invoked.exitCode ?? -1, reportPath };
|
|
742
|
+
catch (error) {
|
|
743
|
+
const evidenceReceipt = error?.evidenceReceipt;
|
|
744
|
+
if (evidenceReceipt)
|
|
745
|
+
evidenceReceipts.push(...(error.evidenceReceipts ?? [evidenceReceipt]));
|
|
746
|
+
return { evidenceReceipt, evidenceReceipts, pass: false, kind: retainedFirstVerdict ? "work" : "infra", classification: retainedFirstVerdict ? "regression" : "infra", exitCode: -1, reportPath,
|
|
747
|
+
details: (retainedFirstVerdict ? `${retainedFirstVerdict.details}\n` : "") + (recovery ? `single fork retry ${recovery.retryNonce} after ${recovery.firstNonce}: ` : "") + (error instanceof Error ? error.message : String(error)),
|
|
748
|
+
meta: { ...(retainedFirstVerdict?.meta ?? { classification: "infra", infra: true }), ...(retainedFirstVerdict ? { firstPass: retainedFirstVerdict.meta } : {}), manifestDiscoveryFailed: true, spawnedCommand, verification,
|
|
749
|
+
...(recovery ? { recovery, retryable: false } : {}), ...(manifestPath ? { manifestPath } : {}) } };
|
|
750
|
+
}
|
|
751
|
+
});
|
|
665
752
|
}
|
|
666
753
|
catch (error) {
|
|
667
|
-
const
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
meta: { classification: "infra", infra: true, manifestDiscoveryFailed: true, spawnedCommand, verification,
|
|
673
|
-
...(recovery ? { recovery, retryable: false } : {}), ...(manifestPath ? { manifestPath } : {}) } };
|
|
754
|
+
const job = error;
|
|
755
|
+
return { pass: false, kind: "infra", classification: "infra", exitCode: -1, reportPath,
|
|
756
|
+
details: error instanceof Error ? error.message : String(error),
|
|
757
|
+
meta: { infra: true, classification: "infra", retryable: false,
|
|
758
|
+
verificationJobPath: job.verificationJobPath, verificationJob: job.verificationJob } };
|
|
674
759
|
}
|
|
675
760
|
}
|