tickmarkr 2.6.5 → 2.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/adapters/claude-code.js +8 -2
- package/dist/adapters/codex-commit-check.d.ts +16 -4
- package/dist/adapters/codex-commit-check.js +150 -26
- package/dist/adapters/codex.d.ts +1 -0
- package/dist/adapters/codex.js +20 -8
- package/dist/adapters/types.d.ts +4 -1
- package/dist/cli/commands/beat.d.ts +6 -1
- package/dist/cli/commands/beat.js +123 -76
- package/dist/cli/commands/doctor.js +3 -1
- package/dist/cli/commands/fleet.js +368 -85
- package/dist/cli/commands/init.js +23 -9
- package/dist/cli/commands/plan.js +12 -1
- package/dist/cli/commands/verify.js +37 -9
- package/dist/cli/help.d.ts +1 -0
- package/dist/cli/help.js +10 -5
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/config/fleet-overlay.d.ts +14 -0
- package/dist/config/fleet-overlay.js +154 -25
- package/dist/drivers/orca.d.ts +17 -3
- package/dist/drivers/orca.js +64 -20
- package/dist/gates/baseline.d.ts +9 -0
- package/dist/gates/baseline.js +14 -3
- package/dist/gates/cache.js +1 -1
- package/dist/gates/review.js +15 -0
- package/dist/gates/run-gates.d.ts +20 -7
- package/dist/gates/run-gates.js +345 -210
- package/dist/gates/test-manifest.d.ts +11 -0
- package/dist/gates/test-manifest.js +151 -111
- package/dist/gates/timeout-shaped.d.ts +3 -0
- package/dist/gates/timeout-shaped.js +10 -0
- package/dist/run/beat-lifecycle.d.ts +154 -0
- package/dist/run/beat-lifecycle.js +825 -0
- package/dist/run/daemon.js +202 -126
- package/dist/run/git-trust.d.ts +27 -0
- package/dist/run/git-trust.js +178 -0
- package/dist/run/git.js +34 -14
- package/dist/run/journal.js +15 -3
- package/dist/run/lease.d.ts +23 -0
- package/dist/run/lease.js +232 -38
- package/dist/run/merge.js +2 -2
- package/dist/run/stall.js +20 -1
- package/dist/run/supervision.d.ts +8 -0
- package/dist/run/supervision.js +24 -4
- package/dist/run/verification-job.d.ts +51 -0
- package/dist/run/verification-job.js +137 -0
- package/dist/tui/ink/fleet-app.d.ts +35 -5
- package/dist/tui/ink/fleet-app.js +244 -124
- package/package.json +1 -1
- package/skills/tickmarkr-auto/SKILL.md +19 -10
- package/skills/tickmarkr-loop/SKILL.md +19 -10
- package/skills/tickmarkr-overseer/SKILL.md +25 -16
- package/skills/tickmarkr-overseer/scripts/classify-vitest-log.sh +58 -16
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +19 -6
- package/skills/tickmarkr-overseer/scripts/watch-parks.sh +121 -64
|
@@ -87,6 +87,14 @@ export declare function verifyManifestReport(opts: {
|
|
|
87
87
|
/** How much longer than its baseline measurement one file may legitimately run before it is a hang. */
|
|
88
88
|
export declare const FILE_HANG_SLACK = 3;
|
|
89
89
|
export declare const DEFAULT_FILE_HANG_BUDGET_MS = 60000;
|
|
90
|
+
/**
|
|
91
|
+
* No file is a hang before this much active time, whatever its baseline says (the battery ceiling still wins below it). A fast
|
|
92
|
+
* suite's per-file timings are a few ms, so 3 × longest killed a new or slower file that would have passed, or a scheduling gap, as an infra hang.
|
|
93
|
+
*/
|
|
94
|
+
export declare const MIN_FILE_HANG_BUDGET_MS = 10000;
|
|
95
|
+
/** Tests of the baseline-derived budget's kill mechanics opt out of the floor; production never does. */
|
|
96
|
+
export declare const setHangBudgetFloorForTests: (ms: number) => void;
|
|
97
|
+
export declare const resetHangBudgetFloorForTests: () => void;
|
|
90
98
|
export declare function fileHangBudgetMs(file: string, baselineDurations?: readonly BaselineFileDuration[] | null, ceilingMs?: number, longestFile?: BaselineFileDuration | null): number;
|
|
91
99
|
/**
|
|
92
100
|
* OBS-953 (+add): a per-file hang budget counted in raw wall time charged a lid-close to the file that
|
|
@@ -208,4 +216,7 @@ export declare function evaluateManifestedTest(cmd: string, cwd: string, opts: {
|
|
|
208
216
|
/** OBS-1166: the configured command BEFORE a selected screen narrowed it, so a stranded retry's
|
|
209
217
|
* only positional filters are the stranded files. Absent (a full suite), `cmd` is that command. */
|
|
210
218
|
retryBaseCommand?: string;
|
|
219
|
+
/** v2.6.6 C: false on the manifest gate's one outer re-measurement — the second sample is one
|
|
220
|
+
* execution and never stacks the stranded single-fork recovery on top of the retry it already is. */
|
|
221
|
+
allowStrandedRecovery?: boolean;
|
|
211
222
|
}): Promise<ManifestGateOutcome>;
|
|
@@ -4,6 +4,7 @@ import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileS
|
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
6
|
import { performance } from "node:perf_hooks";
|
|
7
|
+
import { verificationPhase, withVerificationJob } from "../run/verification-job.js";
|
|
7
8
|
import { TEST_REPORTER_SOURCE } from "./test-reporter.js";
|
|
8
9
|
import { shq } from "../adapters/types.js";
|
|
9
10
|
import { beginGateEvidence, redactGateOutput } from "./baseline.js";
|
|
@@ -329,6 +330,15 @@ export function verifyManifestReport(opts) {
|
|
|
329
330
|
/** How much longer than its baseline measurement one file may legitimately run before it is a hang. */
|
|
330
331
|
export const FILE_HANG_SLACK = 3;
|
|
331
332
|
export const DEFAULT_FILE_HANG_BUDGET_MS = 60_000;
|
|
333
|
+
/**
|
|
334
|
+
* No file is a hang before this much active time, whatever its baseline says (the battery ceiling still wins below it). A fast
|
|
335
|
+
* suite's per-file timings are a few ms, so 3 × longest killed a new or slower file that would have passed, or a scheduling gap, as an infra hang.
|
|
336
|
+
*/
|
|
337
|
+
export const MIN_FILE_HANG_BUDGET_MS = 10_000;
|
|
338
|
+
let hangBudgetFloorMs = MIN_FILE_HANG_BUDGET_MS;
|
|
339
|
+
/** Tests of the baseline-derived budget's kill mechanics opt out of the floor; production never does. */
|
|
340
|
+
export const setHangBudgetFloorForTests = (ms) => { hangBudgetFloorMs = ms; };
|
|
341
|
+
export const resetHangBudgetFloorForTests = () => { hangBudgetFloorMs = MIN_FILE_HANG_BUDGET_MS; };
|
|
332
342
|
const usable = (n) => n !== undefined && Number.isFinite(n) && n > 0;
|
|
333
343
|
export function fileHangBudgetMs(file, baselineDurations, ceilingMs = DEFAULT_FILE_HANG_BUDGET_MS, longestFile) {
|
|
334
344
|
const ceiling = usable(ceilingMs) ? ceilingMs : DEFAULT_FILE_HANG_BUDGET_MS;
|
|
@@ -336,7 +346,10 @@ export function fileHangBudgetMs(file, baselineDurations, ceilingMs = DEFAULT_FI
|
|
|
336
346
|
const longest = Math.max(0, ...timings.map((d) => d.durationMs), usable(longestFile?.durationMs) ? longestFile.durationMs : 0);
|
|
337
347
|
if (!longest)
|
|
338
348
|
return ceiling;
|
|
339
|
-
|
|
349
|
+
// A file that was RED in the baseline ended early there: its duration is no measure of its green runtime. The
|
|
350
|
+
// failure is read from EVERY entry, before the usable-duration filter can drop a 0 ms or non-finite red one.
|
|
351
|
+
const red = baselineDurations?.some((d) => d.file === file && d.failed) ?? false;
|
|
352
|
+
const known = red ? undefined : timings.find((d) => d.file === file)?.durationMs;
|
|
340
353
|
return Math.min(ceiling, known === undefined ? FILE_HANG_SLACK * longest : Math.max(FILE_HANG_SLACK * known, longest));
|
|
341
354
|
}
|
|
342
355
|
/**
|
|
@@ -401,7 +414,7 @@ export function runManifestedTest(cmd, cwd, opts) {
|
|
|
401
414
|
const startedAt = report.started[file];
|
|
402
415
|
if (startedAt === undefined || file in report.completed)
|
|
403
416
|
continue;
|
|
404
|
-
const budget = fileHangBudgetMs(file, opts.baselineDurations, overallCeilingMs, opts.longestFile);
|
|
417
|
+
const budget = Math.min(overallCeilingMs, Math.max(hangBudgetFloorMs, fileHangBudgetMs(file, opts.baselineDurations, overallCeilingMs, opts.longestFile)));
|
|
405
418
|
const wallMs = at.wall - startedAt;
|
|
406
419
|
const activeMs = wallMs - suspendedSince(interruptions, startedAt);
|
|
407
420
|
if (activeMs >= budget || (atCeiling && budget === overallCeilingMs)) {
|
|
@@ -532,9 +545,14 @@ function strandedSingleForkFiles(files, nonce, run) {
|
|
|
532
545
|
const parallel = files.filter(f => !scheduling[f].singleFork);
|
|
533
546
|
if (!single.length || !parallel.length || single.some(f => Object.hasOwn(r.started, f) || Object.hasOwn(r.completed, f)))
|
|
534
547
|
return;
|
|
535
|
-
if (parallel.some(f =>
|
|
536
|
-
|
|
537
|
-
|
|
548
|
+
if (parallel.some(f => {
|
|
549
|
+
const completion = r.completed[f];
|
|
550
|
+
if (!Object.hasOwn(r.started, f) || !completion)
|
|
551
|
+
return true;
|
|
552
|
+
if (completion.status === "failed")
|
|
553
|
+
return !completion.failures?.length || !(completion.tests && completion.tests.failed > 0);
|
|
554
|
+
return !["passed", "skipped"].includes(completion.status) || (completion.tests?.failed ?? 0) !== 0;
|
|
555
|
+
}))
|
|
538
556
|
return;
|
|
539
557
|
return single;
|
|
540
558
|
}
|
|
@@ -561,115 +579,137 @@ export async function evaluateManifestedTest(cmd, cwd, opts) {
|
|
|
561
579
|
const dir = opts.artifactDir ?? mkdtempSync(join(tmpdir(), "tickmarkr-test-report-"));
|
|
562
580
|
let nonce = randomBytes(16).toString("hex");
|
|
563
581
|
let reportPath = join(dir, `test-manifest-report-${nonce}.json`);
|
|
564
|
-
const reporterPath = join(dir, `test-reporter-${nonce}.mjs`);
|
|
565
|
-
const evidenceReceipts = [];
|
|
566
|
-
let spawnedCommand = cmd;
|
|
567
|
-
let manifestPath;
|
|
568
|
-
let recovery;
|
|
569
|
-
const { env, verification } = manifestEnvironment(cwd);
|
|
570
582
|
try {
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
583
|
+
return await withVerificationJob(cmd, cwd, reportPath, opts.evidence, async () => {
|
|
584
|
+
const reporterPath = join(dir, `test-reporter-${nonce}.mjs`);
|
|
585
|
+
const evidenceReceipts = [];
|
|
586
|
+
let spawnedCommand = cmd;
|
|
587
|
+
let manifestPath;
|
|
588
|
+
let retainedFirstVerdict;
|
|
589
|
+
let recovery;
|
|
590
|
+
const { env, verification } = manifestEnvironment(cwd);
|
|
591
|
+
try {
|
|
592
|
+
const invocation = await verificationPhase("discovery", `${nonce}-listing`, join(dir, `listing-${nonce}.json`), () => discoverTestManifest(cmd, cwd, { dir, nonce, env, overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } }));
|
|
593
|
+
evidenceReceipts.push(...invocation.evidenceReceipts);
|
|
594
|
+
const files = invocation.files;
|
|
595
|
+
// R41: the EXPECTED manifest is evidence in its own right — persisted beside the report with the
|
|
596
|
+
// exact discovery invocation, so a later reader can tell what this invocation was asked to prove
|
|
597
|
+
// without reconstructing it from the runner's own claims.
|
|
598
|
+
manifestPath = join(dir, `test-manifest-expected-${nonce}.json`);
|
|
599
|
+
writeFileSync(manifestPath, JSON.stringify({
|
|
600
|
+
nonce, verification, listingCommand: invocation.listing, listingExit: invocation.listingExit,
|
|
601
|
+
listingStdoutSha256: createHash("sha256").update(invocation.listingStdout).digest("hex"),
|
|
602
|
+
discoveredAt: Date.now(), files,
|
|
603
|
+
}, null, 2) + "\n");
|
|
604
|
+
writeFileSync(reporterPath, TEST_REPORTER_SOURCE);
|
|
605
|
+
spawnedCommand = `${cmd}${invocation.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
|
|
606
|
+
let invoked = await verificationPhase("first-pass", nonce, reportPath, () => runManifestedTest(spawnedCommand, cwd, {
|
|
607
|
+
evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: files, nonce, reportPath, env,
|
|
608
|
+
baselineDurations: opts.baselineDurations?.map((d) => ({ ...d, file: toManifestPath(d.file, cwd) })),
|
|
609
|
+
longestFile: opts.longestFile,
|
|
610
|
+
overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS,
|
|
611
|
+
pollMs: 20,
|
|
612
|
+
}));
|
|
613
|
+
let verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode,
|
|
614
|
+
report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
|
|
615
|
+
hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
|
|
616
|
+
evidenceReceipts.push(...invoked.evidenceReceipts);
|
|
617
|
+
const interruptions = [...invoked.interruptions];
|
|
618
|
+
const stranded = opts.allowStrandedRecovery === false ? undefined : strandedSingleForkFiles(files, nonce, invoked);
|
|
619
|
+
if (stranded) {
|
|
620
|
+
const first = invoked.report;
|
|
621
|
+
if (verdict.kind === "work")
|
|
622
|
+
retainedFirstVerdict = verdict;
|
|
623
|
+
const retryNonce = randomBytes(16).toString("hex");
|
|
624
|
+
recovery = { firstNonce: nonce, firstReportPath: reportPath, retryNonce, files: stranded };
|
|
625
|
+
nonce = retryNonce;
|
|
626
|
+
reportPath = join(dir, `test-manifest-report-${nonce}.json`);
|
|
627
|
+
const retryCommand = singleForkRetryCommand(opts.retryBaseCommand ?? cmd, cwd, stranded, files.filter(f => !stranded.includes(f)));
|
|
628
|
+
const listed = await verificationPhase("continuation-discovery", `${nonce}-listing`, join(dir, `listing-${nonce}.json`), () => discoverTestManifest(retryCommand, cwd, { dir, nonce, env,
|
|
629
|
+
overallCeilingMs: opts.overallCeilingMs, evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir } }), recovery.firstNonce);
|
|
630
|
+
evidenceReceipts.push(...listed.evidenceReceipts);
|
|
631
|
+
if (listed.files.length !== stranded.length || listed.files.some(f => !stranded.includes(f)))
|
|
632
|
+
throw new Error("single fork retry discovery does not match the stranded set");
|
|
633
|
+
writeFileSync(join(dir, `test-manifest-expected-${nonce}.json`), JSON.stringify({ nonce, files: stranded,
|
|
634
|
+
firstNonce: recovery.firstNonce, listingCommand: listed.listing, verification }, null, 2) + "\n");
|
|
635
|
+
spawnedCommand = `${retryCommand}${listed.separator} --reporter=${shq(reporterPath)} --outputFile=${shq(reportPath)}`;
|
|
636
|
+
invoked = await verificationPhase("continuation", nonce, reportPath, () => runManifestedTest(spawnedCommand, cwd, {
|
|
637
|
+
evidence: { ...opts.evidence, artifactDir: opts.evidence?.artifactDir ?? dir }, manifest: stranded, nonce, reportPath, env,
|
|
638
|
+
baselineDurations: opts.baselineDurations?.map(d => ({ ...d, file: toManifestPath(d.file, cwd) })),
|
|
639
|
+
longestFile: opts.longestFile, overallCeilingMs: opts.overallCeilingMs ?? DEFAULT_FILE_HANG_BUDGET_MS, pollMs: 20,
|
|
640
|
+
}), recovery.firstNonce);
|
|
641
|
+
evidenceReceipts.push(...invoked.evidenceReceipts);
|
|
642
|
+
interruptions.push(...invoked.interruptions);
|
|
643
|
+
verdict = verifyManifestReport({ manifest: stranded, nonce, exitCode: invoked.exitCode,
|
|
644
|
+
report: invoked.report, killedFile: invoked.killedFile, hangBudgetMs: invoked.hangBudgetMs,
|
|
645
|
+
hangActiveMs: invoked.hangActiveMs, hangWallMs: invoked.hangWallMs });
|
|
646
|
+
// An all-skipped retry may contribute lifecycle accounting, but only the combined manifest
|
|
647
|
+
// can establish executed success. Never rewrite either invocation's persisted certificate.
|
|
648
|
+
if (verdict.pass || verdict.meta.noExecutedModules) {
|
|
649
|
+
const retry = invoked.report;
|
|
650
|
+
if (retry.certificate?.errors !== 0 || !Array.isArray(retry.certificate.diagnostics) || retry.certificate.diagnostics.length)
|
|
651
|
+
verdict = { kind: "fail-closed", pass: false, details: "single fork retry has unknown or nonempty runner diagnostics", meta: { classification: "infra", infra: true } };
|
|
652
|
+
else
|
|
653
|
+
verdict = verifyManifestReport({ manifest: files, nonce, exitCode: invoked.exitCode, report: {
|
|
654
|
+
...retry, requested: files, started: { ...first.started, ...retry.started }, completed: { ...first.completed, ...retry.completed },
|
|
655
|
+
} });
|
|
656
|
+
}
|
|
657
|
+
const firstFailures = verifyManifestReport({ manifest: files, nonce: recovery.firstNonce, exitCode: 1, report: first });
|
|
658
|
+
if (firstFailures.kind === "work")
|
|
659
|
+
verdict = { ...firstFailures,
|
|
660
|
+
details: `${firstFailures.details}\ncontinuation: ${verdict.details}`,
|
|
661
|
+
meta: { ...firstFailures.meta, continuation: verdict.meta,
|
|
662
|
+
failingTests: [...new Set([...firstFailures.meta.failingTests, ...(verdict.meta.failingTests ?? [])])],
|
|
663
|
+
failingFiles: [...new Set([...firstFailures.meta.failingFiles, ...(verdict.meta.failingFiles ?? [])])],
|
|
664
|
+
failureEvidence: [...firstFailures.meta.failureEvidence, ...(verdict.meta.failureEvidence ?? [])] } };
|
|
665
|
+
verdict.details = `single fork retry ${nonce} after worker RPC timeout in ${recovery.firstNonce}: ${verdict.details}`;
|
|
666
|
+
}
|
|
667
|
+
// Preserve the validator's verdict and classification; runner evidence only explains it.
|
|
668
|
+
const evidenceReceipt = invoked.evidenceReceipt;
|
|
669
|
+
const evidenceRoot = opts.evidence?.artifactDir ?? dir;
|
|
670
|
+
const stdoutPath = join(evidenceRoot, evidenceReceipt.stdout.path);
|
|
671
|
+
const stderrPath = join(evidenceRoot, evidenceReceipt.stderr.path);
|
|
672
|
+
const stdoutTail = Buffer.from(redactGateOutput(invoked.stdout, env)).subarray(-16 * 1024);
|
|
673
|
+
const stderrTail = Buffer.from(redactGateOutput(invoked.stderr, env)).subarray(-16 * 1024);
|
|
674
|
+
const report = invoked.report?.nonce === nonce ? invoked.report : undefined;
|
|
675
|
+
const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
|
|
676
|
+
const errors = report?.certificate?.errors;
|
|
677
|
+
const reporterErrors = typeof errors === "number" && Number.isInteger(errors) && errors >= 0 ? errors : "unknown";
|
|
678
|
+
const reportedDiagnostics = report?.certificate?.diagnostics;
|
|
679
|
+
const runnerErrors = Array.isArray(reportedDiagnostics) ? reportedDiagnostics.filter(error => typeof error === "string") : [];
|
|
680
|
+
const diagnostics = !verdict.pass
|
|
681
|
+
? `\nclassification: ${verdict.meta.classification ?? "unknown"}; runner-level diagnostic: never-started ${neverStarted}; reporter errors ${reporterErrors}; runner vitest`
|
|
682
|
+
+ [...runnerErrors,
|
|
683
|
+
stdoutTail.toString(), stderrTail.toString()].filter(Boolean).map(text => `\n${text}`).join("")
|
|
684
|
+
: "";
|
|
685
|
+
// The interruption is reported beside the verdict, never folded into it or into raw wall service.
|
|
686
|
+
const interrupted = interruptions.length
|
|
687
|
+
? `\nhost interruptions: ${interruptions.map(i => `${i.kind} ${Math.round(i.wallMs)}ms wall / ${Math.round(i.monoMs)}ms monotonic, ${Math.round(i.subtractedMs)}ms subtracted`).join("; ")}`
|
|
688
|
+
: "";
|
|
689
|
+
return { evidenceReceipt, evidenceReceipts, pass: verdict.pass, kind: verdict.kind,
|
|
690
|
+
details: verdict.details + interrupted + diagnostics,
|
|
691
|
+
classification: verdict.meta.classification,
|
|
692
|
+
meta: { ...verdict.meta, nonce, manifest: files, manifestPath, listingCommand: invocation.listing, verification,
|
|
693
|
+
...(recovery ? { recovery, retryable: false } : {}), ...(interruptions.length ? { interruptions } : {}),
|
|
694
|
+
spawnedCommand, processExit: invoked.exitCode, pid: invoked.pid, stdoutPath, stderrPath },
|
|
695
|
+
exitCode: invoked.exitCode ?? -1, reportPath };
|
|
633
696
|
}
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
|
|
645
|
-
const errors = report?.certificate?.errors;
|
|
646
|
-
const reporterErrors = typeof errors === "number" && Number.isInteger(errors) && errors >= 0 ? errors : "unknown";
|
|
647
|
-
const reportedDiagnostics = report?.certificate?.diagnostics;
|
|
648
|
-
const runnerErrors = Array.isArray(reportedDiagnostics) ? reportedDiagnostics.filter(error => typeof error === "string") : [];
|
|
649
|
-
const diagnostics = !verdict.pass
|
|
650
|
-
? `\nclassification: ${verdict.meta.classification ?? "unknown"}; runner-level diagnostic: never-started ${neverStarted}; reporter errors ${reporterErrors}; runner vitest`
|
|
651
|
-
+ [...runnerErrors,
|
|
652
|
-
stdoutTail.toString(), stderrTail.toString()].filter(Boolean).map(text => `\n${text}`).join("")
|
|
653
|
-
: "";
|
|
654
|
-
// The interruption is reported beside the verdict, never folded into it or into raw wall service.
|
|
655
|
-
const interrupted = interruptions.length
|
|
656
|
-
? `\nhost interruptions: ${interruptions.map(i => `${i.kind} ${Math.round(i.wallMs)}ms wall / ${Math.round(i.monoMs)}ms monotonic, ${Math.round(i.subtractedMs)}ms subtracted`).join("; ")}`
|
|
657
|
-
: "";
|
|
658
|
-
return { evidenceReceipt, evidenceReceipts, pass: verdict.pass, kind: verdict.kind,
|
|
659
|
-
details: verdict.details + interrupted + diagnostics,
|
|
660
|
-
classification: verdict.meta.classification,
|
|
661
|
-
meta: { ...verdict.meta, nonce, manifest: files, manifestPath, listingCommand: invocation.listing, verification,
|
|
662
|
-
...(recovery ? { recovery, retryable: false } : {}), ...(interruptions.length ? { interruptions } : {}),
|
|
663
|
-
spawnedCommand, processExit: invoked.exitCode, pid: invoked.pid, stdoutPath, stderrPath },
|
|
664
|
-
exitCode: invoked.exitCode ?? -1, reportPath };
|
|
697
|
+
catch (error) {
|
|
698
|
+
const evidenceReceipt = error?.evidenceReceipt;
|
|
699
|
+
if (evidenceReceipt)
|
|
700
|
+
evidenceReceipts.push(...(error.evidenceReceipts ?? [evidenceReceipt]));
|
|
701
|
+
return { evidenceReceipt, evidenceReceipts, pass: false, kind: retainedFirstVerdict ? "work" : "infra", classification: retainedFirstVerdict ? "regression" : "infra", exitCode: -1, reportPath,
|
|
702
|
+
details: (retainedFirstVerdict ? `${retainedFirstVerdict.details}\n` : "") + (recovery ? `single fork retry ${recovery.retryNonce} after ${recovery.firstNonce}: ` : "") + (error instanceof Error ? error.message : String(error)),
|
|
703
|
+
meta: { ...(retainedFirstVerdict?.meta ?? { classification: "infra", infra: true }), ...(retainedFirstVerdict ? { firstPass: retainedFirstVerdict.meta } : {}), manifestDiscoveryFailed: true, spawnedCommand, verification,
|
|
704
|
+
...(recovery ? { recovery, retryable: false } : {}), ...(manifestPath ? { manifestPath } : {}) } };
|
|
705
|
+
}
|
|
706
|
+
});
|
|
665
707
|
}
|
|
666
708
|
catch (error) {
|
|
667
|
-
const
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
meta: { classification: "infra", infra: true, manifestDiscoveryFailed: true, spawnedCommand, verification,
|
|
673
|
-
...(recovery ? { recovery, retryable: false } : {}), ...(manifestPath ? { manifestPath } : {}) } };
|
|
709
|
+
const job = error;
|
|
710
|
+
return { pass: false, kind: "infra", classification: "infra", exitCode: -1, reportPath,
|
|
711
|
+
details: error instanceof Error ? error.message : String(error),
|
|
712
|
+
meta: { infra: true, classification: "infra", retryable: false,
|
|
713
|
+
verificationJobPath: job.verificationJobPath, verificationJob: job.verificationJob } };
|
|
674
714
|
}
|
|
675
715
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { readableExcerpt } from "../run/journal.js";
|
|
2
|
+
// OBS-1106 residual: a red whose assertion is a timeout or a wall-clock budget, and the runner-level
|
|
3
|
+
// diagnostics (never-started files, a worker RPC timeout) that make such a red infrastructure-SHAPED.
|
|
4
|
+
// Shape alone never parks anything: it only buys one isolated re-observation before a charge. A
|
|
5
|
+
// fresh-fingerprint section masks digits to `#`, so both spellings of a number are read.
|
|
6
|
+
// v2.6.6 C: the daemon's declarations (src/run/daemon.ts at 75dc3187:492-494), exported verbatim so the
|
|
7
|
+
// manifest gate's one remeasurement and the daemon's isolated adjudication read ONE predicate.
|
|
8
|
+
export const TIMEOUT_SHAPED_RE = /\b(?:Test|Hook) timed out in (?:\d+|#)\s*ms\b|\bexpected (?:\d+|#)(?:\.(?:\d+|#))? to be (?:less than|below)(?: or equal to)? (?:\d+|#)/i;
|
|
9
|
+
export const RUNNER_INFRA_DIAGNOSTIC_RE = /\bnever-started (?:[1-9]\d*|#)(?![\d#])|\[vitest-worker\]: Timeout calling\b/;
|
|
10
|
+
export const timeoutShaped = (details) => TIMEOUT_SHAPED_RE.test(readableExcerpt(details));
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { type ChildProcess, type SpawnOptions } from "node:child_process";
|
|
2
|
+
import { type SupervisionArm, type SupervisionTier } from "./supervision.js";
|
|
3
|
+
/** start's whole budget — launch, read-back and any overrun — before it must refuse and roll back. */
|
|
4
|
+
export declare const BEAT_STARTUP_CEILING_MS = 60000;
|
|
5
|
+
/** The generation a started child carries; it lets the child act under its launcher's claim only. */
|
|
6
|
+
export declare const BEAT_GENERATION_ENV = "TICKMARKR_BEAT_GENERATION";
|
|
7
|
+
export interface ProcessIdentity {
|
|
8
|
+
birth: string;
|
|
9
|
+
command: string;
|
|
10
|
+
cwd: string;
|
|
11
|
+
pgid: number;
|
|
12
|
+
}
|
|
13
|
+
export type IdentityRead = ProcessIdentity | "DEAD" | "UNKNOWN";
|
|
14
|
+
/** Every boundary the lifecycle crosses, injectable so failures are exercised where they really occur. */
|
|
15
|
+
export interface BeatLifecycleDeps {
|
|
16
|
+
/** Command prefix that re-executes this same CLI (node, its flags, the entry the launcher ran). */
|
|
17
|
+
cli: readonly string[];
|
|
18
|
+
spawn: (command: string, args: readonly string[], options: SpawnOptions) => ChildProcess;
|
|
19
|
+
kill: (pid: number, signal: NodeJS.Signals) => void;
|
|
20
|
+
identity: (pid: number) => IdentityRead;
|
|
21
|
+
openLog: (path: string) => number;
|
|
22
|
+
now: () => number;
|
|
23
|
+
sleep: (ms: number) => Promise<void>;
|
|
24
|
+
env: NodeJS.ProcessEnv;
|
|
25
|
+
}
|
|
26
|
+
export declare function defaultBeatDeps(): BeatLifecycleDeps;
|
|
27
|
+
/** A named, nonzero refusal: dispatch renders the message and exits 1. */
|
|
28
|
+
export declare class BeatRefusal extends Error {
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The live process behind a pid: birth (lstart), full argv, cwd and process group. A missing process is
|
|
32
|
+
* DEAD only when the kernel says so (ESRCH or a zombie); anything unreadable is UNKNOWN, never DEAD.
|
|
33
|
+
*/
|
|
34
|
+
export declare function readProcessIdentity(pid: number): IdentityRead;
|
|
35
|
+
export declare const beatClaimPath: (repoRoot: string, tier: SupervisionTier) => string;
|
|
36
|
+
export declare const beatOwnerPath: (repoRoot: string, tier: SupervisionTier) => string;
|
|
37
|
+
export declare const beatLogPath: (repoRoot: string, tier: SupervisionTier) => string;
|
|
38
|
+
/**
|
|
39
|
+
* The exact generation start launched. pid/birth/command/pgid are absent only while it is launching;
|
|
40
|
+
* priorArmId/priorArmEpoch name the (already fenced) arm on disk at launch, which the child's own arm then
|
|
41
|
+
* replaces; armEpoch is bound from the exact child's checked beat, and ownership is the COMPLETE durable arm
|
|
42
|
+
* generation (armId AND armEpoch) — a same-id arm with another epoch is a replacement.
|
|
43
|
+
*/
|
|
44
|
+
export interface BeatOwner {
|
|
45
|
+
tier: SupervisionTier;
|
|
46
|
+
seat: string;
|
|
47
|
+
generation: string;
|
|
48
|
+
armId: string;
|
|
49
|
+
cwd: string;
|
|
50
|
+
startedAt: string;
|
|
51
|
+
armEpoch?: string;
|
|
52
|
+
priorArmId?: string;
|
|
53
|
+
priorArmEpoch?: string;
|
|
54
|
+
pid?: number;
|
|
55
|
+
birth?: string;
|
|
56
|
+
command?: string;
|
|
57
|
+
pgid?: number;
|
|
58
|
+
}
|
|
59
|
+
type Read<T> = T | "NONE" | "UNREADABLE";
|
|
60
|
+
export declare function readBeatOwner(repoRoot: string, tier: SupervisionTier): Read<BeatOwner>;
|
|
61
|
+
/** Atomic first-writer-wins claim; a held claim is BUSY and is never taken over. */
|
|
62
|
+
export declare function acquireBeatClaim(repoRoot: string, tier: SupervisionTier, token: string): void;
|
|
63
|
+
export declare function releaseBeatClaim(repoRoot: string, tier: SupervisionTier, token: string): void;
|
|
64
|
+
/**
|
|
65
|
+
* A legacy writer: the launch generation it carries (if any), its seat, the arm a running loop holds, and the
|
|
66
|
+
* boundaries its checks read through.
|
|
67
|
+
*/
|
|
68
|
+
export interface LegacyWriter {
|
|
69
|
+
generation?: string;
|
|
70
|
+
seat: string;
|
|
71
|
+
settled?: boolean;
|
|
72
|
+
arm?: SupervisionArm;
|
|
73
|
+
deps: BeatLifecycleDeps;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* The recheck a legacy write calls immediately before it mutates; `arm` is the arm the writer itself holds, and
|
|
77
|
+
* `created` says this writer published that arm under this claim a moment ago (an owned re-arm).
|
|
78
|
+
*/
|
|
79
|
+
export type LegacyRecheck = (arm?: SupervisionArm, created?: boolean) => void;
|
|
80
|
+
/**
|
|
81
|
+
* Run one legacy write (one-shot tick, new arm, EVERY loop tick, stand-down) under the tier claim. A launched
|
|
82
|
+
* child acts under its launcher's claim while that claim is still held for its generation; afterwards it takes
|
|
83
|
+
* the tier claim itself like every other writer. `act` receives the recheck it must call immediately before it
|
|
84
|
+
* writes: the claim still names this holder and the tier's ownership still admits this writer. A claim held by
|
|
85
|
+
* anyone else refuses with nothing written — `waits` only lets a running loop's tick outlast a brief holder.
|
|
86
|
+
*/
|
|
87
|
+
export declare function withLegacyClaim<T>(repoRoot: string, tier: SupervisionTier, writer: LegacyWriter, act: (recheck: LegacyRecheck) => T | Promise<T>, waits?: {
|
|
88
|
+
tries: number;
|
|
89
|
+
sleep: (ms: number) => Promise<void>;
|
|
90
|
+
}): Promise<T>;
|
|
91
|
+
/** The ownership a legacy writer needs before each mutation: its own COMPLETE launched generation, or no owner at all. */
|
|
92
|
+
export declare function assertLegacyOwnership(repoRoot: string, tier: SupervisionTier, writer: LegacyWriter, arm?: SupervisionArm, created?: boolean, held?: BeatOwner): BeatOwner | undefined;
|
|
93
|
+
/**
|
|
94
|
+
* ONE RULE: every legacy write is preceded by the shared complete owned-generation check of the CURRENT owner
|
|
95
|
+
* record and durable arm against the EXACT generation this writer holds — token, seat, pid (once settled), armId
|
|
96
|
+
* and armEpoch. `held` is the owner this claim's previous check validated and `arm` the arm the writer holds (a
|
|
97
|
+
* loop's own arm; before the launcher binds the epoch that arm is the evidence). The first check of a claim by a
|
|
98
|
+
* writer holding neither takes the generation on record; from then on the held generation changes in exactly
|
|
99
|
+
* two verified ways, and any other change — also one between this check's own reads — refuses with no write:
|
|
100
|
+
* (i) the writer's OWN re-arm (`created`: it published `arm` under this claim after the complete check): the
|
|
101
|
+
* arm on disk must be exactly that arm and the owner exactly the owner checked before the arm write; only
|
|
102
|
+
* then is the owner rebound to the arm (same token, seat, process), and the complete check runs AGAIN as
|
|
103
|
+
* the new generation before the beat. After any mismatch the owner is never rewritten.
|
|
104
|
+
* (ii) the launcher completing an UNBOUND owner for this writer (launcherBound) — never adopted from disk.
|
|
105
|
+
* Once `settled` (past its first beat) the record must name its pid: a launch that refused without proving it
|
|
106
|
+
* could signal its child leaves no pid behind, and that child retires itself.
|
|
107
|
+
*/
|
|
108
|
+
export declare function assertOwnedGeneration(repoRoot: string, tier: SupervisionTier, writer: LegacyWriter & {
|
|
109
|
+
generation: string;
|
|
110
|
+
}, arm?: SupervisionArm | undefined, created?: boolean, held?: BeatOwner, pid?: number): BeatOwner;
|
|
111
|
+
export type CheckedBeat = {
|
|
112
|
+
kind: "EMPTY";
|
|
113
|
+
} | {
|
|
114
|
+
kind: "DISARMED";
|
|
115
|
+
seat?: string;
|
|
116
|
+
} | {
|
|
117
|
+
kind: "OWN-LIVE";
|
|
118
|
+
owner: BeatOwner & {
|
|
119
|
+
pid: number;
|
|
120
|
+
};
|
|
121
|
+
advancing: boolean;
|
|
122
|
+
} | {
|
|
123
|
+
kind: "OWN-DEAD";
|
|
124
|
+
owner: BeatOwner & {
|
|
125
|
+
pid: number;
|
|
126
|
+
};
|
|
127
|
+
} | {
|
|
128
|
+
kind: "FOREIGN";
|
|
129
|
+
reason: string;
|
|
130
|
+
} | {
|
|
131
|
+
kind: "PID-REUSED";
|
|
132
|
+
owner: BeatOwner & {
|
|
133
|
+
pid: number;
|
|
134
|
+
};
|
|
135
|
+
} | {
|
|
136
|
+
kind: "UNKNOWN";
|
|
137
|
+
reason: string;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* The checked state of one tier, read-only: no claim, no write, no signal. Every authoritative record —
|
|
141
|
+
* owner, marker, beat and durable arm — is validated before any outcome is selected, and ownership binds to
|
|
142
|
+
* this repository's canonical root, the recorded seat and the recorded arm before the process is consulted.
|
|
143
|
+
*/
|
|
144
|
+
export declare function inspectBeat(repoRoot: string, tier: SupervisionTier, seat: string | undefined, deps: BeatLifecycleDeps): CheckedBeat;
|
|
145
|
+
/** Status: the recorded liveness, rendered without writing; healthy states exit 0, the rest nonzero. */
|
|
146
|
+
export declare function beatStatus(repoRoot: string, tier: SupervisionTier, seat: string | undefined, deps: BeatLifecycleDeps): {
|
|
147
|
+
out: string;
|
|
148
|
+
code: number;
|
|
149
|
+
};
|
|
150
|
+
/** start: one claim, one child; exit zero only after this exact child's own advancing beat reads back. */
|
|
151
|
+
export declare function startBeat(repoRoot: string, tier: SupervisionTier, seat: string, deps: BeatLifecycleDeps): Promise<string>;
|
|
152
|
+
/** stop: retire only the recorded generation, signalling its exact pid, then read back DISARMED. */
|
|
153
|
+
export declare function stopBeat(repoRoot: string, tier: SupervisionTier, seat: string, deps: BeatLifecycleDeps): Promise<string>;
|
|
154
|
+
export {};
|