tickmarkr 2.5.7 → 2.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/qwen.js +30 -3
- package/dist/cli/commands/approve.js +23 -4
- package/dist/cli/commands/beat.d.ts +2 -0
- package/dist/cli/commands/beat.js +28 -29
- package/dist/cli/commands/compile.js +18 -0
- package/dist/cli/commands/fleet.js +61 -53
- package/dist/cli/commands/plan.d.ts +5 -0
- package/dist/cli/commands/plan.js +28 -23
- package/dist/cli/commands/resume.js +1 -1
- package/dist/cli/commands/run.js +1 -1
- package/dist/cli/commands/verify.d.ts +4 -1
- package/dist/cli/commands/verify.js +16 -5
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.js +6 -4
- package/dist/compile/native.js +68 -7
- package/dist/compile/retired-literals.d.ts +22 -0
- package/dist/compile/retired-literals.js +271 -0
- package/dist/config/config.d.ts +41 -3
- package/dist/config/config.js +48 -17
- package/dist/config/fleet-overlay.js +47 -62
- package/dist/drivers/index.d.ts +4 -2
- package/dist/drivers/index.js +54 -6
- package/dist/gates/baseline.d.ts +65 -0
- package/dist/gates/baseline.js +163 -9
- package/dist/gates/review.d.ts +6 -4
- package/dist/gates/review.js +62 -21
- package/dist/gates/run-gates.d.ts +6 -1
- package/dist/gates/run-gates.js +25 -13
- package/dist/gates/test-manifest.d.ts +20 -1
- package/dist/gates/test-manifest.js +50 -22
- package/dist/gates/test-reporter.js +22 -1
- package/dist/graph/schema.d.ts +28 -0
- package/dist/graph/schema.js +13 -1
- package/dist/run/daemon.d.ts +19 -0
- package/dist/run/daemon.js +363 -118
- package/dist/run/journal.d.ts +54 -3
- package/dist/run/journal.js +142 -11
- package/dist/run/merge.d.ts +15 -2
- package/dist/run/merge.js +74 -11
- package/dist/run/protocol.d.ts +82 -0
- package/dist/run/protocol.js +35 -0
- package/dist/run/receipt-resolver.d.ts +18 -0
- package/dist/run/receipt-resolver.js +132 -0
- package/dist/run/repair-disposition.d.ts +41 -0
- package/dist/run/repair-disposition.js +77 -0
- package/dist/run/supervision.d.ts +14 -1
- package/dist/run/supervision.js +122 -24
- package/dist/tui/cockpit/evidence-view.d.ts +10 -1
- package/dist/tui/cockpit/evidence-view.js +37 -5
- package/dist/tui/cockpit/home-view.js +45 -30
- package/dist/tui/cockpit/live-store.d.ts +18 -0
- package/dist/tui/ink/fleet-app.d.ts +12 -22
- package/dist/tui/ink/fleet-app.js +520 -131
- package/package.json +1 -1
- package/schema/rungraph.schema.json +54 -0
- package/skills/tickmarkr-overseer/SKILL.md +168 -36
- package/skills/tickmarkr-overseer/scripts/watch-context.sh +14 -2
package/dist/run/daemon.js
CHANGED
|
@@ -31,10 +31,11 @@ import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
|
31
31
|
import { runEnvironment } from "./environment.js";
|
|
32
32
|
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
33
33
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
34
|
-
import {
|
|
34
|
+
import { classifyRepairDisposition, resolveScopeHints } from "./repair-disposition.js";
|
|
35
|
+
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingApprovalActions, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
35
36
|
import { isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
36
37
|
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
37
|
-
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
38
|
+
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, reusedTipEvidence, verifyIntegrationTip } from "./merge.js";
|
|
38
39
|
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
39
40
|
import { desiredPanes } from "./reconcile.js";
|
|
40
41
|
import { readTierLiveness, readWatchBoard, supervisionBeatPath } from "./supervision.js";
|
|
@@ -163,6 +164,27 @@ export function resolveRunMode(repoRoot, opts = {}) {
|
|
|
163
164
|
: undefined;
|
|
164
165
|
return { cfg: resolved.cfg, mode: resolved.mode, source, ...(conflict ? { conflict } : {}) };
|
|
165
166
|
}
|
|
167
|
+
// Bind approval context before dispatch consumes the release. Restore reconstructs the same
|
|
168
|
+
// binding from journal order; a newer approval, including one without a reason, supersedes it.
|
|
169
|
+
function approvalReviewContext(events, taskId, restore = false) {
|
|
170
|
+
let pending;
|
|
171
|
+
let bound;
|
|
172
|
+
let approved = false;
|
|
173
|
+
for (const row of events) {
|
|
174
|
+
if (row.taskId !== taskId)
|
|
175
|
+
continue;
|
|
176
|
+
if (row.event === "task-approved") {
|
|
177
|
+
pending = typeof row.data.reason === "string" ? row.data.reason.trim() || undefined : undefined;
|
|
178
|
+
approved = true;
|
|
179
|
+
}
|
|
180
|
+
else if (row.event === "task-dispatch" || (row.event === "recheck-battery" && approved)) {
|
|
181
|
+
bound = pending;
|
|
182
|
+
pending = undefined;
|
|
183
|
+
approved = false;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return restore && !approved ? bound : pending;
|
|
187
|
+
}
|
|
166
188
|
// T14: the events that prove an approval was ENACTED — narrowly causal, never merely subsequent.
|
|
167
189
|
// Ordinary, attempt-cap and review-upheld approvals buy a WORKER, so their proof is a dispatch;
|
|
168
190
|
// recheck has its own battery enactment below. Generic terminal events are deliberately NOT proof: an approved human-gate
|
|
@@ -175,6 +197,17 @@ const RECHECK_ENACTMENT = "recheck-battery";
|
|
|
175
197
|
// persisted task branch after the approved gate (execTask's satisfiedGate branch), whose first act
|
|
176
198
|
// for the task is worktree-recreation. That event is causal for this path, not incidental.
|
|
177
199
|
const GATE_SATISFIED_ENACTMENT = "worktree-recreation";
|
|
200
|
+
// Older daemons enacted rechecks through worker-launch, before recheck-battery existed.
|
|
201
|
+
// Preserve that consumption at every scheduling read without changing continuing permission.
|
|
202
|
+
function pendingDaemonApprovalActions(events) {
|
|
203
|
+
const actions = pendingApprovalActions(events);
|
|
204
|
+
const rechecks = pendingRechecks(events);
|
|
205
|
+
for (const [id, action] of actions) {
|
|
206
|
+
if (action.authority === "battery" && !rechecks.has(id))
|
|
207
|
+
actions.delete(id);
|
|
208
|
+
}
|
|
209
|
+
return actions;
|
|
210
|
+
}
|
|
178
211
|
/**
|
|
179
212
|
* T14, amended by v2.2 T3: approvals the run accepted and never acted on. `approved` above is still
|
|
180
213
|
* built ONCE at startup — replay determinism depends on it — but a live approval is no longer inert:
|
|
@@ -357,6 +390,14 @@ function repairDiffFilesCut(diff, cap, names) {
|
|
|
357
390
|
});
|
|
358
391
|
return files;
|
|
359
392
|
}
|
|
393
|
+
/** Assertion evidence is context, never part of the normalized failure identity. */
|
|
394
|
+
function repairFindingsBrief(results) {
|
|
395
|
+
return results.filter(gateFailed).map((g) => {
|
|
396
|
+
const evidence = g.meta?.failureEvidence;
|
|
397
|
+
return `${g.gate}: ${g.details}` + (Array.isArray(evidence) && evidence.length
|
|
398
|
+
? `\nAssertion evidence:\n${JSON.stringify(evidence, null, 2)}` : "");
|
|
399
|
+
}).join("\n\n");
|
|
400
|
+
}
|
|
360
401
|
/** The fix-only contract: the findings verbatim, then the diff content of the work already landed. */
|
|
361
402
|
function repairBrief(findings, diff, baseRef) {
|
|
362
403
|
const repair = { findings, diff };
|
|
@@ -644,7 +685,7 @@ function lastVerifyCycle(events) {
|
|
|
644
685
|
if (e.event === "tip-verify-start") {
|
|
645
686
|
const { tip, cmdHash } = e.data;
|
|
646
687
|
cur = typeof tip === "string" && typeof cmdHash === "string"
|
|
647
|
-
? { tip, cmdHash, gates: new Set(), failed: false, forgiven: false, capacities: [e.data.capacity] }
|
|
688
|
+
? { tip, cmdHash, gates: new Set(), failed: false, forgiven: false, capacities: [e.data.capacity], evidenceByGate: new Map() }
|
|
648
689
|
: undefined;
|
|
649
690
|
afterRunEnd = false;
|
|
650
691
|
continue;
|
|
@@ -667,7 +708,7 @@ function lastVerifyCycle(events) {
|
|
|
667
708
|
continue;
|
|
668
709
|
}
|
|
669
710
|
if (!cur || afterRunEnd || cur.tip !== tip || cur.cmdHash !== cmdHash) {
|
|
670
|
-
cur = { tip, cmdHash, gates: new Set(), failed: false, forgiven: false, capacities: [] };
|
|
711
|
+
cur = { tip, cmdHash, gates: new Set(), failed: false, forgiven: false, capacities: [], evidenceByGate: new Map() };
|
|
671
712
|
}
|
|
672
713
|
afterRunEnd = false;
|
|
673
714
|
// T7: EVERY verdict row's own capacity, not just the start row's. The start row is a statement of
|
|
@@ -679,6 +720,7 @@ function lastVerifyCycle(events) {
|
|
|
679
720
|
cur.failed = true;
|
|
680
721
|
else {
|
|
681
722
|
cur.gates.add(gate);
|
|
723
|
+
cur.evidenceByGate.set(gate, e.data);
|
|
682
724
|
// Q121s review M1: a forgiven green depends on the baseline that forgave it. Baselines are not
|
|
683
725
|
// part of the cache key, so a forgiven cycle is NEVER cache-eligible — an unchanged tip whose
|
|
684
726
|
// baseline later vanishes or changes must re-run the full verify, not inherit the green.
|
|
@@ -688,6 +730,47 @@ function lastVerifyCycle(events) {
|
|
|
688
730
|
}
|
|
689
731
|
return cur;
|
|
690
732
|
}
|
|
733
|
+
/**
|
|
734
|
+
* OBS-1077 close rider: what the engagement's LATEST verification cycle proved — its
|
|
735
|
+
* `tip-verify-start` row and what followed, never a commit comparison. Exactly one kind per close:
|
|
736
|
+
* fresh (every tip command ran AND passed), reused (an eligible cached cycle carried forward),
|
|
737
|
+
* failed, or incomplete (cancelled, cut short, mixed or undelimited). Nothing before the start row
|
|
738
|
+
* is read, so an unfinished cycle inherits nothing from an earlier green one.
|
|
739
|
+
*/
|
|
740
|
+
export function runEndTipProof(events) {
|
|
741
|
+
const start = events.map((e) => e.event).lastIndexOf("tip-verify-start");
|
|
742
|
+
if (start < 0)
|
|
743
|
+
return { kind: "incomplete" };
|
|
744
|
+
const { tip: rawTip, gates: rawGates } = events[start].data;
|
|
745
|
+
const tip = typeof rawTip === "string" ? rawTip : undefined;
|
|
746
|
+
const proof = (kind) => ({ kind, ...(tip ? { tip } : {}) });
|
|
747
|
+
const after = events.slice(start + 1);
|
|
748
|
+
if (after.some((e) => e.event === "tip-verify-failed"))
|
|
749
|
+
return proof("failed");
|
|
750
|
+
if (after.some((e) => e.event === "tip-verify-cancelled"))
|
|
751
|
+
return proof("incomplete");
|
|
752
|
+
const rows = after.filter((e) => e.event === "tip-verify" && e.data.pass === true && e.data.tip === tip);
|
|
753
|
+
const gates = Array.isArray(rawGates) ? rawGates : [];
|
|
754
|
+
if (!tip || gates.length === 0 || !gates.every((g) => rows.some((r) => r.data.gate === g)))
|
|
755
|
+
return proof("incomplete");
|
|
756
|
+
const carried = rows.filter((r) => r.data.cached === true).length;
|
|
757
|
+
const cachedRow = after.some((e) => e.event === "tip-verify-cached");
|
|
758
|
+
// A whole-cycle carry must be declared by the start row, stated by the cache row and borne by every verdict row.
|
|
759
|
+
if ((events[start].data.cached === true) !== cachedRow || (cachedRow && carried !== rows.length))
|
|
760
|
+
return proof("incomplete");
|
|
761
|
+
// D-131: fresh only when EVERY command executed; one per-gate persisted verdict makes the cycle reused.
|
|
762
|
+
return carried > 0 ? proof("reused") : proof("fresh");
|
|
763
|
+
}
|
|
764
|
+
/** The close notification's statement of the proof — one clause per kind. */
|
|
765
|
+
export function formatTipProof(p) {
|
|
766
|
+
const at = p.tip ? ` ${p.tip.slice(0, 12)}` : "";
|
|
767
|
+
switch (p.kind) {
|
|
768
|
+
case "fresh": return `tip proof: fresh — every tip command ran and passed on${at || " the integration tip"}`;
|
|
769
|
+
case "reused": return `tip proof: reused — carried from verified commit${at}, commands not re-run`;
|
|
770
|
+
case "failed": return `tip proof: failed —${at ? ` commit${at}` : ""} latest verification cycle is red`;
|
|
771
|
+
case "incomplete": return `tip proof: incomplete —${at ? ` commit${at}` : ""} latest verification cycle did not finish`;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
691
774
|
// Isolate the battery so cancellation can stop its control flow as well as its shell children.
|
|
692
775
|
// The child uses the same verifier and capacity; only the daemon writes lifecycle verdict rows.
|
|
693
776
|
async function cancellableTipBattery(intWt, commands, runDir, baseline, signal) {
|
|
@@ -848,7 +931,7 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
848
931
|
// pass — `cached: true` keeps it honest about not having re-run the command.
|
|
849
932
|
for (const gate of gates) {
|
|
850
933
|
const cmd = gate === "test" && commands.tipTest ? commands.tipTest : commands[gate];
|
|
851
|
-
journal.append("tip-verify", undefined, { gate, cmd, pass: true, exitCode: 0, cached: true, tip, cmdHash, capacity });
|
|
934
|
+
journal.append("tip-verify", undefined, { ...reusedTipEvidence(last.evidenceByGate.get(gate) ?? {}), gate, cmd, pass: true, exitCode: 0, cached: true, tip, cmdHash, capacity });
|
|
852
935
|
}
|
|
853
936
|
return false;
|
|
854
937
|
}
|
|
@@ -857,12 +940,22 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
857
940
|
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal)
|
|
858
941
|
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline))) {
|
|
859
942
|
const measuredCapacity = r.capacity ?? capacity;
|
|
943
|
+
const evidence = r.reused ? reusedTipEvidence(r) : {
|
|
944
|
+
...(r.originRunRoot ? { originRunRoot: r.originRunRoot } : {}),
|
|
945
|
+
...(r.cause ? { cause: r.cause } : {}),
|
|
946
|
+
...(r.evidenceReceipt ? { evidenceReceipt: r.evidenceReceipt } : {}),
|
|
947
|
+
...(r.evidenceReceipts ? { evidenceReceipts: r.evidenceReceipts } : {}),
|
|
948
|
+
...(r.evidenceAbsence ? { evidenceAbsence: r.evidenceAbsence } : {}),
|
|
949
|
+
...Object.fromEntries(["nonce", "stdoutPath", "stderrPath"]
|
|
950
|
+
.filter(key => r[key] !== undefined).map(key => [key, r[key]])),
|
|
951
|
+
};
|
|
860
952
|
if (r.pass) {
|
|
861
953
|
// Q121s: a forgiven pass journals its fingerprints — honest about what was carried, never a silent green.
|
|
862
|
-
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
954
|
+
journal.append("tip-verify", undefined, { ...evidence, gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.reused ? { cached: true } : {}), ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
863
955
|
}
|
|
864
956
|
else {
|
|
865
957
|
journal.append("tip-verify-failed", undefined, {
|
|
958
|
+
...evidence,
|
|
866
959
|
gate: r.gate,
|
|
867
960
|
cmd: r.cmd,
|
|
868
961
|
exitCode: r.exitCode,
|
|
@@ -1411,11 +1504,23 @@ const terminateTornJournalTail = (journalPath) => {
|
|
|
1411
1504
|
// a persistently unwritable sink reports a crash naming the journal path and carrying NO terminal
|
|
1412
1505
|
// record, rather than fabricating an ended run on evidence the harness could not write. (OBS-313:
|
|
1413
1506
|
// with the sink dead the crash CAUSE is unrecordable — recorded as an observation, not papered over.)
|
|
1414
|
-
function recordFatalRunEnd(journal, runId, branch, err) {
|
|
1507
|
+
export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "setup") {
|
|
1415
1508
|
const original = err instanceof Error ? err.message : String(err);
|
|
1509
|
+
// A fatal close never finished a cycle: fresh/reused green is withheld, a recorded red still stands.
|
|
1510
|
+
let tipProof = { kind: "incomplete" };
|
|
1416
1511
|
try {
|
|
1417
|
-
|
|
1418
|
-
|
|
1512
|
+
// Only the latest engagement can already own this terminal outcome.
|
|
1513
|
+
const events = journal.read();
|
|
1514
|
+
let from = events.length;
|
|
1515
|
+
while (from > 0) {
|
|
1516
|
+
const event = events[--from];
|
|
1517
|
+
if (event.event === "run-start" || event.event === "run-resume")
|
|
1518
|
+
break;
|
|
1519
|
+
if (event.event === "run-end")
|
|
1520
|
+
return undefined;
|
|
1521
|
+
}
|
|
1522
|
+
const latest = runEndTipProof(events.slice(from));
|
|
1523
|
+
tipProof = { ...latest, kind: latest.kind === "failed" ? "failed" : "incomplete" };
|
|
1419
1524
|
}
|
|
1420
1525
|
catch (readErr) {
|
|
1421
1526
|
console.error(`tickmarkr ${runId}: journal read failed while recording the fatal run-end (${readErr instanceof Error ? readErr.message : String(readErr)}) — original error: ${original}`);
|
|
@@ -1423,18 +1528,19 @@ function recordFatalRunEnd(journal, runId, branch, err) {
|
|
|
1423
1528
|
const record = {
|
|
1424
1529
|
runId,
|
|
1425
1530
|
branch,
|
|
1426
|
-
done: [],
|
|
1427
|
-
failed: [],
|
|
1428
|
-
human: [],
|
|
1429
|
-
blocked: [],
|
|
1430
|
-
pending: [],
|
|
1431
|
-
phase
|
|
1531
|
+
done: graph?.tasks.filter((t) => t.status === "done").map((t) => t.id) ?? [],
|
|
1532
|
+
failed: graph?.tasks.filter((t) => t.status === "failed").map((t) => t.id) ?? [],
|
|
1533
|
+
human: graph?.tasks.filter((t) => t.status === "human").map((t) => t.id) ?? [],
|
|
1534
|
+
blocked: graph ? blockedTasks(graph).map((t) => t.id) : [],
|
|
1535
|
+
pending: graph ? pendingTasks(graph).map((t) => t.id) : [],
|
|
1536
|
+
phase,
|
|
1432
1537
|
fatal: true,
|
|
1433
1538
|
error: original,
|
|
1539
|
+
tipProof,
|
|
1434
1540
|
};
|
|
1435
1541
|
try {
|
|
1436
1542
|
journal.append("run-end", undefined, record);
|
|
1437
|
-
return;
|
|
1543
|
+
return tipProof;
|
|
1438
1544
|
}
|
|
1439
1545
|
catch {
|
|
1440
1546
|
// one retry below — the first failure is reported only if the retry also fails
|
|
@@ -1443,6 +1549,7 @@ function recordFatalRunEnd(journal, runId, branch, err) {
|
|
|
1443
1549
|
try {
|
|
1444
1550
|
terminateTornJournalTail(journalPath);
|
|
1445
1551
|
journal.append("run-end", undefined, record);
|
|
1552
|
+
return tipProof;
|
|
1446
1553
|
}
|
|
1447
1554
|
catch (retryErr) {
|
|
1448
1555
|
console.error(`tickmarkr ${runId}: run crashed — no terminal record written; journal sink unwritable at ${journalPath} (${retryErr instanceof Error ? retryErr.message : String(retryErr)}) — original error: ${original}`);
|
|
@@ -1512,12 +1619,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1512
1619
|
let journal;
|
|
1513
1620
|
let runStarted = false;
|
|
1514
1621
|
let taskLoopStarted = false;
|
|
1622
|
+
let graph;
|
|
1623
|
+
let fatalPhase = "setup";
|
|
1624
|
+
let deliberateTermination = false;
|
|
1625
|
+
const fatalStop = new AbortController();
|
|
1626
|
+
const inflight = new Map();
|
|
1627
|
+
let retireFatalSlots;
|
|
1515
1628
|
let branch = "";
|
|
1516
1629
|
let releaseApprovalSerialization;
|
|
1517
1630
|
let baselineCapture = Promise.resolve();
|
|
1518
1631
|
let baselineFailed = false;
|
|
1519
1632
|
try {
|
|
1520
|
-
|
|
1633
|
+
graph = loadGraph(repoRoot);
|
|
1521
1634
|
// One bounded snapshot supplies every dispatch. On resume the current journal still participates
|
|
1522
1635
|
// in the fold so its task-done/ordinary-approval events can retire older evidence, but findings it
|
|
1523
1636
|
// produced are suppressed: same-run feedback already replays those bytes and must not deliver them
|
|
@@ -1629,6 +1742,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1629
1742
|
await reapWorker(slot);
|
|
1630
1743
|
await driver.close(slot);
|
|
1631
1744
|
} }, s);
|
|
1745
|
+
retireFatalSlots = async () => {
|
|
1746
|
+
for (const slot of Array.from(liveSlots)) {
|
|
1747
|
+
try {
|
|
1748
|
+
await closeSlot(slot);
|
|
1749
|
+
}
|
|
1750
|
+
catch { /* preserve the original fatal error */ }
|
|
1751
|
+
}
|
|
1752
|
+
};
|
|
1632
1753
|
const budgetSlots = new Map();
|
|
1633
1754
|
const trackedDriver = {
|
|
1634
1755
|
id: driver.id,
|
|
@@ -1637,19 +1758,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1637
1758
|
...(driver.describe ? { describe: driver.describe.bind(driver) } : {}),
|
|
1638
1759
|
...(driver.focus ? { focus: driver.focus.bind(driver) } : {}),
|
|
1639
1760
|
slot: async (cwd, name, o) => {
|
|
1761
|
+
fatalStop.signal.throwIfAborted();
|
|
1640
1762
|
const signal = executionSignal();
|
|
1641
1763
|
signal?.throwIfAborted();
|
|
1642
1764
|
const s = await driver.slot(cwd, name, o);
|
|
1643
1765
|
liveSlots.add(s);
|
|
1644
1766
|
if (signal)
|
|
1645
1767
|
budgetSlots.get(signal)?.add(s);
|
|
1646
|
-
if (signal?.aborted) {
|
|
1768
|
+
if (signal?.aborted || fatalStop.signal.aborted) {
|
|
1647
1769
|
await withoutExecutionBudget(() => closeSlot(s));
|
|
1648
|
-
signal.throwIfAborted();
|
|
1770
|
+
fatalStop.signal.throwIfAborted();
|
|
1771
|
+
signal?.throwIfAborted();
|
|
1649
1772
|
}
|
|
1650
1773
|
return s;
|
|
1651
1774
|
},
|
|
1652
|
-
run: (s, cmd) =>
|
|
1775
|
+
run: (s, cmd) => {
|
|
1776
|
+
fatalStop.signal.throwIfAborted();
|
|
1777
|
+
return driver.run(s, cmd);
|
|
1778
|
+
},
|
|
1653
1779
|
waitOutput: (s, p, ms, o) => driver.waitOutput(s, p, ms, o),
|
|
1654
1780
|
waitAgentStatus: (s, st, ms) => driver.waitAgentStatus(s, st, ms),
|
|
1655
1781
|
status: (s) => driver.status(s),
|
|
@@ -1692,6 +1818,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1692
1818
|
const exit = opts.exit ?? ((code) => process.exit(code));
|
|
1693
1819
|
onTermination = (sig) => {
|
|
1694
1820
|
termSignal = sig;
|
|
1821
|
+
deliberateTermination = true;
|
|
1695
1822
|
void (async () => {
|
|
1696
1823
|
if (!reaping) {
|
|
1697
1824
|
reaping = true;
|
|
@@ -1794,7 +1921,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1794
1921
|
// flag. Startup approvals seed the first scheduling pass; live approvals are folded at task
|
|
1795
1922
|
// boundaries below so a sibling can release parked work without waiting for run-end + resume.
|
|
1796
1923
|
const approvalStartupEvents = journal.read();
|
|
1797
|
-
|
|
1924
|
+
// Continuing human-gate permission survives enactment; malformed releases grant none.
|
|
1925
|
+
const validApproval = (e) => e.event === "task-approved" && e.taskId
|
|
1926
|
+
&& pendingApprovalActions([e]).get(e.taskId)?.authority !== "inert";
|
|
1927
|
+
const approved = new Set(approvalStartupEvents.filter(validApproval).map((e) => e.taskId));
|
|
1928
|
+
const startupActions = pendingDaemonApprovalActions(approvalStartupEvents);
|
|
1798
1929
|
let approvalSweepCursor = approvalStartupEvents.length;
|
|
1799
1930
|
const commands = detectGateCommands(repoRoot, cfg);
|
|
1800
1931
|
const watchName = formatOwnedName({ role: "watch", taskId: "run", attempt: 0, runId });
|
|
@@ -1900,6 +2031,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1900
2031
|
const startedAt = Date.now();
|
|
1901
2032
|
for (;;) {
|
|
1902
2033
|
signal?.throwIfAborted();
|
|
2034
|
+
fatalStop.signal.throwIfAborted();
|
|
1903
2035
|
executionSignal()?.throwIfAborted();
|
|
1904
2036
|
const count = await liveSuiteCount(repoRoot);
|
|
1905
2037
|
if (count === 0)
|
|
@@ -1941,11 +2073,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1941
2073
|
// Empty Map on fresh runs — every seed below conditions on resume.get(t.id), never on opts.resume (the
|
|
1942
2074
|
// GATE-08 lesson at the humanGate guard: condition on the data, not the code path). Dead-code
|
|
1943
2075
|
// equivalence to the router.ts:194 profile⇒undefined pattern: no map entry ⇒ today's literal.
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
2076
|
+
const resume = opts.resume ? journal.replayResumeState() : new Map();
|
|
2077
|
+
const satisfiedGates = opts.resume ? journal.replaySatisfiedGates() : new Map();
|
|
2078
|
+
const replayedGateResults = resumeLifecycleOpen
|
|
1947
2079
|
? journal.replayCurrentAttemptGateResults()
|
|
1948
2080
|
: new Map();
|
|
2081
|
+
for (const [taskId, action] of startupActions) {
|
|
2082
|
+
if (action.authority === "worker")
|
|
2083
|
+
replayedGateResults.delete(taskId);
|
|
2084
|
+
}
|
|
1949
2085
|
// T7: the capacity THIS session resolved — read inside the run's fork budget, so it is the number
|
|
1950
2086
|
// every shell this run spawns will divide the machine by. Recorded evidence from another session
|
|
1951
2087
|
// is only reusable against this.
|
|
@@ -1998,7 +2134,29 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1998
2134
|
}
|
|
1999
2135
|
baseline = JSON.parse(readFileSync(join(journal.dir, "baseline.json"), "utf8"));
|
|
2000
2136
|
const replayEvents = journal.read();
|
|
2001
|
-
|
|
2137
|
+
// journal.replayStatuses predates typed releases and re-pends even inert rows. Keep that
|
|
2138
|
+
// legacy reader intact; scheduling accepts only the fold's recognised authorities.
|
|
2139
|
+
const statuses = new Map();
|
|
2140
|
+
for (const e of replayEvents) {
|
|
2141
|
+
if (!e.taskId)
|
|
2142
|
+
continue;
|
|
2143
|
+
if (e.event === "task-dispatch")
|
|
2144
|
+
statuses.set(e.taskId, "pending");
|
|
2145
|
+
else if (e.event === "task-done")
|
|
2146
|
+
statuses.set(e.taskId, "done");
|
|
2147
|
+
else if (e.event === "task-failed")
|
|
2148
|
+
statuses.set(e.taskId, "failed");
|
|
2149
|
+
else if (e.event === "task-human")
|
|
2150
|
+
statuses.set(e.taskId, "human");
|
|
2151
|
+
else if (validApproval(e))
|
|
2152
|
+
statuses.set(e.taskId, "pending");
|
|
2153
|
+
}
|
|
2154
|
+
for (const [id, action] of startupActions) {
|
|
2155
|
+
// Legacy gate-only approvals can finish and merge without a dispatch consuming funding.
|
|
2156
|
+
if (action.authority !== "inert" && statuses.get(id) !== "done")
|
|
2157
|
+
statuses.set(id, "pending");
|
|
2158
|
+
}
|
|
2159
|
+
for (const [id, st] of statuses) {
|
|
2002
2160
|
if (opts.retryFailed && st === "failed" && recordedTaskFailureKind(replayEvents, id) === "dispatch") {
|
|
2003
2161
|
graph = setStatus(graph, id, "pending");
|
|
2004
2162
|
// OBS-254: clear ATTEMPT AND CHANNEL STATE ONLY. Deleting the whole entry also deleted
|
|
@@ -2021,6 +2179,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2021
2179
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
2022
2180
|
...(opts.graphChanged ? { graphChanged: true } : {}), // OBS-1073: the release is the engagement's
|
|
2023
2181
|
});
|
|
2182
|
+
runStarted = true;
|
|
2024
2183
|
await placeBoard();
|
|
2025
2184
|
}
|
|
2026
2185
|
else {
|
|
@@ -2142,6 +2301,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2142
2301
|
let mergeChain = Promise.resolve();
|
|
2143
2302
|
const mergeSerial = (taskBranch, t, gated) => {
|
|
2144
2303
|
const next = mergeChain.then(() => {
|
|
2304
|
+
fatalStop.signal.throwIfAborted();
|
|
2145
2305
|
executionSignal()?.throwIfAborted();
|
|
2146
2306
|
journal.phaseStart(t.id, "merge");
|
|
2147
2307
|
return mergeTask(intWt, taskBranch, `tickmarkr: merge ${t.id} ${t.title}`, gated);
|
|
@@ -2173,15 +2333,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2173
2333
|
const tree = await shGit("git ls-files -z", wt);
|
|
2174
2334
|
return new Set([...(tree.code === 0 ? tree.stdout.split("\0").filter(Boolean) : []), ...await diffPaths(base, wt)]);
|
|
2175
2335
|
};
|
|
2336
|
+
// The base each task was dispatched on: the integration branch moves when a sibling merges, the pin does not.
|
|
2337
|
+
const taskBases = new Map();
|
|
2176
2338
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
2177
2339
|
// OBS-979: a worker refusal can identify the missing authoring scope even when gates
|
|
2178
2340
|
// subsequently supply the park's disposition. Keep that actionable path on the park itself.
|
|
2179
2341
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
2180
2342
|
if (!details.approveCommand && t.files.length > 0 && worker?.data.ok === false && typeof worker.data.summary === "string") {
|
|
2181
2343
|
const allowed = filesGlob(t.files);
|
|
2182
|
-
|
|
2344
|
+
// OBS-1072 residual: the same inventory as the main path — the task tree and its diff from the
|
|
2345
|
+
// task's PINNED base, never the main checkout and never the moving integration branch.
|
|
2346
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
2347
|
+
const inventory = existsSync(wt) ? await treeOrDiffPaths(taskBases.get(t.id) ?? "HEAD", wt) : new Set();
|
|
2348
|
+
const paths = resolveScopeHints(namedPaths(worker.data.summary).filter((path) => !allowed(path)), inventory)
|
|
2349
|
+
.resolved.filter((path) => !allowed(path));
|
|
2183
2350
|
if (paths.length)
|
|
2184
|
-
reason += ` — files[] repair hint: ${
|
|
2351
|
+
reason += ` — files[] repair hint: ${paths.join(", ")}`;
|
|
2185
2352
|
}
|
|
2186
2353
|
graph = setStatus(graph, t.id, "human");
|
|
2187
2354
|
saveGraph(repoRoot, graph);
|
|
@@ -2212,42 +2379,34 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2212
2379
|
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode, changed = new Set()) => {
|
|
2213
2380
|
const scopeRed = results.find((g) => g.gate === "scope" && gateFailed(g));
|
|
2214
2381
|
const verdict = scopeRed?.meta?.collateral;
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
const
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
journal.append("scope-request", t.id, classification);
|
|
2242
|
-
else
|
|
2243
|
-
journal.append("scope-authoring", t.id, classification);
|
|
2244
|
-
// OBS-547 (Leg-2 T3 material): unchargeable ⇒ metered 0, exactly as the predicted path below —
|
|
2245
|
-
// passing the physical count would write `meteredAttempts: 1` beside `attempts: 0`.
|
|
2246
|
-
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${[...new Set(refusal.length ? refusal : paths)].map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2247
|
-
await park(t, `${scopeRequest ? "scope request" : "authoring defect"} — files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}; current files[]: ${t.files.join(", ")}${scopeRequest ? `\n${approveCommand}` : ""}`, scopeRequest ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, scopeRequest ? { paths: [...new Set(refusal.length ? refusal : paths)], approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
2248
|
-
return true;
|
|
2249
|
-
}
|
|
2382
|
+
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
2383
|
+
const disposition = classifyRepairDisposition({
|
|
2384
|
+
results, files: t.files, inventory: changed,
|
|
2385
|
+
refusalSummary: worker?.data.ok === false && typeof worker.data.summary === "string" ? worker.data.summary : undefined,
|
|
2386
|
+
});
|
|
2387
|
+
for (const diagnostic of disposition.diagnostics) {
|
|
2388
|
+
journal.append("scope-hint-unresolved", t.id, {
|
|
2389
|
+
paths: [diagnostic.candidate], reason: diagnostic.reason, diagnostic, attempt: attempts + 1,
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
if (disposition.blocker === "infra")
|
|
2393
|
+
return false;
|
|
2394
|
+
if (!disposition.blocker && (disposition.kind === "scope-request" || disposition.kind === "authoring")) {
|
|
2395
|
+
const paths = disposition.paths;
|
|
2396
|
+
const scopeRequest = disposition.kind === "scope-request";
|
|
2397
|
+
const classification = {
|
|
2398
|
+
gate: disposition.source, paths, repair: `files[] repair hint: ${paths.join(", ")}`,
|
|
2399
|
+
attempt: attempts + 1, chargeable: false, source: "diagnostic",
|
|
2400
|
+
};
|
|
2401
|
+
if (scopeRequest)
|
|
2402
|
+
journal.append("scope-request", t.id, classification);
|
|
2403
|
+
else
|
|
2404
|
+
journal.append("scope-authoring", t.id, classification);
|
|
2405
|
+
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${paths.map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2406
|
+
await park(t, `${scopeRequest ? "scope request" : "authoring defect"} — files[] repair hint: ${paths.join(", ")}; current files[]: ${t.files.join(", ")}${scopeRequest ? `\n${approveCommand}` : ""}`, scopeRequest ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, scopeRequest ? { paths, approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
2407
|
+
return true;
|
|
2250
2408
|
}
|
|
2409
|
+
// A detection site permits the ordinary chargeable policy; it does not itself grant budget.
|
|
2251
2410
|
if (!verdict)
|
|
2252
2411
|
return false;
|
|
2253
2412
|
if (verdict.authoring) {
|
|
@@ -2284,17 +2443,33 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2284
2443
|
approvalSweepCursor = events.length;
|
|
2285
2444
|
if (approvals.length === 0)
|
|
2286
2445
|
return;
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2446
|
+
// Replay every authority from this pass's single snapshot. In particular, helper reads must
|
|
2447
|
+
// neither rescan the file under load nor observe approvals beyond approvalSweepCursor.
|
|
2448
|
+
// Keep audit writes on the live journal so narration and persistence retain their owner.
|
|
2449
|
+
const sweepJournal = Journal.open(repoRoot, runId);
|
|
2450
|
+
sweepJournal.read = () => events;
|
|
2451
|
+
sweepJournal.append = journal.append.bind(journal);
|
|
2452
|
+
graph = applyScopeAmendments(graph, sweepJournal, false, opts.graphChanged === true); // OBS-1073: same release as launch
|
|
2453
|
+
const actions = pendingDaemonApprovalActions(events);
|
|
2454
|
+
const nextResume = sweepJournal.replayResumeState();
|
|
2455
|
+
const nextSatisfied = sweepJournal.replaySatisfiedGates();
|
|
2293
2456
|
let changed = false;
|
|
2294
|
-
for (const
|
|
2295
|
-
const
|
|
2457
|
+
for (const taskId of new Set(approvals.map((e) => e.taskId))) {
|
|
2458
|
+
const action = actions.get(taskId);
|
|
2459
|
+
if (!action || action.authority === "inert")
|
|
2460
|
+
continue;
|
|
2296
2461
|
approved.add(taskId);
|
|
2297
|
-
|
|
2462
|
+
const state = nextResume.get(taskId);
|
|
2463
|
+
if (state)
|
|
2464
|
+
resume.set(taskId, state);
|
|
2465
|
+
else
|
|
2466
|
+
resume.delete(taskId);
|
|
2467
|
+
const gate = nextSatisfied.get(taskId);
|
|
2468
|
+
if (gate)
|
|
2469
|
+
satisfiedGates.set(taskId, gate);
|
|
2470
|
+
else
|
|
2471
|
+
satisfiedGates.delete(taskId);
|
|
2472
|
+
if (action.authority !== "waiver")
|
|
2298
2473
|
replayedGateResults.delete(taskId);
|
|
2299
2474
|
try {
|
|
2300
2475
|
const task = getTask(graph, taskId);
|
|
@@ -2311,6 +2486,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2311
2486
|
saveGraph(repoRoot, graph);
|
|
2312
2487
|
};
|
|
2313
2488
|
const execTask = async (t) => {
|
|
2489
|
+
const approvalAction = pendingDaemonApprovalActions(journal.read()).get(t.id);
|
|
2490
|
+
const fundedRerun = approvalAction?.authority === "worker";
|
|
2314
2491
|
const startMs = Date.now();
|
|
2315
2492
|
let stallSeat;
|
|
2316
2493
|
let stallReaps = 0;
|
|
@@ -2405,6 +2582,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2405
2582
|
// run; a park-instead policy can come later if it ever bites.
|
|
2406
2583
|
}
|
|
2407
2584
|
const taskHistory = journal.read().filter((e) => e.taskId === t.id);
|
|
2585
|
+
// Lifetime identity counts every dispatch, including legacy and unchargeable rows. Releases
|
|
2586
|
+
// reset the budget below, never this counter; observational annotations do not consume it.
|
|
2587
|
+
let nextWorkerDispatchOrdinal = taskHistory.filter((e) => e.event === "task-dispatch").length;
|
|
2408
2588
|
const lastApproval = taskHistory.map((e) => e.event).lastIndexOf("task-approved");
|
|
2409
2589
|
const priorClimb = taskHistory.slice(lastApproval + 1).reverse().find((e) => e.event === "tier-escalated");
|
|
2410
2590
|
if (!hintsChanged && priorClimb && taskHistory.indexOf(priorClimb) > taskHistory.map((e) => e.event).lastIndexOf("task-dispatch")) {
|
|
@@ -2421,7 +2601,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2421
2601
|
// exclusion-list-equality oracle. Daemon-side append only — no journal.ts write-path change (Phase 48
|
|
2422
2602
|
// stays unblocked); inert to replayStatuses (unknown events ignored, pinned at journal.test.ts:70-80).
|
|
2423
2603
|
if (rs)
|
|
2424
|
-
journal.append("resume-restore", t.id, {
|
|
2604
|
+
journal.append("resume-restore", t.id, {
|
|
2605
|
+
attempts: rs.attempts, tried: [...tried], assignment,
|
|
2606
|
+
workerDispatchOrdinal: previousDispatch?.data.workerDispatchOrdinal ?? null,
|
|
2607
|
+
});
|
|
2425
2608
|
// Keep one live list: recovery retries must see exclusions added by onGate during the round.
|
|
2426
2609
|
const badReviewers = [...replayedReviewerExclusions];
|
|
2427
2610
|
const noteReviewEvent = (e) => {
|
|
@@ -2540,7 +2723,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2540
2723
|
...(typeof g.meta.preservedRef === "string" ? { preservedRef: g.meta.preservedRef } : {}),
|
|
2541
2724
|
} : {}),
|
|
2542
2725
|
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
2543
|
-
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision"]
|
|
2726
|
+
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision", "failureEvidence"]
|
|
2544
2727
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
2545
2728
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
2546
2729
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
@@ -2588,6 +2771,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2588
2771
|
// every recalibration this telemetry funds, a gap is honest and a zero is a lie. The
|
|
2589
2772
|
// seven-gate closed set is asserted end-to-end in tests/run/gate-telemetry.test.ts.
|
|
2590
2773
|
...gateMeasurement(g.meta),
|
|
2774
|
+
// Fresh producer evidence is copied verbatim; absent/historical evidence is never minted here.
|
|
2775
|
+
...(g.meta?.reused ? {} : {
|
|
2776
|
+
...(g.evidenceReceipt ? { evidenceReceipt: g.evidenceReceipt } : {}),
|
|
2777
|
+
...(g.evidenceReceipts ? { evidenceReceipts: g.evidenceReceipts } : {}),
|
|
2778
|
+
...Object.fromEntries(["nonce", "stdoutPath", "stderrPath", "classification"]
|
|
2779
|
+
.filter(key => g.meta?.[key] !== undefined).map(key => [key, g.meta[key]])),
|
|
2780
|
+
}),
|
|
2591
2781
|
// T7: the capacity the gate's own command child ran under, lifted verbatim from the result
|
|
2592
2782
|
// the battery produced — read where the shell built that child's environment, never
|
|
2593
2783
|
// re-derived from the run's own budget, which would answer a different number than the
|
|
@@ -2816,10 +3006,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2816
3006
|
// An operator approval skips its exact failed gate by authority; observed results skip only the
|
|
2817
3007
|
// contiguous green prefix whose recorded commit is still the task branch tip.
|
|
2818
3008
|
const satisfiedGate = satisfiedGates.get(t.id);
|
|
2819
|
-
const replayedGates = replayedGateResults.get(t.id);
|
|
2820
|
-
const recheck =
|
|
3009
|
+
const replayedGates = fundedRerun ? undefined : replayedGateResults.get(t.id);
|
|
3010
|
+
const recheck = approvalAction?.authority === "battery";
|
|
2821
3011
|
resumeGateReplay: if (satisfiedGate || replayedGates || recheck) {
|
|
2822
3012
|
const taskBase = await integrationHead(intWt);
|
|
3013
|
+
taskBases.set(t.id, taskBase);
|
|
2823
3014
|
const taskBranch = `${branch}--${t.id}`;
|
|
2824
3015
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
2825
3016
|
// OBS-1022: the ref `approve --recheck` validated (commits ahead of the base) is the tree the
|
|
@@ -2920,6 +3111,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2920
3111
|
continue;
|
|
2921
3112
|
priorResults.set(e.data.gate, e);
|
|
2922
3113
|
}
|
|
3114
|
+
const declaredGates = GATE_NAMES.filter((gate) => t.gates.includes(gate));
|
|
3115
|
+
const reused = [];
|
|
2923
3116
|
let remainingGates;
|
|
2924
3117
|
if (recheck) {
|
|
2925
3118
|
remainingGates = GATE_NAMES.filter((gate) => t.gates.includes(gate));
|
|
@@ -2973,49 +3166,42 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2973
3166
|
});
|
|
2974
3167
|
}
|
|
2975
3168
|
let reusable = sameWorld && sameProtocol && (exactCurrentCommit || canonicalCurrentCommit || recreatedLegacyCommit);
|
|
2976
|
-
const reused = [];
|
|
2977
|
-
const declaredGates = GATE_NAMES.filter((gate) => t.gates.includes(gate));
|
|
2978
3169
|
for (const gate of declaredGates) {
|
|
2979
3170
|
if (!reusable || replayedGates.results.get(gate) !== true)
|
|
2980
3171
|
reusable = false;
|
|
2981
3172
|
else
|
|
2982
3173
|
reused.push(gate);
|
|
2983
3174
|
}
|
|
2984
|
-
// OBS-1049: build outputs are not commits. A replayed build verdict says the tree WAS green;
|
|
2985
|
-
// it says nothing about whether the RECREATED checkout holds what that build produced (dist),
|
|
2986
|
-
// and the host's npm lifecycle (ignore-scripts) may never rebuild it before the suite runs.
|
|
2987
|
-
// So a replayed build re-runs its own command here as provisioning, journaled by name; a red
|
|
2988
|
-
// provisioning is not reused — build (and everything behind it) re-enters the battery as a gate.
|
|
2989
|
-
// Every resume on this path recreates the checkout (recreateTaskWorktree above), so "reused
|
|
2990
|
-
// build" is exactly "build replayed onto a recreated tree".
|
|
2991
|
-
// Journal order is the criterion's order: a GREEN provisioning is recorded AFTER the reuse
|
|
2992
|
-
// rows it belongs to (gate-reused build, then gate-provisioned build); a RED provisioning is
|
|
2993
|
-
// recorded alone, and no reuse row follows it.
|
|
2994
|
-
let provisionedRow;
|
|
2995
|
-
if (reused.includes("build") && commands.build !== undefined) {
|
|
2996
|
-
// Provisioning is a verification command like any gate: it waits for the run's baseline and
|
|
2997
|
-
// takes the command lease (suite admission), never a bare shell beside sibling gates.
|
|
2998
|
-
await waitForBaseline(t.id);
|
|
2999
|
-
const startedAt = Date.now();
|
|
3000
|
-
const provisioned = await withCommandContext(t.id, () => sh(commands.build, wt));
|
|
3001
|
-
provisionedRow = {
|
|
3002
|
-
gate: "build", commit: replayedGates.commit, exitCode: provisioned.code, durationMs: Date.now() - startedAt,
|
|
3003
|
-
};
|
|
3004
|
-
if (provisioned.code !== 0) {
|
|
3005
|
-
reused.splice(reused.indexOf("build"));
|
|
3006
|
-
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
3007
|
-
provisionedRow = undefined;
|
|
3008
|
-
}
|
|
3009
|
-
}
|
|
3010
|
-
for (const gate of reused) {
|
|
3011
|
-
journal.append("gate-reused", t.id, { gate, commit: replayedGates.commit });
|
|
3012
|
-
}
|
|
3013
|
-
if (provisionedRow !== undefined)
|
|
3014
|
-
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
3015
3175
|
remainingGates = declaredGates.slice(reused.length);
|
|
3016
3176
|
}
|
|
3177
|
+
// OBS-1094/1049: every replay recreates the checkout, but build outputs are not commits.
|
|
3178
|
+
// Provision whenever build will not run as a gate, including an operator waiver restore.
|
|
3179
|
+
// Defer reuse rows until provisioning succeeds: green keeps reuse-then-provision order;
|
|
3180
|
+
// red records only provisioning and puts build and every declared successor back in gates,
|
|
3181
|
+
// unless build itself was waived: provisioning must not undo the operator's release.
|
|
3182
|
+
let provisionedRow;
|
|
3183
|
+
if (!remainingGates.includes("build") && commands.build !== undefined) {
|
|
3184
|
+
await waitForBaseline(t.id);
|
|
3185
|
+
const startedAt = Date.now();
|
|
3186
|
+
const provisioned = await withCommandContext(t.id, () => sh(commands.build, wt));
|
|
3187
|
+
provisionedRow = {
|
|
3188
|
+
gate: "build", commit: !satisfiedGate && !recheck ? replayedGates.commit : currentTaskSubject,
|
|
3189
|
+
exitCode: provisioned.code, durationMs: Date.now() - startedAt,
|
|
3190
|
+
};
|
|
3191
|
+
if (provisioned.code !== 0 && satisfiedGate !== "build") {
|
|
3192
|
+
reused.length = 0;
|
|
3193
|
+
remainingGates = ["build", ...declaredGates.filter((gate) => gate !== "build")];
|
|
3194
|
+
}
|
|
3195
|
+
}
|
|
3196
|
+
for (const gate of reused) {
|
|
3197
|
+
journal.append("gate-reused", t.id, { gate, commit: replayedGates.commit });
|
|
3198
|
+
}
|
|
3199
|
+
if (provisionedRow !== undefined)
|
|
3200
|
+
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
3017
3201
|
const resumedTask = { ...t, gates: remainingGates };
|
|
3202
|
+
const operatorContext = approvalReviewContext(journal.read(), t.id, true);
|
|
3018
3203
|
gateLoop: while (true) {
|
|
3204
|
+
fatalStop.signal.throwIfAborted();
|
|
3019
3205
|
executionSignal()?.throwIfAborted();
|
|
3020
3206
|
const gated = await gitHead(wt);
|
|
3021
3207
|
gateSubject = {
|
|
@@ -3031,6 +3217,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3031
3217
|
journal.phaseStart(t.id, "gates");
|
|
3032
3218
|
const { results } = await withCommandContext(t.id, () => runReviewRecovery(resumedTask, {
|
|
3033
3219
|
carriedFindings: outstandingReviewFindings(journal.read(), t.id),
|
|
3220
|
+
operatorContext,
|
|
3034
3221
|
worktree: wt, baseRef: taskBase, result: priorResult, author: gateAuthor,
|
|
3035
3222
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
3036
3223
|
collateral: collateral.get(t.id) ?? [],
|
|
@@ -3073,7 +3260,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3073
3260
|
},
|
|
3074
3261
|
}, false));
|
|
3075
3262
|
results.forEach(classifySignalOnlyTest);
|
|
3076
|
-
if (
|
|
3263
|
+
if (pendingDaemonApprovalActions(journal.read()).get(t.id)?.authority === "battery") {
|
|
3077
3264
|
journal.append("recheck-battery", t.id, {
|
|
3078
3265
|
commit: gateSubject.commit,
|
|
3079
3266
|
gates: resumedTask.gates,
|
|
@@ -3125,6 +3312,26 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3125
3312
|
}
|
|
3126
3313
|
assignment = { adapter: seat.adapter, model: seat.model, channel: seat.channel, tier: seat.tier };
|
|
3127
3314
|
}
|
|
3315
|
+
// Carry completeness was verified before this replay battery. Apply the ordinary
|
|
3316
|
+
// repair bounds here too; a restored red must fund the same findings-bearing dispatch.
|
|
3317
|
+
// Funding is durable intent only: repairReachSinceApproval charges it after launch
|
|
3318
|
+
// reaches a funded gate, and a restart before launch must not duplicate that intent.
|
|
3319
|
+
const failing = results.filter(gateFailed);
|
|
3320
|
+
const battery = failing.some((g) => g.gate !== "review") ? failing.filter((g) => g.gate !== "review") : failing;
|
|
3321
|
+
const landed = await commitsAheadOf(taskBase, wt);
|
|
3322
|
+
const events = journal.read();
|
|
3323
|
+
const drawn = repairsSinceApproval(events, t.id);
|
|
3324
|
+
const repeated = failing.some((g) => isDeterministicFailure(g)
|
|
3325
|
+
&& identicalGateFailures(events, t.id, g.gate, normalizeGateFailure(g.details)) >= GATE_FINGERPRINT_CAP);
|
|
3326
|
+
if (narrowRepairBattery(battery) && landed.length > 0 && drawn < MAX_REPAIRS
|
|
3327
|
+
&& (rs?.attempts ?? 0) < MAX_ATTEMPTS && !repeated
|
|
3328
|
+
&& pendingRepairFindings(events, t.id) === undefined) {
|
|
3329
|
+
journal.append("repair-attempt", t.id, {
|
|
3330
|
+
repair: repairReachSinceApproval(events, t.id).length + 1, charge: drawn + 1, of: MAX_REPAIRS,
|
|
3331
|
+
gates: failing.map((g) => g.gate), commits: landed.length,
|
|
3332
|
+
findings: withCarriedEvidence(repairFindingsBrief(results)),
|
|
3333
|
+
});
|
|
3334
|
+
}
|
|
3128
3335
|
break gateLoop;
|
|
3129
3336
|
}
|
|
3130
3337
|
await park(t, gateFailApprovalReason(t.id, "post-approval gate failed", results.some((g) => g.gate === "review" && gateFailed(g))), "gate-fail", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
@@ -3170,6 +3377,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3170
3377
|
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
3171
3378
|
let supersededWorkerSlot;
|
|
3172
3379
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
3380
|
+
// Funding belongs to the next attempt. Later automatic repairs retain ordinary red reuse.
|
|
3381
|
+
const explicitlyFundedAttempt = fundedRerun && attempt === (rs?.attempts ?? 0);
|
|
3382
|
+
fatalStop.signal.throwIfAborted();
|
|
3173
3383
|
executionSignal()?.throwIfAborted();
|
|
3174
3384
|
if (attempt !== providerDeathAttempt) {
|
|
3175
3385
|
providerDeathRequeues = 0;
|
|
@@ -3255,6 +3465,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3255
3465
|
// fresh re-dispatches. Read from the JOURNAL, before this dispatch's own event lands: a run that
|
|
3256
3466
|
// stopped between funding the repair and sending it resumes still carrying the findings.
|
|
3257
3467
|
const journaledSoFar = journal.read(); // read BEFORE this dispatch's own event lands
|
|
3468
|
+
const operatorContext = approvalReviewContext(journaledSoFar, t.id);
|
|
3258
3469
|
let repairFindings = pendingRepairFindings(journaledSoFar, t.id);
|
|
3259
3470
|
// OBS-254, one layer below the upheld brief: the ordinary gate-fail brief was loop-local, so any
|
|
3260
3471
|
// path that rebuilt this task's state (a resume, `--retry-failed`, a fresh daemon) dispatched a
|
|
@@ -3344,9 +3555,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3344
3555
|
// T6: a dispatch that carries an outstanding finding says so, and names it. Without this the
|
|
3345
3556
|
// ledger cannot tell a carried dispatch from an amnesiac one — the exact question a run that
|
|
3346
3557
|
// spends two frontier attempts re-deriving a known defect has to be able to answer afterwards.
|
|
3558
|
+
fatalStop.signal.throwIfAborted();
|
|
3559
|
+
const workerDispatchOrdinal = nextWorkerDispatchOrdinal++;
|
|
3347
3560
|
journal.append("task-dispatch", t.id, {
|
|
3348
3561
|
...(scopeApproval?.data.release === "scope-request" ? { files: t.files, graphDefinitionHash: graphDefinitionHash(graph) } : {}),
|
|
3349
|
-
assignment, attempt, provenance: dispatchProvenance([
|
|
3562
|
+
assignment, attempt, workerDispatchOrdinal, provenance: dispatchProvenance([
|
|
3350
3563
|
channelKey(assignment) === channelKey(r.assignment) ? r.provenance
|
|
3351
3564
|
: t.routingHints?.pin ? `pin ${t.routingHints.pin.via}:${t.routingHints.pin.model} not re-tried` : "ladder assignment",
|
|
3352
3565
|
`dispatch ${channelKey(assignment)}`, climbProvenance,
|
|
@@ -3362,6 +3575,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3362
3575
|
});
|
|
3363
3576
|
journal.phaseStart(t.id, "worker", { attempt, assignment });
|
|
3364
3577
|
const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
|
|
3578
|
+
taskBases.set(t.id, taskBase);
|
|
3365
3579
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
3366
3580
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
3367
3581
|
const recreating = existsSync(priorWt);
|
|
@@ -3452,7 +3666,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3452
3666
|
// anything the live brief holds beyond the journaled findings (a consult's guidance) is kept:
|
|
3453
3667
|
// a repair adds the diff and the fix-only contract, it never subtracts what was already known.
|
|
3454
3668
|
feedback = feedback && !brief.includes(feedback) ? `${brief}\n\n${feedback}` : brief;
|
|
3455
|
-
journal.append("repair-dispatch", t.id, { diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
3669
|
+
journal.append("repair-dispatch", t.id, { workerDispatchOrdinal, diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
3456
3670
|
}
|
|
3457
3671
|
if (feedback || priorNamed.length > 0) {
|
|
3458
3672
|
feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
|
|
@@ -3869,6 +4083,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3869
4083
|
while (Date.now() < hardDeadline) {
|
|
3870
4084
|
if (termSignal)
|
|
3871
4085
|
throw new Error(`terminated by ${termSignal}`);
|
|
4086
|
+
fatalStop.signal.throwIfAborted();
|
|
3872
4087
|
executionSignal()?.throwIfAborted();
|
|
3873
4088
|
driverProbeFailed = false;
|
|
3874
4089
|
const sliceStart = Date.now();
|
|
@@ -4452,6 +4667,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4452
4667
|
// there is the trailer regex; the exit marker only sets exitCode), and the cause taxonomy
|
|
4453
4668
|
// already names this shape "clean-exit-no-trailer" — unreachable in print mode until now.
|
|
4454
4669
|
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
4670
|
+
fatalStop.signal.throwIfAborted();
|
|
4455
4671
|
executionSignal()?.throwIfAborted();
|
|
4456
4672
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
4457
4673
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
@@ -4881,6 +5097,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4881
5097
|
let results = [];
|
|
4882
5098
|
let commits = [];
|
|
4883
5099
|
gateLoop: while (true) {
|
|
5100
|
+
fatalStop.signal.throwIfAborted();
|
|
4884
5101
|
executionSignal()?.throwIfAborted();
|
|
4885
5102
|
const gated = await gitHead(wt);
|
|
4886
5103
|
gateSubject = { commit: await gateCommitSubject(taskBase, gated, wt), attempt };
|
|
@@ -4895,7 +5112,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4895
5112
|
&& e.data.attempt === attempt - 1)
|
|
4896
5113
|
: [];
|
|
4897
5114
|
journal.phaseStart(t.id, "gates");
|
|
4898
|
-
const replay = previousRows.length > 0
|
|
5115
|
+
const replay = !explicitlyFundedAttempt && previousRows.length > 0
|
|
4899
5116
|
&& previousRows.every((e) => e.data.commit === gateSubject.commit)
|
|
4900
5117
|
&& previousRows.some((e) => e.data.pass === false && e.data.skipped !== true);
|
|
4901
5118
|
if (replay) {
|
|
@@ -4918,10 +5135,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4918
5135
|
else {
|
|
4919
5136
|
({ results, commits } = await withCommandContext(t.id, () => runReviewRecovery(t, {
|
|
4920
5137
|
carriedFindings: outstandingFindings,
|
|
5138
|
+
operatorContext,
|
|
4921
5139
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
4922
5140
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4923
5141
|
collateral: collateral.get(t.id) ?? [],
|
|
4924
5142
|
selectTests: !testGateFailed,
|
|
5143
|
+
cachedRedBypass: explicitlyFundedAttempt ? "operator-rerun" : undefined,
|
|
4925
5144
|
via: cfg.visibility.llm === "pane"
|
|
4926
5145
|
? {
|
|
4927
5146
|
driver: trackedDriver,
|
|
@@ -4960,6 +5179,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4960
5179
|
// gate-fail brief is, so prior-RUN evidence (retired by its own rule, not by this reviewer)
|
|
4961
5180
|
// survives and only this run's settled findings go.
|
|
4962
5181
|
feedback = withCarriedEvidence("");
|
|
5182
|
+
fatalStop.signal.throwIfAborted();
|
|
4963
5183
|
executionSignal()?.throwIfAborted();
|
|
4964
5184
|
const m = await mergeSerial(taskBranch, t, gated);
|
|
4965
5185
|
if (m.tipMoved) {
|
|
@@ -5029,7 +5249,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5029
5249
|
// v1.53 T3: prefer the CLI's own session id captured from this attempt's output (kimi's resume
|
|
5030
5250
|
// trailer) over the harness slot name; absent hook or no capture keeps today's slot-name id.
|
|
5031
5251
|
retrySession = { channel: channelKey(assignment), id: adapter.sessionIdFrom?.(output) ?? sessionId, contextTokens };
|
|
5032
|
-
feedback = withCarriedEvidence(results
|
|
5252
|
+
feedback = withCarriedEvidence(repairFindingsBrief(results));
|
|
5033
5253
|
// OBS-189/G3 (park-economics patch): a request-changes review is a findings brief, not a worker
|
|
5034
5254
|
// defect — the fix attempt stays on the same channel with the findings as feedback and consumes
|
|
5035
5255
|
// no escalation-ladder rung. Bounded by the engagement round cap at the top of this loop.
|
|
@@ -5057,7 +5277,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5057
5277
|
const repairable = narrowRepairBattery(repairBattery) && lostCommits.length === 0 && landed.length > 0;
|
|
5058
5278
|
const repairHistory = repairReachSinceApproval(journal.read(), t.id);
|
|
5059
5279
|
const repairsDrawn = repairsSinceApproval(journal.read(), t.id);
|
|
5060
|
-
const repair = repairable && repairsDrawn < MAX_REPAIRS;
|
|
5280
|
+
const repair = repairable && repairsDrawn < MAX_REPAIRS && attempt + 1 < MAX_ATTEMPTS;
|
|
5061
5281
|
// v1.85 T3: the fingerprint cap. Two normalized-identical failures of one DETERMINISTIC gate on
|
|
5062
5282
|
// one task (volatile tokens — worktree prefixes, line refs, durations, run ids — are not
|
|
5063
5283
|
// information) mean the round about to be bought is a re-measurement: ~663m across 5 runs went to
|
|
@@ -5217,7 +5437,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5217
5437
|
});
|
|
5218
5438
|
};
|
|
5219
5439
|
taskLoopStarted = true;
|
|
5220
|
-
|
|
5440
|
+
fatalPhase = "scheduler";
|
|
5221
5441
|
// A settled worker can release a dependency or an approval in the same poll tick.
|
|
5222
5442
|
// Audit the current graph at every empty-flight boundary, never a prior ready snapshot.
|
|
5223
5443
|
// WB-1 rider: `end-condition-held` names a CLOSE the daemon refused, so only a close attempt journals
|
|
@@ -5236,6 +5456,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5236
5456
|
return true;
|
|
5237
5457
|
};
|
|
5238
5458
|
closeLoop: while (true) {
|
|
5459
|
+
fatalPhase = "scheduler";
|
|
5239
5460
|
let approvalDeadline;
|
|
5240
5461
|
while (true) {
|
|
5241
5462
|
// v1.54 T2: a signal that landed while nothing was racing `aborted` (empty inflight window)
|
|
@@ -5262,6 +5483,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5262
5483
|
cleanupErrors.push({ slot: slot.name, attempt: owner.attempt, error: String(error) });
|
|
5263
5484
|
}
|
|
5264
5485
|
}
|
|
5486
|
+
if (fatalStop.signal.aborted)
|
|
5487
|
+
return;
|
|
5265
5488
|
const cleanupEvidence = cleanupErrors.length ? { cleanupErrors } : {};
|
|
5266
5489
|
if (err instanceof HeldProbeExhausted) {
|
|
5267
5490
|
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
@@ -5364,7 +5587,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5364
5587
|
blocked: blockedTasks(graph).map((t) => t.id),
|
|
5365
5588
|
pending: pendingTasks(graph).map((t) => t.id),
|
|
5366
5589
|
};
|
|
5590
|
+
fatalPhase = "baseline";
|
|
5367
5591
|
await baselineCapture; // also drain capture when every worker parks or fails before its gates
|
|
5592
|
+
fatalPhase = "tip-verify";
|
|
5368
5593
|
// OBS-34: post-merge integration-tip verify — strict exit codes, no baseline forgiveness.
|
|
5369
5594
|
const lastMergedTask = [...journal.read()].reverse().find((e) => e.event === "merge" && e.taskId)?.taskId;
|
|
5370
5595
|
if (summary.done.length > 0 && Object.keys(commands).length > 0) {
|
|
@@ -5407,9 +5632,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5407
5632
|
clearTimeout(poll);
|
|
5408
5633
|
}
|
|
5409
5634
|
summary.tipVerify = tipFailed ? "failed" : "passed";
|
|
5635
|
+
summary.tipProof = runEndTipProof(journal.read());
|
|
5410
5636
|
if (tipFailed && lastMergedTask)
|
|
5411
5637
|
summary.lastMergedTask = lastMergedTask;
|
|
5412
5638
|
}
|
|
5639
|
+
else {
|
|
5640
|
+
// Exactly one proof per close: no cycle ran for THIS close, and an earlier one is not inherited.
|
|
5641
|
+
summary.tipProof = { kind: "incomplete" };
|
|
5642
|
+
}
|
|
5413
5643
|
// T14: read the journal, not the startup `approved` set — an approval appended DURING this run is
|
|
5414
5644
|
// exactly the one the set cannot see, and it is the one the record has to name.
|
|
5415
5645
|
//
|
|
@@ -5417,6 +5647,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5417
5647
|
// graph.lock release in the outer finally: an approval that wins first is included below, while an
|
|
5418
5648
|
// approval that loses cannot append until the live owner is gone and reports recorded-no-owner.
|
|
5419
5649
|
// Thus no accepted approval can land after this sample while still being attributed to this run.
|
|
5650
|
+
fatalPhase = "scheduler";
|
|
5420
5651
|
const approvalSerialization = await acquireApprovalSerialization(repoRoot, runId);
|
|
5421
5652
|
releaseApprovalSerialization = approvalSerialization.release;
|
|
5422
5653
|
// Close the last poll-to-run-end race while holding the same serializer as approve.
|
|
@@ -5448,7 +5679,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5448
5679
|
const tipFail = summary.tipVerify === "failed"
|
|
5449
5680
|
? ` — TIP VERIFY FAILED on ${summary.lastMergedTask ? `last merge ${summary.lastMergedTask}` : "integration tip"}`
|
|
5450
5681
|
: "";
|
|
5451
|
-
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
|
|
5682
|
+
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — ${formatTipProof(summary.tipProof)} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
|
|
5452
5683
|
return summary;
|
|
5453
5684
|
}
|
|
5454
5685
|
});
|
|
@@ -5458,8 +5689,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5458
5689
|
// T7 (v1.86): guarded — a journal read/append failure while recording the fatal run-end is
|
|
5459
5690
|
// reported alongside err, never instead of it; recordFatalRunEnd never throws, so the original
|
|
5460
5691
|
// error (message, stack, cause) always reaches the caller verbatim.
|
|
5461
|
-
|
|
5462
|
-
|
|
5692
|
+
// Freeze failure-time buckets, stop new dispatch, and drain all journal writers before closing.
|
|
5693
|
+
if (!deliberateTermination) {
|
|
5694
|
+
const failedGraph = graph;
|
|
5695
|
+
const phase = taskLoopStarted ? (baselineFailed ? "baseline" : fatalPhase) : "setup";
|
|
5696
|
+
fatalStop.abort(err);
|
|
5697
|
+
await retireFatalSlots?.();
|
|
5698
|
+
await Promise.allSettled(inflight.values());
|
|
5699
|
+
await baselineCapture.catch(() => { });
|
|
5700
|
+
if (runStarted && !deliberateTermination) {
|
|
5701
|
+
const tipProof = recordFatalRunEnd(journal, runId, branch, err, taskLoopStarted || opts.resume ? failedGraph : undefined, phase);
|
|
5702
|
+
// The fatal close states its proof too; a dead notifier never replaces the original error.
|
|
5703
|
+
if (tipProof) {
|
|
5704
|
+
await driver.notify(`tickmarkr ${runId}: run crashed — ${formatTipProof(tipProof)} — integration branch ${branch}`, { tier: "attention" }).catch(() => { });
|
|
5705
|
+
}
|
|
5706
|
+
}
|
|
5707
|
+
}
|
|
5463
5708
|
throw err;
|
|
5464
5709
|
}
|
|
5465
5710
|
finally {
|