tickmarkr 2.5.6 → 2.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/prompt.js +21 -1
- package/dist/cli/commands/approve.js +13 -6
- package/dist/cli/commands/compile.js +7 -0
- package/dist/cli/commands/plan.d.ts +5 -0
- package/dist/cli/commands/plan.js +28 -23
- package/dist/cli/commands/resume.js +1 -1
- package/dist/cli/commands/run.js +1 -1
- package/dist/cli/commands/status.js +95 -34
- package/dist/cli/commands/verify.js +6 -4
- package/dist/compile/native.js +68 -7
- package/dist/compile/retired-literals.d.ts +22 -0
- package/dist/compile/retired-literals.js +271 -0
- package/dist/drivers/index.d.ts +4 -2
- package/dist/drivers/index.js +54 -6
- package/dist/gates/baseline.d.ts +8 -3
- package/dist/gates/baseline.js +6 -3
- package/dist/gates/review.js +12 -1
- package/dist/gates/run-gates.d.ts +6 -1
- package/dist/gates/run-gates.js +67 -9
- package/dist/gates/test-manifest.d.ts +12 -0
- package/dist/gates/test-manifest.js +29 -3
- package/dist/gates/test-reporter.js +28 -2
- package/dist/graph/graph.d.ts +2 -0
- package/dist/graph/graph.js +5 -0
- package/dist/graph/schema.d.ts +28 -0
- package/dist/graph/schema.js +13 -1
- package/dist/run/activity.d.ts +28 -0
- package/dist/run/activity.js +194 -0
- package/dist/run/daemon.d.ts +28 -0
- package/dist/run/daemon.js +466 -110
- package/dist/run/git.d.ts +6 -1
- package/dist/run/git.js +63 -11
- package/dist/run/journal.d.ts +57 -5
- package/dist/run/journal.js +103 -9
- package/dist/run/merge.d.ts +2 -0
- package/dist/run/merge.js +1 -0
- package/dist/run/operator-page-summary.d.ts +56 -0
- package/dist/run/operator-page-summary.js +68 -0
- package/dist/run/operator-summary.d.ts +69 -0
- package/dist/run/operator-summary.js +77 -0
- package/dist/run/protocol.d.ts +71 -0
- package/dist/run/protocol.js +32 -0
- package/dist/run/repair-disposition.d.ts +41 -0
- package/dist/run/repair-disposition.js +77 -0
- package/dist/tui/cockpit/board.d.ts +9 -0
- package/dist/tui/cockpit/board.js +10 -0
- package/dist/tui/cockpit/derive.d.ts +35 -0
- package/dist/tui/cockpit/derive.js +152 -10
- package/dist/tui/cockpit/evidence-view.d.ts +2 -0
- package/dist/tui/cockpit/evidence-view.js +42 -12
- package/dist/tui/cockpit/home-view.js +45 -30
- package/dist/tui/cockpit/live-store.d.ts +18 -0
- package/dist/tui/cockpit/run-cockpit.d.ts +8 -1
- package/dist/tui/cockpit/run-cockpit.js +95 -1
- package/dist/tui/cockpit/run-view.d.ts +37 -0
- package/dist/tui/cockpit/run-view.js +189 -2
- package/package.json +1 -1
- package/schema/rungraph.schema.json +54 -0
- package/skills/tickmarkr-overseer/SKILL.md +162 -3
package/dist/run/daemon.js
CHANGED
|
@@ -31,7 +31,8 @@ import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
|
31
31
|
import { runEnvironment } from "./environment.js";
|
|
32
32
|
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
33
33
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
34
|
-
import {
|
|
34
|
+
import { classifyRepairDisposition, resolveScopeHints } from "./repair-disposition.js";
|
|
35
|
+
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingApprovalActions, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
35
36
|
import { isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
36
37
|
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
37
38
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
@@ -51,6 +52,79 @@ export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
|
51
52
|
throw error;
|
|
52
53
|
}
|
|
53
54
|
}
|
|
55
|
+
// A transport timeout says nothing about whether a mutation reached the terminal.
|
|
56
|
+
class HeldProbeExhausted extends Error {
|
|
57
|
+
}
|
|
58
|
+
function isTransportTimeout(error) {
|
|
59
|
+
if (!(error instanceof Error))
|
|
60
|
+
return false;
|
|
61
|
+
const e = error;
|
|
62
|
+
return e.timedOut === true || ["ETIMEDOUT", "transport_timeout", "timeout"].includes(e.code ?? "")
|
|
63
|
+
|| /\b(?:timed? out|timeout)\b/i.test(e.reason ?? e.message);
|
|
64
|
+
}
|
|
65
|
+
export function heldWorkerTransport(driver, slot, dispatchId, held, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
|
|
66
|
+
const delays = [250, 500, 1_000];
|
|
67
|
+
const probe = async (operation, call) => {
|
|
68
|
+
for (let retry = 0;; retry++) {
|
|
69
|
+
try {
|
|
70
|
+
return await call();
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
if (!isTransportTimeout(error))
|
|
74
|
+
throw error;
|
|
75
|
+
held({ operation, retry, state: "held", error: String(error), backoffMs: delays[retry] });
|
|
76
|
+
if (retry === delays.length)
|
|
77
|
+
throw new HeldProbeExhausted(`${operation} transport retry budget exhausted`);
|
|
78
|
+
await sleep(delays[retry]);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
return {
|
|
83
|
+
read: (s, n) => probe("read", () => driver.read(s, n)),
|
|
84
|
+
status: (s) => probe("status", () => driver.status(s)),
|
|
85
|
+
waitOutput: (s, p, ms, o) => probe("waitOutput", () => driver.waitOutput(s, p, ms, o)),
|
|
86
|
+
waitAgentStatus: (s, st, ms) => probe("waitAgentStatus", () => driver.waitAgentStatus(s, st, ms)),
|
|
87
|
+
run: async (s, command) => {
|
|
88
|
+
const observer = driver;
|
|
89
|
+
// A second delivery needs a receipt for THIS mutation, not an empty or idle terminal.
|
|
90
|
+
for (let delivery = 0; delivery < 2; delivery++) {
|
|
91
|
+
try {
|
|
92
|
+
await driver.run(s, command);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
if (!isTransportTimeout(error))
|
|
97
|
+
throw error;
|
|
98
|
+
let notAccepted = false;
|
|
99
|
+
for (let retry = 0; retry <= delays.length; retry++) {
|
|
100
|
+
held({ operation: "run", retry, delivery, state: "held", dispatchId, error: String(error), backoffMs: delays[retry] });
|
|
101
|
+
try {
|
|
102
|
+
const receipt = await observer.observeDispatch?.(slot, command, dispatchId);
|
|
103
|
+
if (receipt?.authoritative === true && receipt.slotId === slot.id
|
|
104
|
+
&& receipt.command === command && receipt.dispatchId === dispatchId) {
|
|
105
|
+
if (receipt.outcome === "accepted")
|
|
106
|
+
return;
|
|
107
|
+
if (receipt.outcome === "not-accepted") {
|
|
108
|
+
notAccepted = true;
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const text = await driver.read(slot, PANE_READ_ROWS);
|
|
113
|
+
if (text.split(/\r?\n/).some((line) => line === `TICKMARKR_DISPATCH_${dispatchId}`))
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
catch { /* failed observations prove neither acceptance nor nonacceptance */ }
|
|
117
|
+
if (retry < delays.length)
|
|
118
|
+
await sleep(delays[retry]);
|
|
119
|
+
}
|
|
120
|
+
if (!notAccepted)
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
throw new HeldProbeExhausted("dispatch outcome remained uncertain; no delivery authorized");
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
|
54
128
|
// A fixed attempt ceiling is independent of the rolling inactivity clock.
|
|
55
129
|
let attemptHardTimeoutMs;
|
|
56
130
|
export function setAttemptHardTimeoutMsForTests(ms) { attemptHardTimeoutMs = ms; }
|
|
@@ -102,6 +176,17 @@ const RECHECK_ENACTMENT = "recheck-battery";
|
|
|
102
176
|
// persisted task branch after the approved gate (execTask's satisfiedGate branch), whose first act
|
|
103
177
|
// for the task is worktree-recreation. That event is causal for this path, not incidental.
|
|
104
178
|
const GATE_SATISFIED_ENACTMENT = "worktree-recreation";
|
|
179
|
+
// Older daemons enacted rechecks through worker-launch, before recheck-battery existed.
|
|
180
|
+
// Preserve that consumption at every scheduling read without changing continuing permission.
|
|
181
|
+
function pendingDaemonApprovalActions(events) {
|
|
182
|
+
const actions = pendingApprovalActions(events);
|
|
183
|
+
const rechecks = pendingRechecks(events);
|
|
184
|
+
for (const [id, action] of actions) {
|
|
185
|
+
if (action.authority === "battery" && !rechecks.has(id))
|
|
186
|
+
actions.delete(id);
|
|
187
|
+
}
|
|
188
|
+
return actions;
|
|
189
|
+
}
|
|
105
190
|
/**
|
|
106
191
|
* T14, amended by v2.2 T3: approvals the run accepted and never acted on. `approved` above is still
|
|
107
192
|
* built ONCE at startup — replay determinism depends on it — but a live approval is no longer inert:
|
|
@@ -284,6 +369,14 @@ function repairDiffFilesCut(diff, cap, names) {
|
|
|
284
369
|
});
|
|
285
370
|
return files;
|
|
286
371
|
}
|
|
372
|
+
/** Assertion evidence is context, never part of the normalized failure identity. */
|
|
373
|
+
function repairFindingsBrief(results) {
|
|
374
|
+
return results.filter(gateFailed).map((g) => {
|
|
375
|
+
const evidence = g.meta?.failureEvidence;
|
|
376
|
+
return `${g.gate}: ${g.details}` + (Array.isArray(evidence) && evidence.length
|
|
377
|
+
? `\nAssertion evidence:\n${JSON.stringify(evidence, null, 2)}` : "");
|
|
378
|
+
}).join("\n\n");
|
|
379
|
+
}
|
|
287
380
|
/** The fix-only contract: the findings verbatim, then the diff content of the work already landed. */
|
|
288
381
|
function repairBrief(findings, diff, baseRef) {
|
|
289
382
|
const repair = { findings, diff };
|
|
@@ -474,7 +567,9 @@ async function captureWorkerStream(journalDir, taskId, attempt, driver, slot, fa
|
|
|
474
567
|
try {
|
|
475
568
|
captured = await driver.read(slot, PANE_READ_ROWS);
|
|
476
569
|
}
|
|
477
|
-
catch {
|
|
570
|
+
catch (error) {
|
|
571
|
+
if (error instanceof HeldProbeExhausted)
|
|
572
|
+
throw error;
|
|
478
573
|
// Earlier successful reads are still a captured stream. Persist them rather than losing the
|
|
479
574
|
// only failure evidence because the pane vanished between its terminal read and this snapshot.
|
|
480
575
|
}
|
|
@@ -613,6 +708,47 @@ function lastVerifyCycle(events) {
|
|
|
613
708
|
}
|
|
614
709
|
return cur;
|
|
615
710
|
}
|
|
711
|
+
/**
|
|
712
|
+
* OBS-1077 close rider: what the engagement's LATEST verification cycle proved — its
|
|
713
|
+
* `tip-verify-start` row and what followed, never a commit comparison. Exactly one kind per close:
|
|
714
|
+
* fresh (every tip command ran AND passed), reused (an eligible cached cycle carried forward),
|
|
715
|
+
* failed, or incomplete (cancelled, cut short, mixed or undelimited). Nothing before the start row
|
|
716
|
+
* is read, so an unfinished cycle inherits nothing from an earlier green one.
|
|
717
|
+
*/
|
|
718
|
+
export function runEndTipProof(events) {
|
|
719
|
+
const start = events.map((e) => e.event).lastIndexOf("tip-verify-start");
|
|
720
|
+
if (start < 0)
|
|
721
|
+
return { kind: "incomplete" };
|
|
722
|
+
const { tip: rawTip, gates: rawGates } = events[start].data;
|
|
723
|
+
const tip = typeof rawTip === "string" ? rawTip : undefined;
|
|
724
|
+
const proof = (kind) => ({ kind, ...(tip ? { tip } : {}) });
|
|
725
|
+
const after = events.slice(start + 1);
|
|
726
|
+
if (after.some((e) => e.event === "tip-verify-failed"))
|
|
727
|
+
return proof("failed");
|
|
728
|
+
if (after.some((e) => e.event === "tip-verify-cancelled"))
|
|
729
|
+
return proof("incomplete");
|
|
730
|
+
const rows = after.filter((e) => e.event === "tip-verify" && e.data.pass === true && e.data.tip === tip);
|
|
731
|
+
const gates = Array.isArray(rawGates) ? rawGates : [];
|
|
732
|
+
if (!tip || gates.length === 0 || !gates.every((g) => rows.some((r) => r.data.gate === g)))
|
|
733
|
+
return proof("incomplete");
|
|
734
|
+
const carried = rows.filter((r) => r.data.cached === true).length;
|
|
735
|
+
const cachedRow = after.some((e) => e.event === "tip-verify-cached");
|
|
736
|
+
// A whole-cycle carry must be declared by the start row, stated by the cache row and borne by every verdict row.
|
|
737
|
+
if ((events[start].data.cached === true) !== cachedRow || (cachedRow && carried !== rows.length))
|
|
738
|
+
return proof("incomplete");
|
|
739
|
+
// D-131: fresh only when EVERY command executed; one per-gate persisted verdict makes the cycle reused.
|
|
740
|
+
return carried > 0 ? proof("reused") : proof("fresh");
|
|
741
|
+
}
|
|
742
|
+
/** The close notification's statement of the proof — one clause per kind. */
|
|
743
|
+
export function formatTipProof(p) {
|
|
744
|
+
const at = p.tip ? ` ${p.tip.slice(0, 12)}` : "";
|
|
745
|
+
switch (p.kind) {
|
|
746
|
+
case "fresh": return `tip proof: fresh — every tip command ran and passed on${at || " the integration tip"}`;
|
|
747
|
+
case "reused": return `tip proof: reused — carried from verified commit${at}, commands not re-run`;
|
|
748
|
+
case "failed": return `tip proof: failed —${at ? ` commit${at}` : ""} latest verification cycle is red`;
|
|
749
|
+
case "incomplete": return `tip proof: incomplete —${at ? ` commit${at}` : ""} latest verification cycle did not finish`;
|
|
750
|
+
}
|
|
751
|
+
}
|
|
616
752
|
// Isolate the battery so cancellation can stop its control flow as well as its shell children.
|
|
617
753
|
// The child uses the same verifier and capacity; only the daemon writes lifecycle verdict rows.
|
|
618
754
|
async function cancellableTipBattery(intWt, commands, runDir, baseline, signal) {
|
|
@@ -784,7 +920,7 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
784
920
|
const measuredCapacity = r.capacity ?? capacity;
|
|
785
921
|
if (r.pass) {
|
|
786
922
|
// Q121s: a forgiven pass journals its fingerprints — honest about what was carried, never a silent green.
|
|
787
|
-
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
923
|
+
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.reused ? { cached: true } : {}), ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
788
924
|
}
|
|
789
925
|
else {
|
|
790
926
|
journal.append("tip-verify-failed", undefined, {
|
|
@@ -1336,11 +1472,23 @@ const terminateTornJournalTail = (journalPath) => {
|
|
|
1336
1472
|
// a persistently unwritable sink reports a crash naming the journal path and carrying NO terminal
|
|
1337
1473
|
// record, rather than fabricating an ended run on evidence the harness could not write. (OBS-313:
|
|
1338
1474
|
// with the sink dead the crash CAUSE is unrecordable — recorded as an observation, not papered over.)
|
|
1339
|
-
function recordFatalRunEnd(journal, runId, branch, err) {
|
|
1475
|
+
export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "setup") {
|
|
1340
1476
|
const original = err instanceof Error ? err.message : String(err);
|
|
1477
|
+
// A fatal close never finished a cycle: fresh/reused green is withheld, a recorded red still stands.
|
|
1478
|
+
let tipProof = { kind: "incomplete" };
|
|
1341
1479
|
try {
|
|
1342
|
-
|
|
1343
|
-
|
|
1480
|
+
// Only the latest engagement can already own this terminal outcome.
|
|
1481
|
+
const events = journal.read();
|
|
1482
|
+
let from = events.length;
|
|
1483
|
+
while (from > 0) {
|
|
1484
|
+
const event = events[--from];
|
|
1485
|
+
if (event.event === "run-start" || event.event === "run-resume")
|
|
1486
|
+
break;
|
|
1487
|
+
if (event.event === "run-end")
|
|
1488
|
+
return undefined;
|
|
1489
|
+
}
|
|
1490
|
+
const latest = runEndTipProof(events.slice(from));
|
|
1491
|
+
tipProof = { ...latest, kind: latest.kind === "failed" ? "failed" : "incomplete" };
|
|
1344
1492
|
}
|
|
1345
1493
|
catch (readErr) {
|
|
1346
1494
|
console.error(`tickmarkr ${runId}: journal read failed while recording the fatal run-end (${readErr instanceof Error ? readErr.message : String(readErr)}) — original error: ${original}`);
|
|
@@ -1348,18 +1496,19 @@ function recordFatalRunEnd(journal, runId, branch, err) {
|
|
|
1348
1496
|
const record = {
|
|
1349
1497
|
runId,
|
|
1350
1498
|
branch,
|
|
1351
|
-
done: [],
|
|
1352
|
-
failed: [],
|
|
1353
|
-
human: [],
|
|
1354
|
-
blocked: [],
|
|
1355
|
-
pending: [],
|
|
1356
|
-
phase
|
|
1499
|
+
done: graph?.tasks.filter((t) => t.status === "done").map((t) => t.id) ?? [],
|
|
1500
|
+
failed: graph?.tasks.filter((t) => t.status === "failed").map((t) => t.id) ?? [],
|
|
1501
|
+
human: graph?.tasks.filter((t) => t.status === "human").map((t) => t.id) ?? [],
|
|
1502
|
+
blocked: graph ? blockedTasks(graph).map((t) => t.id) : [],
|
|
1503
|
+
pending: graph ? pendingTasks(graph).map((t) => t.id) : [],
|
|
1504
|
+
phase,
|
|
1357
1505
|
fatal: true,
|
|
1358
1506
|
error: original,
|
|
1507
|
+
tipProof,
|
|
1359
1508
|
};
|
|
1360
1509
|
try {
|
|
1361
1510
|
journal.append("run-end", undefined, record);
|
|
1362
|
-
return;
|
|
1511
|
+
return tipProof;
|
|
1363
1512
|
}
|
|
1364
1513
|
catch {
|
|
1365
1514
|
// one retry below — the first failure is reported only if the retry also fails
|
|
@@ -1368,6 +1517,7 @@ function recordFatalRunEnd(journal, runId, branch, err) {
|
|
|
1368
1517
|
try {
|
|
1369
1518
|
terminateTornJournalTail(journalPath);
|
|
1370
1519
|
journal.append("run-end", undefined, record);
|
|
1520
|
+
return tipProof;
|
|
1371
1521
|
}
|
|
1372
1522
|
catch (retryErr) {
|
|
1373
1523
|
console.error(`tickmarkr ${runId}: run crashed — no terminal record written; journal sink unwritable at ${journalPath} (${retryErr instanceof Error ? retryErr.message : String(retryErr)}) — original error: ${original}`);
|
|
@@ -1437,12 +1587,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1437
1587
|
let journal;
|
|
1438
1588
|
let runStarted = false;
|
|
1439
1589
|
let taskLoopStarted = false;
|
|
1590
|
+
let graph;
|
|
1591
|
+
let fatalPhase = "setup";
|
|
1592
|
+
let deliberateTermination = false;
|
|
1593
|
+
const fatalStop = new AbortController();
|
|
1594
|
+
const inflight = new Map();
|
|
1595
|
+
let retireFatalSlots;
|
|
1440
1596
|
let branch = "";
|
|
1441
1597
|
let releaseApprovalSerialization;
|
|
1442
1598
|
let baselineCapture = Promise.resolve();
|
|
1443
1599
|
let baselineFailed = false;
|
|
1444
1600
|
try {
|
|
1445
|
-
|
|
1601
|
+
graph = loadGraph(repoRoot);
|
|
1446
1602
|
// One bounded snapshot supplies every dispatch. On resume the current journal still participates
|
|
1447
1603
|
// in the fold so its task-done/ordinary-approval events can retire older evidence, but findings it
|
|
1448
1604
|
// produced are suppressed: same-run feedback already replays those bytes and must not deliver them
|
|
@@ -1554,6 +1710,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1554
1710
|
await reapWorker(slot);
|
|
1555
1711
|
await driver.close(slot);
|
|
1556
1712
|
} }, s);
|
|
1713
|
+
retireFatalSlots = async () => {
|
|
1714
|
+
for (const slot of Array.from(liveSlots)) {
|
|
1715
|
+
try {
|
|
1716
|
+
await closeSlot(slot);
|
|
1717
|
+
}
|
|
1718
|
+
catch { /* preserve the original fatal error */ }
|
|
1719
|
+
}
|
|
1720
|
+
};
|
|
1557
1721
|
const budgetSlots = new Map();
|
|
1558
1722
|
const trackedDriver = {
|
|
1559
1723
|
id: driver.id,
|
|
@@ -1562,19 +1726,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1562
1726
|
...(driver.describe ? { describe: driver.describe.bind(driver) } : {}),
|
|
1563
1727
|
...(driver.focus ? { focus: driver.focus.bind(driver) } : {}),
|
|
1564
1728
|
slot: async (cwd, name, o) => {
|
|
1729
|
+
fatalStop.signal.throwIfAborted();
|
|
1565
1730
|
const signal = executionSignal();
|
|
1566
1731
|
signal?.throwIfAborted();
|
|
1567
1732
|
const s = await driver.slot(cwd, name, o);
|
|
1568
1733
|
liveSlots.add(s);
|
|
1569
1734
|
if (signal)
|
|
1570
1735
|
budgetSlots.get(signal)?.add(s);
|
|
1571
|
-
if (signal?.aborted) {
|
|
1736
|
+
if (signal?.aborted || fatalStop.signal.aborted) {
|
|
1572
1737
|
await withoutExecutionBudget(() => closeSlot(s));
|
|
1573
|
-
signal.throwIfAborted();
|
|
1738
|
+
fatalStop.signal.throwIfAborted();
|
|
1739
|
+
signal?.throwIfAborted();
|
|
1574
1740
|
}
|
|
1575
1741
|
return s;
|
|
1576
1742
|
},
|
|
1577
|
-
run: (s, cmd) =>
|
|
1743
|
+
run: (s, cmd) => {
|
|
1744
|
+
fatalStop.signal.throwIfAborted();
|
|
1745
|
+
return driver.run(s, cmd);
|
|
1746
|
+
},
|
|
1578
1747
|
waitOutput: (s, p, ms, o) => driver.waitOutput(s, p, ms, o),
|
|
1579
1748
|
waitAgentStatus: (s, st, ms) => driver.waitAgentStatus(s, st, ms),
|
|
1580
1749
|
status: (s) => driver.status(s),
|
|
@@ -1617,6 +1786,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1617
1786
|
const exit = opts.exit ?? ((code) => process.exit(code));
|
|
1618
1787
|
onTermination = (sig) => {
|
|
1619
1788
|
termSignal = sig;
|
|
1789
|
+
deliberateTermination = true;
|
|
1620
1790
|
void (async () => {
|
|
1621
1791
|
if (!reaping) {
|
|
1622
1792
|
reaping = true;
|
|
@@ -1719,7 +1889,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1719
1889
|
// flag. Startup approvals seed the first scheduling pass; live approvals are folded at task
|
|
1720
1890
|
// boundaries below so a sibling can release parked work without waiting for run-end + resume.
|
|
1721
1891
|
const approvalStartupEvents = journal.read();
|
|
1722
|
-
|
|
1892
|
+
// Continuing human-gate permission survives enactment; malformed releases grant none.
|
|
1893
|
+
const validApproval = (e) => e.event === "task-approved" && e.taskId
|
|
1894
|
+
&& pendingApprovalActions([e]).get(e.taskId)?.authority !== "inert";
|
|
1895
|
+
const approved = new Set(approvalStartupEvents.filter(validApproval).map((e) => e.taskId));
|
|
1896
|
+
const startupActions = pendingDaemonApprovalActions(approvalStartupEvents);
|
|
1723
1897
|
let approvalSweepCursor = approvalStartupEvents.length;
|
|
1724
1898
|
const commands = detectGateCommands(repoRoot, cfg);
|
|
1725
1899
|
const watchName = formatOwnedName({ role: "watch", taskId: "run", attempt: 0, runId });
|
|
@@ -1825,6 +1999,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1825
1999
|
const startedAt = Date.now();
|
|
1826
2000
|
for (;;) {
|
|
1827
2001
|
signal?.throwIfAborted();
|
|
2002
|
+
fatalStop.signal.throwIfAborted();
|
|
1828
2003
|
executionSignal()?.throwIfAborted();
|
|
1829
2004
|
const count = await liveSuiteCount(repoRoot);
|
|
1830
2005
|
if (count === 0)
|
|
@@ -1866,11 +2041,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1866
2041
|
// Empty Map on fresh runs — every seed below conditions on resume.get(t.id), never on opts.resume (the
|
|
1867
2042
|
// GATE-08 lesson at the humanGate guard: condition on the data, not the code path). Dead-code
|
|
1868
2043
|
// equivalence to the router.ts:194 profile⇒undefined pattern: no map entry ⇒ today's literal.
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
2044
|
+
const resume = opts.resume ? journal.replayResumeState() : new Map();
|
|
2045
|
+
const satisfiedGates = opts.resume ? journal.replaySatisfiedGates() : new Map();
|
|
2046
|
+
const replayedGateResults = resumeLifecycleOpen
|
|
1872
2047
|
? journal.replayCurrentAttemptGateResults()
|
|
1873
2048
|
: new Map();
|
|
2049
|
+
for (const [taskId, action] of startupActions) {
|
|
2050
|
+
if (action.authority === "worker")
|
|
2051
|
+
replayedGateResults.delete(taskId);
|
|
2052
|
+
}
|
|
1874
2053
|
// T7: the capacity THIS session resolved — read inside the run's fork budget, so it is the number
|
|
1875
2054
|
// every shell this run spawns will divide the machine by. Recorded evidence from another session
|
|
1876
2055
|
// is only reusable against this.
|
|
@@ -1907,7 +2086,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1907
2086
|
// The SAME comparator status uses (engagementComparable); one decision, two consumers. Fail closed:
|
|
1908
2087
|
// no resume path silently accepts a mismatched or unbound journal. --graph-changed is the operator's
|
|
1909
2088
|
// audited release for the stop-amend-resume workflow, journaling a graph-rehash naming both hashes.
|
|
1910
|
-
graph = applyScopeAmendments(graph, journal, true);
|
|
2089
|
+
graph = applyScopeAmendments(graph, journal, true, opts.graphChanged === true);
|
|
1911
2090
|
const loadedHash = graphDefinitionHash(graph);
|
|
1912
2091
|
const cmp = engagementComparable(journal.read(), loadedHash);
|
|
1913
2092
|
if (!cmp.comparable) {
|
|
@@ -1923,7 +2102,29 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1923
2102
|
}
|
|
1924
2103
|
baseline = JSON.parse(readFileSync(join(journal.dir, "baseline.json"), "utf8"));
|
|
1925
2104
|
const replayEvents = journal.read();
|
|
1926
|
-
|
|
2105
|
+
// journal.replayStatuses predates typed releases and re-pends even inert rows. Keep that
|
|
2106
|
+
// legacy reader intact; scheduling accepts only the fold's recognised authorities.
|
|
2107
|
+
const statuses = new Map();
|
|
2108
|
+
for (const e of replayEvents) {
|
|
2109
|
+
if (!e.taskId)
|
|
2110
|
+
continue;
|
|
2111
|
+
if (e.event === "task-dispatch")
|
|
2112
|
+
statuses.set(e.taskId, "pending");
|
|
2113
|
+
else if (e.event === "task-done")
|
|
2114
|
+
statuses.set(e.taskId, "done");
|
|
2115
|
+
else if (e.event === "task-failed")
|
|
2116
|
+
statuses.set(e.taskId, "failed");
|
|
2117
|
+
else if (e.event === "task-human")
|
|
2118
|
+
statuses.set(e.taskId, "human");
|
|
2119
|
+
else if (validApproval(e))
|
|
2120
|
+
statuses.set(e.taskId, "pending");
|
|
2121
|
+
}
|
|
2122
|
+
for (const [id, action] of startupActions) {
|
|
2123
|
+
// Legacy gate-only approvals can finish and merge without a dispatch consuming funding.
|
|
2124
|
+
if (action.authority !== "inert" && statuses.get(id) !== "done")
|
|
2125
|
+
statuses.set(id, "pending");
|
|
2126
|
+
}
|
|
2127
|
+
for (const [id, st] of statuses) {
|
|
1927
2128
|
if (opts.retryFailed && st === "failed" && recordedTaskFailureKind(replayEvents, id) === "dispatch") {
|
|
1928
2129
|
graph = setStatus(graph, id, "pending");
|
|
1929
2130
|
// OBS-254: clear ATTEMPT AND CHANNEL STATE ONLY. Deleting the whole entry also deleted
|
|
@@ -1944,7 +2145,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1944
2145
|
pid: process.pid, // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
1945
2146
|
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
1946
2147
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
2148
|
+
...(opts.graphChanged ? { graphChanged: true } : {}), // OBS-1073: the release is the engagement's
|
|
1947
2149
|
});
|
|
2150
|
+
runStarted = true;
|
|
1948
2151
|
await placeBoard();
|
|
1949
2152
|
}
|
|
1950
2153
|
else {
|
|
@@ -2066,6 +2269,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2066
2269
|
let mergeChain = Promise.resolve();
|
|
2067
2270
|
const mergeSerial = (taskBranch, t, gated) => {
|
|
2068
2271
|
const next = mergeChain.then(() => {
|
|
2272
|
+
fatalStop.signal.throwIfAborted();
|
|
2069
2273
|
executionSignal()?.throwIfAborted();
|
|
2070
2274
|
journal.phaseStart(t.id, "merge");
|
|
2071
2275
|
return mergeTask(intWt, taskBranch, `tickmarkr: merge ${t.id} ${t.title}`, gated);
|
|
@@ -2083,15 +2287,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2083
2287
|
];
|
|
2084
2288
|
return `${identity} — release with ${commands.map((command) => `\`${command}\``).join(" or ")}`;
|
|
2085
2289
|
};
|
|
2086
|
-
//
|
|
2087
|
-
//
|
|
2088
|
-
|
|
2089
|
-
const namedPaths = (text, present) => [...text.matchAll(/(?:^|[\s`'"(])((?:[A-Za-z0-9_@.()[\]-]+\/)+[A-Za-z0-9_@.[\]-]+|[A-Za-z0-9_@-]+(?:\.[A-Za-z0-9_-]+)+)(?=$|[\s`'"),:;.!?])/g)]
|
|
2290
|
+
// Lexing never starts in the middle of a word. Resolution is separate so missing hints
|
|
2291
|
+
// remain visible evidence without becoming executable approval advice.
|
|
2292
|
+
const namedPaths = (text) => [...text.matchAll(/(?:^|[\s`'"])((?:[A-Za-z0-9_@.()[\]-]+\/)+[A-Za-z0-9_@.[\]-]+|[A-Za-z0-9_@-]+(?:\.[A-Za-z0-9_-]+)+)(?=$|[\s`'"),:;.!?])/g)]
|
|
2090
2293
|
.map((match) => match[1].replace(/^\.\//, "").replace(/\.$/, ""))
|
|
2091
|
-
.filter((path) => !path.split("/").includes("..")
|
|
2092
|
-
&& (present
|
|
2093
|
-
? present.has(path)
|
|
2094
|
-
: (/\.[A-Za-z][A-Za-z0-9_-]*$/.test(basename(path)) || existsSync(join(repoRoot, path)))));
|
|
2294
|
+
.filter((path) => !path.split("/").includes(".."));
|
|
2095
2295
|
const diffPaths = async (base, wt) => {
|
|
2096
2296
|
const out = await shGit(`git diff --name-only -z ${shq(base)}..HEAD`, wt);
|
|
2097
2297
|
return new Set(out.code === 0 ? out.stdout.split("\0").filter(Boolean) : []);
|
|
@@ -2101,15 +2301,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2101
2301
|
const tree = await shGit("git ls-files -z", wt);
|
|
2102
2302
|
return new Set([...(tree.code === 0 ? tree.stdout.split("\0").filter(Boolean) : []), ...await diffPaths(base, wt)]);
|
|
2103
2303
|
};
|
|
2304
|
+
// The base each task was dispatched on: the integration branch moves when a sibling merges, the pin does not.
|
|
2305
|
+
const taskBases = new Map();
|
|
2104
2306
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
2105
2307
|
// OBS-979: a worker refusal can identify the missing authoring scope even when gates
|
|
2106
2308
|
// subsequently supply the park's disposition. Keep that actionable path on the park itself.
|
|
2107
2309
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
2108
|
-
if (t.files.length > 0 && worker?.data.ok === false && typeof worker.data.summary === "string") {
|
|
2310
|
+
if (!details.approveCommand && t.files.length > 0 && worker?.data.ok === false && typeof worker.data.summary === "string") {
|
|
2109
2311
|
const allowed = filesGlob(t.files);
|
|
2110
|
-
|
|
2312
|
+
// OBS-1072 residual: the same inventory as the main path — the task tree and its diff from the
|
|
2313
|
+
// task's PINNED base, never the main checkout and never the moving integration branch.
|
|
2314
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
2315
|
+
const inventory = existsSync(wt) ? await treeOrDiffPaths(taskBases.get(t.id) ?? "HEAD", wt) : new Set();
|
|
2316
|
+
const paths = resolveScopeHints(namedPaths(worker.data.summary).filter((path) => !allowed(path)), inventory)
|
|
2317
|
+
.resolved.filter((path) => !allowed(path));
|
|
2111
2318
|
if (paths.length)
|
|
2112
|
-
reason += ` — files[] repair hint: ${
|
|
2319
|
+
reason += ` — files[] repair hint: ${paths.join(", ")}`;
|
|
2113
2320
|
}
|
|
2114
2321
|
graph = setStatus(graph, t.id, "human");
|
|
2115
2322
|
saveGraph(repoRoot, graph);
|
|
@@ -2140,35 +2347,34 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2140
2347
|
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode, changed = new Set()) => {
|
|
2141
2348
|
const scopeRed = results.find((g) => g.gate === "scope" && gateFailed(g));
|
|
2142
2349
|
const verdict = scopeRed?.meta?.collateral;
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
// worker made. Do not replace the scope gate's stronger attribution with that inference.
|
|
2153
|
-
const paths = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details, changed));
|
|
2154
|
-
if (refusal.length || (paths.length > 0 && paths.every((path) => !allowed(path)))) {
|
|
2155
|
-
const classification = {
|
|
2156
|
-
gate: refusal.length ? "worker" : reds[0]?.gate,
|
|
2157
|
-
paths: [...new Set(refusal.length ? refusal : paths)],
|
|
2158
|
-
repair: `files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}`,
|
|
2159
|
-
attempt: attempts + 1, chargeable: false, source: "diagnostic",
|
|
2160
|
-
};
|
|
2161
|
-
if (refusal.length)
|
|
2162
|
-
journal.append("scope-request", t.id, classification);
|
|
2163
|
-
else
|
|
2164
|
-
journal.append("scope-authoring", t.id, classification);
|
|
2165
|
-
// OBS-547 (Leg-2 T3 material): unchargeable ⇒ metered 0, exactly as the predicted path below —
|
|
2166
|
-
// passing the physical count would write `meteredAttempts: 1` beside `attempts: 0`.
|
|
2167
|
-
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${[...new Set(refusal)].map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2168
|
-
await park(t, `${refusal.length ? "scope request" : "authoring defect"} — files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}; current files[]: ${t.files.join(", ")}`, refusal.length ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, refusal.length ? { paths: [...new Set(refusal)], approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
2169
|
-
return true;
|
|
2170
|
-
}
|
|
2350
|
+
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
2351
|
+
const disposition = classifyRepairDisposition({
|
|
2352
|
+
results, files: t.files, inventory: changed,
|
|
2353
|
+
refusalSummary: worker?.data.ok === false && typeof worker.data.summary === "string" ? worker.data.summary : undefined,
|
|
2354
|
+
});
|
|
2355
|
+
for (const diagnostic of disposition.diagnostics) {
|
|
2356
|
+
journal.append("scope-hint-unresolved", t.id, {
|
|
2357
|
+
paths: [diagnostic.candidate], reason: diagnostic.reason, diagnostic, attempt: attempts + 1,
|
|
2358
|
+
});
|
|
2171
2359
|
}
|
|
2360
|
+
if (disposition.blocker === "infra")
|
|
2361
|
+
return false;
|
|
2362
|
+
if (!disposition.blocker && (disposition.kind === "scope-request" || disposition.kind === "authoring")) {
|
|
2363
|
+
const paths = disposition.paths;
|
|
2364
|
+
const scopeRequest = disposition.kind === "scope-request";
|
|
2365
|
+
const classification = {
|
|
2366
|
+
gate: disposition.source, paths, repair: `files[] repair hint: ${paths.join(", ")}`,
|
|
2367
|
+
attempt: attempts + 1, chargeable: false, source: "diagnostic",
|
|
2368
|
+
};
|
|
2369
|
+
if (scopeRequest)
|
|
2370
|
+
journal.append("scope-request", t.id, classification);
|
|
2371
|
+
else
|
|
2372
|
+
journal.append("scope-authoring", t.id, classification);
|
|
2373
|
+
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${paths.map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2374
|
+
await park(t, `${scopeRequest ? "scope request" : "authoring defect"} — files[] repair hint: ${paths.join(", ")}; current files[]: ${t.files.join(", ")}${scopeRequest ? `\n${approveCommand}` : ""}`, scopeRequest ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, scopeRequest ? { paths, approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
2375
|
+
return true;
|
|
2376
|
+
}
|
|
2377
|
+
// A detection site permits the ordinary chargeable policy; it does not itself grant budget.
|
|
2172
2378
|
if (!verdict)
|
|
2173
2379
|
return false;
|
|
2174
2380
|
if (verdict.authoring) {
|
|
@@ -2205,17 +2411,33 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2205
2411
|
approvalSweepCursor = events.length;
|
|
2206
2412
|
if (approvals.length === 0)
|
|
2207
2413
|
return;
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2414
|
+
// Replay every authority from this pass's single snapshot. In particular, helper reads must
|
|
2415
|
+
// neither rescan the file under load nor observe approvals beyond approvalSweepCursor.
|
|
2416
|
+
// Keep audit writes on the live journal so narration and persistence retain their owner.
|
|
2417
|
+
const sweepJournal = Journal.open(repoRoot, runId);
|
|
2418
|
+
sweepJournal.read = () => events;
|
|
2419
|
+
sweepJournal.append = journal.append.bind(journal);
|
|
2420
|
+
graph = applyScopeAmendments(graph, sweepJournal, false, opts.graphChanged === true); // OBS-1073: same release as launch
|
|
2421
|
+
const actions = pendingDaemonApprovalActions(events);
|
|
2422
|
+
const nextResume = sweepJournal.replayResumeState();
|
|
2423
|
+
const nextSatisfied = sweepJournal.replaySatisfiedGates();
|
|
2214
2424
|
let changed = false;
|
|
2215
|
-
for (const
|
|
2216
|
-
const
|
|
2425
|
+
for (const taskId of new Set(approvals.map((e) => e.taskId))) {
|
|
2426
|
+
const action = actions.get(taskId);
|
|
2427
|
+
if (!action || action.authority === "inert")
|
|
2428
|
+
continue;
|
|
2217
2429
|
approved.add(taskId);
|
|
2218
|
-
|
|
2430
|
+
const state = nextResume.get(taskId);
|
|
2431
|
+
if (state)
|
|
2432
|
+
resume.set(taskId, state);
|
|
2433
|
+
else
|
|
2434
|
+
resume.delete(taskId);
|
|
2435
|
+
const gate = nextSatisfied.get(taskId);
|
|
2436
|
+
if (gate)
|
|
2437
|
+
satisfiedGates.set(taskId, gate);
|
|
2438
|
+
else
|
|
2439
|
+
satisfiedGates.delete(taskId);
|
|
2440
|
+
if (action.authority !== "waiver")
|
|
2219
2441
|
replayedGateResults.delete(taskId);
|
|
2220
2442
|
try {
|
|
2221
2443
|
const task = getTask(graph, taskId);
|
|
@@ -2232,6 +2454,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2232
2454
|
saveGraph(repoRoot, graph);
|
|
2233
2455
|
};
|
|
2234
2456
|
const execTask = async (t) => {
|
|
2457
|
+
const approvalAction = pendingDaemonApprovalActions(journal.read()).get(t.id);
|
|
2458
|
+
const fundedRerun = approvalAction?.authority === "worker";
|
|
2235
2459
|
const startMs = Date.now();
|
|
2236
2460
|
let stallSeat;
|
|
2237
2461
|
let stallReaps = 0;
|
|
@@ -2363,6 +2587,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2363
2587
|
ctx.requiredRepairTests = decision.requiredFiles;
|
|
2364
2588
|
ctx.selectionReason = decision.reason;
|
|
2365
2589
|
}
|
|
2590
|
+
// Resume and ordinary verification share this boundary. Count persisted rounds so a
|
|
2591
|
+
// resumed daemon cannot reuse a previous round's identity within the same attempt.
|
|
2592
|
+
ctx.buildReceiptIdentity = {
|
|
2593
|
+
runId, taskId: task.id, attempt: gateSubject?.attempt ?? 0,
|
|
2594
|
+
gateRound: journal.read().filter((row) => row.taskId === task.id
|
|
2595
|
+
&& row.event === "phase-start" && row.data.phase === "gates").length,
|
|
2596
|
+
};
|
|
2366
2597
|
const round = await runGates(task, ctx);
|
|
2367
2598
|
let review = round.results.find((g) => g.gate === "review");
|
|
2368
2599
|
while (review?.meta?.noVerdict === true) {
|
|
@@ -2454,7 +2685,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2454
2685
|
...(typeof g.meta.preservedRef === "string" ? { preservedRef: g.meta.preservedRef } : {}),
|
|
2455
2686
|
} : {}),
|
|
2456
2687
|
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
2457
|
-
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision"]
|
|
2688
|
+
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision", "failureEvidence"]
|
|
2458
2689
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
2459
2690
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
2460
2691
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
@@ -2730,10 +2961,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2730
2961
|
// An operator approval skips its exact failed gate by authority; observed results skip only the
|
|
2731
2962
|
// contiguous green prefix whose recorded commit is still the task branch tip.
|
|
2732
2963
|
const satisfiedGate = satisfiedGates.get(t.id);
|
|
2733
|
-
const replayedGates = replayedGateResults.get(t.id);
|
|
2734
|
-
const recheck =
|
|
2964
|
+
const replayedGates = fundedRerun ? undefined : replayedGateResults.get(t.id);
|
|
2965
|
+
const recheck = approvalAction?.authority === "battery";
|
|
2735
2966
|
resumeGateReplay: if (satisfiedGate || replayedGates || recheck) {
|
|
2736
2967
|
const taskBase = await integrationHead(intWt);
|
|
2968
|
+
taskBases.set(t.id, taskBase);
|
|
2737
2969
|
const taskBranch = `${branch}--${t.id}`;
|
|
2738
2970
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
2739
2971
|
// OBS-1022: the ref `approve --recheck` validated (commits ahead of the base) is the tree the
|
|
@@ -2895,13 +3127,42 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2895
3127
|
else
|
|
2896
3128
|
reused.push(gate);
|
|
2897
3129
|
}
|
|
3130
|
+
// OBS-1049: build outputs are not commits. A replayed build verdict says the tree WAS green;
|
|
3131
|
+
// it says nothing about whether the RECREATED checkout holds what that build produced (dist),
|
|
3132
|
+
// and the host's npm lifecycle (ignore-scripts) may never rebuild it before the suite runs.
|
|
3133
|
+
// So a replayed build re-runs its own command here as provisioning, journaled by name; a red
|
|
3134
|
+
// provisioning is not reused — build (and everything behind it) re-enters the battery as a gate.
|
|
3135
|
+
// Every resume on this path recreates the checkout (recreateTaskWorktree above), so "reused
|
|
3136
|
+
// build" is exactly "build replayed onto a recreated tree".
|
|
3137
|
+
// Journal order is the criterion's order: a GREEN provisioning is recorded AFTER the reuse
|
|
3138
|
+
// rows it belongs to (gate-reused build, then gate-provisioned build); a RED provisioning is
|
|
3139
|
+
// recorded alone, and no reuse row follows it.
|
|
3140
|
+
let provisionedRow;
|
|
3141
|
+
if (reused.includes("build") && commands.build !== undefined) {
|
|
3142
|
+
// Provisioning is a verification command like any gate: it waits for the run's baseline and
|
|
3143
|
+
// takes the command lease (suite admission), never a bare shell beside sibling gates.
|
|
3144
|
+
await waitForBaseline(t.id);
|
|
3145
|
+
const startedAt = Date.now();
|
|
3146
|
+
const provisioned = await withCommandContext(t.id, () => sh(commands.build, wt));
|
|
3147
|
+
provisionedRow = {
|
|
3148
|
+
gate: "build", commit: replayedGates.commit, exitCode: provisioned.code, durationMs: Date.now() - startedAt,
|
|
3149
|
+
};
|
|
3150
|
+
if (provisioned.code !== 0) {
|
|
3151
|
+
reused.splice(reused.indexOf("build"));
|
|
3152
|
+
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
3153
|
+
provisionedRow = undefined;
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
2898
3156
|
for (const gate of reused) {
|
|
2899
3157
|
journal.append("gate-reused", t.id, { gate, commit: replayedGates.commit });
|
|
2900
3158
|
}
|
|
3159
|
+
if (provisionedRow !== undefined)
|
|
3160
|
+
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
2901
3161
|
remainingGates = declaredGates.slice(reused.length);
|
|
2902
3162
|
}
|
|
2903
3163
|
const resumedTask = { ...t, gates: remainingGates };
|
|
2904
3164
|
gateLoop: while (true) {
|
|
3165
|
+
fatalStop.signal.throwIfAborted();
|
|
2905
3166
|
executionSignal()?.throwIfAborted();
|
|
2906
3167
|
const gated = await gitHead(wt);
|
|
2907
3168
|
gateSubject = {
|
|
@@ -2935,6 +3196,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2935
3196
|
// Leg-2 (OBS-1052): the run-scoped two-strike tally — without it retirement is inert and a
|
|
2936
3197
|
// flaking seat is re-asked on every task. (carriedAuthors is held back: see SURGEON-LEG2-REPORT.)
|
|
2937
3198
|
reviewNoVerdicts,
|
|
3199
|
+
recheck, // OBS-1055: a recheck discards cached reds — the battery re-measures what the operator questioned
|
|
2938
3200
|
onGate: async (e) => {
|
|
2939
3201
|
if (e.phase === "start") {
|
|
2940
3202
|
notePhaseStart(e);
|
|
@@ -2958,7 +3220,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2958
3220
|
},
|
|
2959
3221
|
}, false));
|
|
2960
3222
|
results.forEach(classifySignalOnlyTest);
|
|
2961
|
-
if (
|
|
3223
|
+
if (pendingDaemonApprovalActions(journal.read()).get(t.id)?.authority === "battery") {
|
|
2962
3224
|
journal.append("recheck-battery", t.id, {
|
|
2963
3225
|
commit: gateSubject.commit,
|
|
2964
3226
|
gates: resumedTask.gates,
|
|
@@ -2997,8 +3259,41 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2997
3259
|
// Observed green gates are only measurements. If the resumed suffix is red, preserve that
|
|
2998
3260
|
// result in the journal and return to the ordinary attempt/consult ladder, which rebuilds
|
|
2999
3261
|
// feedback from those rows. Only an operator-authorized gate release parks on a new red.
|
|
3000
|
-
if (!satisfiedGate)
|
|
3262
|
+
if (!satisfiedGate) {
|
|
3263
|
+
// OBS-1055: a recheck red funds a repair of the pin's own work. A pinned task's repair sits
|
|
3264
|
+
// on the pin (OBS-1034's exemption keeps the tried list from excluding it); a pin the fleet
|
|
3265
|
+
// cannot seat parks naming the pin — never the ladder.
|
|
3266
|
+
const pin = recheck ? t.routingHints?.pin : undefined;
|
|
3267
|
+
if (pin) {
|
|
3268
|
+
const seat = channels.find((c) => c.adapter === pin.via && c.model === pin.model && !demotedChannels.has(channelKey(c)));
|
|
3269
|
+
if (!seat) {
|
|
3270
|
+
await park(t, `recheck red: pinned ${pin.via}:${pin.model} is unavailable to host the repair — refusing the ladder`, "gate-fail", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3271
|
+
return;
|
|
3272
|
+
}
|
|
3273
|
+
assignment = { adapter: seat.adapter, model: seat.model, channel: seat.channel, tier: seat.tier };
|
|
3274
|
+
}
|
|
3275
|
+
// Carry completeness was verified before this replay battery. Apply the ordinary
|
|
3276
|
+
// repair bounds here too; a restored red must fund the same findings-bearing dispatch.
|
|
3277
|
+
// Funding is durable intent only: repairReachSinceApproval charges it after launch
|
|
3278
|
+
// reaches a funded gate, and a restart before launch must not duplicate that intent.
|
|
3279
|
+
const failing = results.filter(gateFailed);
|
|
3280
|
+
const battery = failing.some((g) => g.gate !== "review") ? failing.filter((g) => g.gate !== "review") : failing;
|
|
3281
|
+
const landed = await commitsAheadOf(taskBase, wt);
|
|
3282
|
+
const events = journal.read();
|
|
3283
|
+
const drawn = repairsSinceApproval(events, t.id);
|
|
3284
|
+
const repeated = failing.some((g) => isDeterministicFailure(g)
|
|
3285
|
+
&& identicalGateFailures(events, t.id, g.gate, normalizeGateFailure(g.details)) >= GATE_FINGERPRINT_CAP);
|
|
3286
|
+
if (narrowRepairBattery(battery) && landed.length > 0 && drawn < MAX_REPAIRS
|
|
3287
|
+
&& (rs?.attempts ?? 0) < MAX_ATTEMPTS && !repeated
|
|
3288
|
+
&& pendingRepairFindings(events, t.id) === undefined) {
|
|
3289
|
+
journal.append("repair-attempt", t.id, {
|
|
3290
|
+
repair: repairReachSinceApproval(events, t.id).length + 1, charge: drawn + 1, of: MAX_REPAIRS,
|
|
3291
|
+
gates: failing.map((g) => g.gate), commits: landed.length,
|
|
3292
|
+
findings: withCarriedEvidence(repairFindingsBrief(results)),
|
|
3293
|
+
});
|
|
3294
|
+
}
|
|
3001
3295
|
break gateLoop;
|
|
3296
|
+
}
|
|
3002
3297
|
await park(t, gateFailApprovalReason(t.id, "post-approval gate failed", results.some((g) => g.gate === "review" && gateFailed(g))), "gate-fail", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3003
3298
|
return;
|
|
3004
3299
|
}
|
|
@@ -3042,6 +3337,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3042
3337
|
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
3043
3338
|
let supersededWorkerSlot;
|
|
3044
3339
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
3340
|
+
// Funding belongs to the next attempt. Later automatic repairs retain ordinary red reuse.
|
|
3341
|
+
const explicitlyFundedAttempt = fundedRerun && attempt === (rs?.attempts ?? 0);
|
|
3342
|
+
fatalStop.signal.throwIfAborted();
|
|
3045
3343
|
executionSignal()?.throwIfAborted();
|
|
3046
3344
|
if (attempt !== providerDeathAttempt) {
|
|
3047
3345
|
providerDeathRequeues = 0;
|
|
@@ -3216,6 +3514,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3216
3514
|
// T6: a dispatch that carries an outstanding finding says so, and names it. Without this the
|
|
3217
3515
|
// ledger cannot tell a carried dispatch from an amnesiac one — the exact question a run that
|
|
3218
3516
|
// spends two frontier attempts re-deriving a known defect has to be able to answer afterwards.
|
|
3517
|
+
fatalStop.signal.throwIfAborted();
|
|
3219
3518
|
journal.append("task-dispatch", t.id, {
|
|
3220
3519
|
...(scopeApproval?.data.release === "scope-request" ? { files: t.files, graphDefinitionHash: graphDefinitionHash(graph) } : {}),
|
|
3221
3520
|
assignment, attempt, provenance: dispatchProvenance([
|
|
@@ -3234,6 +3533,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3234
3533
|
});
|
|
3235
3534
|
journal.phaseStart(t.id, "worker", { attempt, assignment });
|
|
3236
3535
|
const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
|
|
3536
|
+
taskBases.set(t.id, taskBase);
|
|
3237
3537
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
3238
3538
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
3239
3539
|
const recreating = existsSync(priorWt);
|
|
@@ -3396,9 +3696,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3396
3696
|
`if [ -n "$worker_pgid" ] && [ -n "$daemon_pgid" ] && [ "$worker_pgid" != "$daemon_pgid" ]; then printf '%s\\n' "$worker_pgid" > ${shq(groupFile)}; fi`,
|
|
3397
3697
|
"export BASH_SILENCE_DEPRECATION_WARNING=1",
|
|
3398
3698
|
bannerShell(),
|
|
3699
|
+
`printf '%s\\n' 'TICKMARKR_DISPATCH_${nonce}'`,
|
|
3399
3700
|
workerCmd,
|
|
3400
3701
|
exitMarkerCmd,
|
|
3401
3702
|
].join("\n"));
|
|
3703
|
+
const workerTransport = heldWorkerTransport(driver, slot, nonce, (data) => journal.append("held-probe", t.id, { ...data, slot: slot.id, attempt }));
|
|
3402
3704
|
// T2 (OBS-264): the liveness triad, shared by BOTH wait loops (a headless worker stalls on
|
|
3403
3705
|
// finished work exactly as a visible one does — and rode the whole window before this). It
|
|
3404
3706
|
// sits ABOVE the fast-kill and the nudge because the population it governs is the opposite
|
|
@@ -3559,6 +3861,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3559
3861
|
let driverProbeFailed = false;
|
|
3560
3862
|
let heldLegs = [];
|
|
3561
3863
|
const noteDriverUnreadable = (error) => {
|
|
3864
|
+
if (error instanceof HeldProbeExhausted)
|
|
3865
|
+
throw error;
|
|
3562
3866
|
driverProbeFailed = true;
|
|
3563
3867
|
heldLegs = ["quota:driver-unreadable", "stall:driver-unreadable"];
|
|
3564
3868
|
journal.append("contact-unreadable", t.id, { slot: slot.name, attempt, source: "driver", concludes: false,
|
|
@@ -3657,8 +3961,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3657
3961
|
try {
|
|
3658
3962
|
seedResult = await runInteractiveSeed({
|
|
3659
3963
|
driver: trustAnswered ? {
|
|
3660
|
-
run:
|
|
3661
|
-
} :
|
|
3964
|
+
run: workerTransport.run.bind(workerTransport), waitOutput: workerTransport.waitOutput.bind(workerTransport), read: workerTransport.read.bind(workerTransport),
|
|
3965
|
+
} : { ...trackedDriver, ...workerTransport },
|
|
3662
3966
|
slot, adapter, assignment, promptFile, taskTimeoutMinutes,
|
|
3663
3967
|
onTrustAnswered: noteSeedTrustAnswered,
|
|
3664
3968
|
});
|
|
@@ -3675,7 +3979,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3675
3979
|
}
|
|
3676
3980
|
else {
|
|
3677
3981
|
try {
|
|
3678
|
-
await
|
|
3982
|
+
await workerTransport.run(slot, paneDispatchCommand(dispatchScript));
|
|
3679
3983
|
}
|
|
3680
3984
|
catch (error) {
|
|
3681
3985
|
if (!(error instanceof DeliveryReadinessError))
|
|
@@ -3685,7 +3989,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3685
3989
|
return;
|
|
3686
3990
|
}
|
|
3687
3991
|
await noteLaunched();
|
|
3688
|
-
output = await
|
|
3992
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3689
3993
|
}
|
|
3690
3994
|
// The returning paths report the same fact on the result; both callbacks land on the one
|
|
3691
3995
|
// latch, and the second is a no-op. A seed that answered is never re-answered by the loop.
|
|
@@ -3737,6 +4041,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3737
4041
|
while (Date.now() < hardDeadline) {
|
|
3738
4042
|
if (termSignal)
|
|
3739
4043
|
throw new Error(`terminated by ${termSignal}`);
|
|
4044
|
+
fatalStop.signal.throwIfAborted();
|
|
3740
4045
|
executionSignal()?.throwIfAborted();
|
|
3741
4046
|
driverProbeFailed = false;
|
|
3742
4047
|
const sliceStart = Date.now();
|
|
@@ -3758,11 +4063,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3758
4063
|
// every slice before the gate exactly as long as it already was.
|
|
3759
4064
|
slice = Math.min(slice, harvestSliceMs(sliceStart - lastProgressAt));
|
|
3760
4065
|
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
3761
|
-
if (await
|
|
4066
|
+
if (await workerTransport.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true }).catch((error) => { noteDriverUnreadable(error); return false; })) {
|
|
3762
4067
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
3763
4068
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
3764
4069
|
// parseable trailer or a digit-suffixed exit marker in the harvest is completion.
|
|
3765
|
-
output = await
|
|
4070
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; }); // TUI transcripts carry chrome — read deeper than print's 500
|
|
3766
4071
|
finished = sampleTrailer(output);
|
|
3767
4072
|
const exit = exitRe.exec(output);
|
|
3768
4073
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3786,7 +4091,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3786
4091
|
// rolling taskTimeoutMinutes window as the backstop and name the held probe once.
|
|
3787
4092
|
let paneText;
|
|
3788
4093
|
try {
|
|
3789
|
-
paneText = await
|
|
4094
|
+
paneText = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3790
4095
|
}
|
|
3791
4096
|
catch (error) {
|
|
3792
4097
|
noteDriverUnreadable(error);
|
|
@@ -3903,7 +4208,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3903
4208
|
// "unknown"/"working" never page.
|
|
3904
4209
|
let st;
|
|
3905
4210
|
try {
|
|
3906
|
-
st = await
|
|
4211
|
+
st = await workerTransport.status(slot);
|
|
3907
4212
|
}
|
|
3908
4213
|
catch (error) {
|
|
3909
4214
|
noteDriverUnreadable(error);
|
|
@@ -3929,7 +4234,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3929
4234
|
// text matches. tickmarkr created the worktree from the operator's own repo — safe by construction.
|
|
3930
4235
|
if (!trustAnswered && adapter.trustDialog && driver.sendKey) {
|
|
3931
4236
|
try {
|
|
3932
|
-
const paneText = await
|
|
4237
|
+
const paneText = await workerTransport.read(slot, 80);
|
|
3933
4238
|
if (matchesTrustDialog(paneText, adapter.trustDialog)) {
|
|
3934
4239
|
trustAnswered = true;
|
|
3935
4240
|
trustAnsweredSlots.add(slot.id);
|
|
@@ -3943,7 +4248,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3943
4248
|
continue; // do not page — keep waiting for the trailer
|
|
3944
4249
|
}
|
|
3945
4250
|
}
|
|
3946
|
-
catch {
|
|
4251
|
+
catch (error) {
|
|
4252
|
+
if (error instanceof HeldProbeExhausted)
|
|
4253
|
+
throw error;
|
|
3947
4254
|
/* read/send failed — fall through to page the operator */
|
|
3948
4255
|
}
|
|
3949
4256
|
}
|
|
@@ -3975,7 +4282,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3975
4282
|
let paneAbsent = paneAbsentCandidate;
|
|
3976
4283
|
if (paneAbsent) {
|
|
3977
4284
|
try {
|
|
3978
|
-
const confirmation = await
|
|
4285
|
+
const confirmation = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3979
4286
|
paneAbsent = confirmation.trim().length === 0;
|
|
3980
4287
|
if (!paneAbsent) {
|
|
3981
4288
|
everHadOutput = true;
|
|
@@ -3984,6 +4291,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3984
4291
|
}
|
|
3985
4292
|
}
|
|
3986
4293
|
catch (error) {
|
|
4294
|
+
if (error instanceof HeldProbeExhausted)
|
|
4295
|
+
throw error;
|
|
3987
4296
|
paneAbsent = false;
|
|
3988
4297
|
if (!paneReadHeld) {
|
|
3989
4298
|
paneReadHeld = true;
|
|
@@ -3999,7 +4308,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3999
4308
|
: "unmeasurable";
|
|
4000
4309
|
if (processTree === "empty") {
|
|
4001
4310
|
try {
|
|
4002
|
-
const confirmation = await
|
|
4311
|
+
const confirmation = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
4003
4312
|
paneAbsent = confirmation.trim().length === 0;
|
|
4004
4313
|
if (!paneAbsent) {
|
|
4005
4314
|
everHadOutput = true;
|
|
@@ -4008,6 +4317,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4008
4317
|
}
|
|
4009
4318
|
}
|
|
4010
4319
|
catch (error) {
|
|
4320
|
+
if (error instanceof HeldProbeExhausted)
|
|
4321
|
+
throw error;
|
|
4011
4322
|
paneAbsent = false;
|
|
4012
4323
|
if (!paneReadHeld) {
|
|
4013
4324
|
paneReadHeld = true;
|
|
@@ -4167,7 +4478,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4167
4478
|
if (delivered) {
|
|
4168
4479
|
// absorb the nudge's own echo BEFORE arming the grace timer — post-nudge progress
|
|
4169
4480
|
// is measured against this baseline, not against the echo.
|
|
4170
|
-
const echo = await
|
|
4481
|
+
const echo = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4171
4482
|
if (echo !== undefined)
|
|
4172
4483
|
stallProgress.observe({ paneText: echo, contextTokens });
|
|
4173
4484
|
nudgeDeadline = Date.now() + workerNudgeGraceMs;
|
|
@@ -4184,7 +4495,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4184
4495
|
// grace spent, still no post-nudge progress: re-harvest once (the trailer may have
|
|
4185
4496
|
// landed between polls), then conclude the wait as a stall NOW — the consult sees the
|
|
4186
4497
|
// un-answered nudge instead of the remainder of the window.
|
|
4187
|
-
const finalPane = await
|
|
4498
|
+
const finalPane = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4188
4499
|
if (finalPane === undefined)
|
|
4189
4500
|
continue;
|
|
4190
4501
|
nudgeDeadline = undefined;
|
|
@@ -4239,9 +4550,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4239
4550
|
if (hardTimedOut)
|
|
4240
4551
|
journal.append("worker-hard-timeout", t.id, { slot: slot.name, attempt, heldLegs });
|
|
4241
4552
|
try {
|
|
4242
|
-
output = await
|
|
4553
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
4243
4554
|
}
|
|
4244
|
-
catch {
|
|
4555
|
+
catch (error) {
|
|
4556
|
+
if (error instanceof HeldProbeExhausted)
|
|
4557
|
+
throw error;
|
|
4245
4558
|
// The poll loop already recorded the unreadable pane. Retain the last readable bytes
|
|
4246
4559
|
// so this ambiguous path still reaches the ordinary timeout/consult backstop.
|
|
4247
4560
|
}
|
|
@@ -4251,8 +4564,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4251
4564
|
processExited = exit !== null;
|
|
4252
4565
|
}
|
|
4253
4566
|
if (finished && !trailerFrames) {
|
|
4254
|
-
await
|
|
4255
|
-
const settled = await
|
|
4567
|
+
await workerTransport.waitAgentStatus(slot, "idle", 5_000).catch(noteDriverUnreadable);
|
|
4568
|
+
const settled = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; });
|
|
4256
4569
|
if (sampleTrailer(settled))
|
|
4257
4570
|
output = settled;
|
|
4258
4571
|
}
|
|
@@ -4271,7 +4584,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4271
4584
|
if (remaining <= 0)
|
|
4272
4585
|
break;
|
|
4273
4586
|
await new Promise((r) => setTimeout(r, Math.min(settleDelayMs, remaining)));
|
|
4274
|
-
output = await
|
|
4587
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
4275
4588
|
trailerFrames?.sample(output, new RegExp(trailerPattern(nonce)).test(output));
|
|
4276
4589
|
settleParsed = adapter.parse(output, nonce);
|
|
4277
4590
|
settleTries++;
|
|
@@ -4284,7 +4597,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4284
4597
|
}
|
|
4285
4598
|
else {
|
|
4286
4599
|
try {
|
|
4287
|
-
await
|
|
4600
|
+
await workerTransport.run(slot, paneDispatchCommand(dispatchScript));
|
|
4288
4601
|
}
|
|
4289
4602
|
catch (error) {
|
|
4290
4603
|
if (!(error instanceof DeliveryReadinessError))
|
|
@@ -4297,7 +4610,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4297
4610
|
// OBS-54: headless workers have the same output-inactivity budget as visible panes.
|
|
4298
4611
|
// v1.76: same monotonic-progress measure as the interactive site; harvest stays raw.
|
|
4299
4612
|
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
4300
|
-
const initialPane = await
|
|
4613
|
+
const initialPane = await workerTransport.read(slot, 500);
|
|
4301
4614
|
startupFailure = startupFailureInWindow(initialPane, workerLaunchedAt);
|
|
4302
4615
|
let everHadOutput = initialPane.length > 0;
|
|
4303
4616
|
const stallProgress = new StallProgressTracker();
|
|
@@ -4312,6 +4625,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4312
4625
|
// there is the trailer regex; the exit marker only sets exitCode), and the cause taxonomy
|
|
4313
4626
|
// already names this shape "clean-exit-no-trailer" — unreachable in print mode until now.
|
|
4314
4627
|
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
4628
|
+
fatalStop.signal.throwIfAborted();
|
|
4315
4629
|
executionSignal()?.throwIfAborted();
|
|
4316
4630
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
4317
4631
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
@@ -4328,11 +4642,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4328
4642
|
// T2 (OBS-264): same probe cadence the interactive loop uses — see harvestSliceMs.
|
|
4329
4643
|
slice = Math.min(slice, harvestSliceMs(Date.now() - lastProgressAt));
|
|
4330
4644
|
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
4331
|
-
if (await
|
|
4645
|
+
if (await workerTransport.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
4332
4646
|
processExited = true;
|
|
4333
4647
|
break;
|
|
4334
4648
|
}
|
|
4335
|
-
const paneText = await
|
|
4649
|
+
const paneText = await workerTransport.read(slot, 500);
|
|
4336
4650
|
if (paneText.length > 0)
|
|
4337
4651
|
everHadOutput = true;
|
|
4338
4652
|
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
@@ -4357,7 +4671,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4357
4671
|
if (await harvestConcludes(Date.now() - lastProgressAt))
|
|
4358
4672
|
break;
|
|
4359
4673
|
}
|
|
4360
|
-
output = await
|
|
4674
|
+
output = await workerTransport.read(slot, 500);
|
|
4361
4675
|
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
4362
4676
|
// Completion is the trailer, exactly as in the interactive loop. A process that exited
|
|
4363
4677
|
// without one is finished:false with a non-null exitCode — the harvest synthesis then owns
|
|
@@ -4375,13 +4689,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4375
4689
|
// interactive attempts to the same exit marker before close and the post-hoc usage disk read
|
|
4376
4690
|
// so a writer never races the reader (real CLIs can flush usage asynchronously after the trailer).
|
|
4377
4691
|
if (interactive && finished && !exitRe.test(output)) {
|
|
4378
|
-
await
|
|
4692
|
+
await workerTransport.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true }).catch(noteDriverUnreadable);
|
|
4379
4693
|
}
|
|
4380
4694
|
// Free availability reroutes deliberately reuse the charged-attempt ordinal. Stream artifacts
|
|
4381
4695
|
// cannot: every dispatch owns bytes that must remain independently recoverable, so derive the
|
|
4382
4696
|
// append-only dispatch ordinal from the journal instead of overwriting an earlier a<n>.out.
|
|
4383
4697
|
const streamAttempt = journal.read().filter((event) => event.event === "task-dispatch" && event.taskId === t.id).length - 1;
|
|
4384
|
-
const capturedStream = await captureWorkerStream(journal.dir, t.id, streamAttempt,
|
|
4698
|
+
const capturedStream = await captureWorkerStream(journal.dir, t.id, streamAttempt, { ...trackedDriver, ...workerTransport }, slot, output);
|
|
4385
4699
|
// An interactive settle already selected the bounded parse it is allowed to use. Every other
|
|
4386
4700
|
// path parses the common final stream snapshot — notably envelope adapters whose trailer is
|
|
4387
4701
|
// encoded and therefore invisible to the raw wait regex. Keep `output` bounded: every text
|
|
@@ -4466,7 +4780,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4466
4780
|
// (provider outage, quota banner, dead CLI) must still route; the harvest synthesis only
|
|
4467
4781
|
// decides whether THIS attempt's worktree goes to gates, and when routing wins instead, the
|
|
4468
4782
|
// commits survive via the existing commitsToCarry/cherryPickCommits carry-forward.
|
|
4469
|
-
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4783
|
+
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
4470
4784
|
return;
|
|
4471
4785
|
if (!workerFinished && !processExited && (timedOut || deadChannelKilled || quotaBannerKilled)) {
|
|
4472
4786
|
const seat = channelKey(assignment);
|
|
@@ -4741,6 +5055,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4741
5055
|
let results = [];
|
|
4742
5056
|
let commits = [];
|
|
4743
5057
|
gateLoop: while (true) {
|
|
5058
|
+
fatalStop.signal.throwIfAborted();
|
|
4744
5059
|
executionSignal()?.throwIfAborted();
|
|
4745
5060
|
const gated = await gitHead(wt);
|
|
4746
5061
|
gateSubject = { commit: await gateCommitSubject(taskBase, gated, wt), attempt };
|
|
@@ -4755,7 +5070,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4755
5070
|
&& e.data.attempt === attempt - 1)
|
|
4756
5071
|
: [];
|
|
4757
5072
|
journal.phaseStart(t.id, "gates");
|
|
4758
|
-
const replay = previousRows.length > 0
|
|
5073
|
+
const replay = !explicitlyFundedAttempt && previousRows.length > 0
|
|
4759
5074
|
&& previousRows.every((e) => e.data.commit === gateSubject.commit)
|
|
4760
5075
|
&& previousRows.some((e) => e.data.pass === false && e.data.skipped !== true);
|
|
4761
5076
|
if (replay) {
|
|
@@ -4782,6 +5097,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4782
5097
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4783
5098
|
collateral: collateral.get(t.id) ?? [],
|
|
4784
5099
|
selectTests: !testGateFailed,
|
|
5100
|
+
cachedRedBypass: explicitlyFundedAttempt ? "operator-rerun" : undefined,
|
|
4785
5101
|
via: cfg.visibility.llm === "pane"
|
|
4786
5102
|
? {
|
|
4787
5103
|
driver: trackedDriver,
|
|
@@ -4820,6 +5136,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4820
5136
|
// gate-fail brief is, so prior-RUN evidence (retired by its own rule, not by this reviewer)
|
|
4821
5137
|
// survives and only this run's settled findings go.
|
|
4822
5138
|
feedback = withCarriedEvidence("");
|
|
5139
|
+
fatalStop.signal.throwIfAborted();
|
|
4823
5140
|
executionSignal()?.throwIfAborted();
|
|
4824
5141
|
const m = await mergeSerial(taskBranch, t, gated);
|
|
4825
5142
|
if (m.tipMoved) {
|
|
@@ -4889,7 +5206,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4889
5206
|
// v1.53 T3: prefer the CLI's own session id captured from this attempt's output (kimi's resume
|
|
4890
5207
|
// trailer) over the harness slot name; absent hook or no capture keeps today's slot-name id.
|
|
4891
5208
|
retrySession = { channel: channelKey(assignment), id: adapter.sessionIdFrom?.(output) ?? sessionId, contextTokens };
|
|
4892
|
-
feedback = withCarriedEvidence(results
|
|
5209
|
+
feedback = withCarriedEvidence(repairFindingsBrief(results));
|
|
4893
5210
|
// OBS-189/G3 (park-economics patch): a request-changes review is a findings brief, not a worker
|
|
4894
5211
|
// defect — the fix attempt stays on the same channel with the findings as feedback and consumes
|
|
4895
5212
|
// no escalation-ladder rung. Bounded by the engagement round cap at the top of this loop.
|
|
@@ -4917,7 +5234,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4917
5234
|
const repairable = narrowRepairBattery(repairBattery) && lostCommits.length === 0 && landed.length > 0;
|
|
4918
5235
|
const repairHistory = repairReachSinceApproval(journal.read(), t.id);
|
|
4919
5236
|
const repairsDrawn = repairsSinceApproval(journal.read(), t.id);
|
|
4920
|
-
const repair = repairable && repairsDrawn < MAX_REPAIRS;
|
|
5237
|
+
const repair = repairable && repairsDrawn < MAX_REPAIRS && attempt + 1 < MAX_ATTEMPTS;
|
|
4921
5238
|
// v1.85 T3: the fingerprint cap. Two normalized-identical failures of one DETERMINISTIC gate on
|
|
4922
5239
|
// one task (volatile tokens — worktree prefixes, line refs, durations, run ids — are not
|
|
4923
5240
|
// information) mean the round about to be bought is a re-measurement: ~663m across 5 runs went to
|
|
@@ -5077,7 +5394,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5077
5394
|
});
|
|
5078
5395
|
};
|
|
5079
5396
|
taskLoopStarted = true;
|
|
5080
|
-
|
|
5397
|
+
fatalPhase = "scheduler";
|
|
5081
5398
|
// A settled worker can release a dependency or an approval in the same poll tick.
|
|
5082
5399
|
// Audit the current graph at every empty-flight boundary, never a prior ready snapshot.
|
|
5083
5400
|
// WB-1 rider: `end-condition-held` names a CLOSE the daemon refused, so only a close attempt journals
|
|
@@ -5096,6 +5413,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5096
5413
|
return true;
|
|
5097
5414
|
};
|
|
5098
5415
|
closeLoop: while (true) {
|
|
5416
|
+
fatalPhase = "scheduler";
|
|
5099
5417
|
let approvalDeadline;
|
|
5100
5418
|
while (true) {
|
|
5101
5419
|
// v1.54 T2: a signal that landed while nothing was racing `aborted` (empty inflight window)
|
|
@@ -5122,7 +5440,23 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5122
5440
|
cleanupErrors.push({ slot: slot.name, attempt: owner.attempt, error: String(error) });
|
|
5123
5441
|
}
|
|
5124
5442
|
}
|
|
5443
|
+
if (fatalStop.signal.aborted)
|
|
5444
|
+
return;
|
|
5125
5445
|
const cleanupEvidence = cleanupErrors.length ? { cleanupErrors } : {};
|
|
5446
|
+
if (err instanceof HeldProbeExhausted) {
|
|
5447
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
5448
|
+
let ref = await withoutExecutionBudget(() => preserveWorktree(wt));
|
|
5449
|
+
if (!ref) {
|
|
5450
|
+
const head = await gitHead(wt);
|
|
5451
|
+
ref = `refs/tickmarkr/preserved/${head}`;
|
|
5452
|
+
const saved = await shGit(`git update-ref ${shq(ref)} ${shq(head)}`, wt);
|
|
5453
|
+
if (saved.code !== 0)
|
|
5454
|
+
throw new Error(`could not preserve ${head}: ${saved.stderr}`);
|
|
5455
|
+
}
|
|
5456
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
5457
|
+
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: "transport-uncertain", ref, ...cleanupEvidence });
|
|
5458
|
+
return;
|
|
5459
|
+
}
|
|
5126
5460
|
if (err instanceof ExecutionBudgetExceeded) {
|
|
5127
5461
|
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
5128
5462
|
const ref = await preserveWorktree(wt);
|
|
@@ -5210,7 +5544,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5210
5544
|
blocked: blockedTasks(graph).map((t) => t.id),
|
|
5211
5545
|
pending: pendingTasks(graph).map((t) => t.id),
|
|
5212
5546
|
};
|
|
5547
|
+
fatalPhase = "baseline";
|
|
5213
5548
|
await baselineCapture; // also drain capture when every worker parks or fails before its gates
|
|
5549
|
+
fatalPhase = "tip-verify";
|
|
5214
5550
|
// OBS-34: post-merge integration-tip verify — strict exit codes, no baseline forgiveness.
|
|
5215
5551
|
const lastMergedTask = [...journal.read()].reverse().find((e) => e.event === "merge" && e.taskId)?.taskId;
|
|
5216
5552
|
if (summary.done.length > 0 && Object.keys(commands).length > 0) {
|
|
@@ -5253,9 +5589,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5253
5589
|
clearTimeout(poll);
|
|
5254
5590
|
}
|
|
5255
5591
|
summary.tipVerify = tipFailed ? "failed" : "passed";
|
|
5592
|
+
summary.tipProof = runEndTipProof(journal.read());
|
|
5256
5593
|
if (tipFailed && lastMergedTask)
|
|
5257
5594
|
summary.lastMergedTask = lastMergedTask;
|
|
5258
5595
|
}
|
|
5596
|
+
else {
|
|
5597
|
+
// Exactly one proof per close: no cycle ran for THIS close, and an earlier one is not inherited.
|
|
5598
|
+
summary.tipProof = { kind: "incomplete" };
|
|
5599
|
+
}
|
|
5259
5600
|
// T14: read the journal, not the startup `approved` set — an approval appended DURING this run is
|
|
5260
5601
|
// exactly the one the set cannot see, and it is the one the record has to name.
|
|
5261
5602
|
//
|
|
@@ -5263,6 +5604,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5263
5604
|
// graph.lock release in the outer finally: an approval that wins first is included below, while an
|
|
5264
5605
|
// approval that loses cannot append until the live owner is gone and reports recorded-no-owner.
|
|
5265
5606
|
// Thus no accepted approval can land after this sample while still being attributed to this run.
|
|
5607
|
+
fatalPhase = "scheduler";
|
|
5266
5608
|
const approvalSerialization = await acquireApprovalSerialization(repoRoot, runId);
|
|
5267
5609
|
releaseApprovalSerialization = approvalSerialization.release;
|
|
5268
5610
|
// Close the last poll-to-run-end race while holding the same serializer as approve.
|
|
@@ -5294,7 +5636,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5294
5636
|
const tipFail = summary.tipVerify === "failed"
|
|
5295
5637
|
? ` — TIP VERIFY FAILED on ${summary.lastMergedTask ? `last merge ${summary.lastMergedTask}` : "integration tip"}`
|
|
5296
5638
|
: "";
|
|
5297
|
-
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
|
|
5639
|
+
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — ${formatTipProof(summary.tipProof)} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
|
|
5298
5640
|
return summary;
|
|
5299
5641
|
}
|
|
5300
5642
|
});
|
|
@@ -5304,8 +5646,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5304
5646
|
// T7 (v1.86): guarded — a journal read/append failure while recording the fatal run-end is
|
|
5305
5647
|
// reported alongside err, never instead of it; recordFatalRunEnd never throws, so the original
|
|
5306
5648
|
// error (message, stack, cause) always reaches the caller verbatim.
|
|
5307
|
-
|
|
5308
|
-
|
|
5649
|
+
// Freeze failure-time buckets, stop new dispatch, and drain all journal writers before closing.
|
|
5650
|
+
if (!deliberateTermination) {
|
|
5651
|
+
const failedGraph = graph;
|
|
5652
|
+
const phase = taskLoopStarted ? (baselineFailed ? "baseline" : fatalPhase) : "setup";
|
|
5653
|
+
fatalStop.abort(err);
|
|
5654
|
+
await retireFatalSlots?.();
|
|
5655
|
+
await Promise.allSettled(inflight.values());
|
|
5656
|
+
await baselineCapture.catch(() => { });
|
|
5657
|
+
if (runStarted && !deliberateTermination) {
|
|
5658
|
+
const tipProof = recordFatalRunEnd(journal, runId, branch, err, taskLoopStarted || opts.resume ? failedGraph : undefined, phase);
|
|
5659
|
+
// The fatal close states its proof too; a dead notifier never replaces the original error.
|
|
5660
|
+
if (tipProof) {
|
|
5661
|
+
await driver.notify(`tickmarkr ${runId}: run crashed — ${formatTipProof(tipProof)} — integration branch ${branch}`, { tier: "attention" }).catch(() => { });
|
|
5662
|
+
}
|
|
5663
|
+
}
|
|
5664
|
+
}
|
|
5309
5665
|
throw err;
|
|
5310
5666
|
}
|
|
5311
5667
|
finally {
|