tickmarkr 2.5.5 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/adapters/prompt.js +21 -1
- package/dist/cli/commands/approve.js +69 -8
- package/dist/cli/commands/doctor.d.ts +10 -0
- package/dist/cli/commands/fleet.js +4 -0
- package/dist/cli/commands/plan.js +20 -4
- package/dist/cli/commands/status.js +95 -34
- package/dist/cli/commands/verify.js +108 -85
- package/dist/config/config.d.ts +11 -0
- package/dist/config/config.js +21 -12
- package/dist/config/fleet-overlay.js +55 -17
- package/dist/drivers/index.js +2 -1
- package/dist/drivers/orca.d.ts +21 -1
- package/dist/drivers/orca.js +209 -27
- package/dist/gates/baseline.d.ts +21 -5
- package/dist/gates/baseline.js +67 -17
- package/dist/gates/cache.d.ts +14 -13
- package/dist/gates/cache.js +17 -5
- package/dist/gates/llm.d.ts +3 -0
- package/dist/gates/llm.js +11 -0
- package/dist/gates/review.d.ts +28 -3
- package/dist/gates/review.js +118 -15
- package/dist/gates/run-gates.d.ts +10 -2
- package/dist/gates/run-gates.js +362 -108
- package/dist/gates/test-manifest.d.ts +33 -1
- package/dist/gates/test-manifest.js +132 -40
- package/dist/gates/test-reporter.js +20 -7
- package/dist/graph/graph.d.ts +4 -0
- package/dist/graph/graph.js +50 -1
- package/dist/run/activity.d.ts +28 -0
- package/dist/run/activity.js +194 -0
- package/dist/run/consult.js +5 -4
- package/dist/run/daemon.d.ts +11 -0
- package/dist/run/daemon.js +663 -128
- package/dist/run/execution-budget.d.ts +25 -0
- package/dist/run/execution-budget.js +142 -0
- package/dist/run/git.d.ts +46 -1
- package/dist/run/git.js +149 -12
- package/dist/run/journal.d.ts +23 -5
- package/dist/run/journal.js +98 -25
- package/dist/run/lease.d.ts +44 -0
- package/dist/run/lease.js +226 -3
- package/dist/run/operator-page-summary.d.ts +56 -0
- package/dist/run/operator-page-summary.js +68 -0
- package/dist/run/operator-summary.d.ts +69 -0
- package/dist/run/operator-summary.js +77 -0
- package/dist/run/protocol.d.ts +71 -0
- package/dist/run/protocol.js +32 -0
- package/dist/run/recovery.d.ts +8 -0
- package/dist/run/recovery.js +25 -0
- package/dist/run/repair-selection.d.ts +12 -0
- package/dist/run/repair-selection.js +56 -0
- package/dist/run/stall.d.ts +6 -1
- package/dist/run/stall.js +60 -3
- package/dist/tui/cockpit/board.d.ts +9 -0
- package/dist/tui/cockpit/board.js +10 -0
- package/dist/tui/cockpit/derive.d.ts +35 -0
- package/dist/tui/cockpit/derive.js +152 -10
- package/dist/tui/cockpit/evidence-view.d.ts +2 -0
- package/dist/tui/cockpit/evidence-view.js +42 -12
- package/dist/tui/cockpit/run-cockpit.d.ts +8 -1
- package/dist/tui/cockpit/run-cockpit.js +95 -1
- package/dist/tui/cockpit/run-view.d.ts +37 -0
- package/dist/tui/cockpit/run-view.js +189 -2
- package/dist/tui/ink/fleet-app.d.ts +10 -2
- package/dist/tui/ink/fleet-app.js +33 -15
- package/package.json +2 -2
- package/skills/tickmarkr-overseer/SKILL.md +55 -3
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +29 -2
package/dist/run/daemon.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
1
|
+
import { COMMAND_LEASE_TOKEN_ENV, commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
2
2
|
import { execFileSync, spawn } from "node:child_process";
|
|
3
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
4
4
|
import { shq } from "../adapters/types.js";
|
|
@@ -13,7 +13,7 @@ import { allAdapters, getAdapter, probeAll, readDoctor, rolePools } from "../ada
|
|
|
13
13
|
import { SettledTrailerTracker, addUsage, channelKey, matchesInputBox, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
14
14
|
import { bannerShell, paneDispatchCommand } from "../brand.js";
|
|
15
15
|
import { collateralHits } from "../compile/collateral.js";
|
|
16
|
-
import { DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
16
|
+
import { ExecutionPolicySchema, DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
17
17
|
import { DeliveryReadinessError } from "../drivers/herdr.js";
|
|
18
18
|
import { driverEvidence } from "../drivers/index.js";
|
|
19
19
|
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
@@ -25,8 +25,11 @@ import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHa
|
|
|
25
25
|
import { GATE_NAMES } from "../graph/schema.js";
|
|
26
26
|
import { distFingerprint } from "../cli/commands/version.js";
|
|
27
27
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
28
|
+
import { executionSignal, remainingExecutionMs, withExecutionBudget, withoutExecutionBudget, ExecutionBudgetExceeded } from "./execution-budget.js";
|
|
29
|
+
import { repairSelectionDecision } from "./repair-selection.js";
|
|
30
|
+
import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
28
31
|
import { runEnvironment } from "./environment.js";
|
|
29
|
-
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sh, shGit, SUITE_PARENT_ENV, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
32
|
+
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
30
33
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
31
34
|
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
32
35
|
import { isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
@@ -35,7 +38,7 @@ import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verif
|
|
|
35
38
|
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
36
39
|
import { desiredPanes } from "./reconcile.js";
|
|
37
40
|
import { readTierLiveness, readWatchBoard, supervisionBeatPath } from "./supervision.js";
|
|
38
|
-
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, } from "./stall.js";
|
|
41
|
+
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, reapOwnedProcessGroup, readOwnedProcessGroup, } from "./stall.js";
|
|
39
42
|
// The live set is also the ownership claim shared by task cleanup and termination.
|
|
40
43
|
export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
41
44
|
if (!liveSlots.delete(slot))
|
|
@@ -48,6 +51,83 @@ export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
|
48
51
|
throw error;
|
|
49
52
|
}
|
|
50
53
|
}
|
|
54
|
+
// A transport timeout says nothing about whether a mutation reached the terminal.
|
|
55
|
+
class HeldProbeExhausted extends Error {
|
|
56
|
+
}
|
|
57
|
+
function isTransportTimeout(error) {
|
|
58
|
+
if (!(error instanceof Error))
|
|
59
|
+
return false;
|
|
60
|
+
const e = error;
|
|
61
|
+
return e.timedOut === true || ["ETIMEDOUT", "transport_timeout", "timeout"].includes(e.code ?? "")
|
|
62
|
+
|| /\b(?:timed? out|timeout)\b/i.test(e.reason ?? e.message);
|
|
63
|
+
}
|
|
64
|
+
export function heldWorkerTransport(driver, slot, dispatchId, held, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
|
|
65
|
+
const delays = [250, 500, 1_000];
|
|
66
|
+
const probe = async (operation, call) => {
|
|
67
|
+
for (let retry = 0;; retry++) {
|
|
68
|
+
try {
|
|
69
|
+
return await call();
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
if (!isTransportTimeout(error))
|
|
73
|
+
throw error;
|
|
74
|
+
held({ operation, retry, state: "held", error: String(error), backoffMs: delays[retry] });
|
|
75
|
+
if (retry === delays.length)
|
|
76
|
+
throw new HeldProbeExhausted(`${operation} transport retry budget exhausted`);
|
|
77
|
+
await sleep(delays[retry]);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
return {
|
|
82
|
+
read: (s, n) => probe("read", () => driver.read(s, n)),
|
|
83
|
+
status: (s) => probe("status", () => driver.status(s)),
|
|
84
|
+
waitOutput: (s, p, ms, o) => probe("waitOutput", () => driver.waitOutput(s, p, ms, o)),
|
|
85
|
+
waitAgentStatus: (s, st, ms) => probe("waitAgentStatus", () => driver.waitAgentStatus(s, st, ms)),
|
|
86
|
+
run: async (s, command) => {
|
|
87
|
+
const observer = driver;
|
|
88
|
+
// A second delivery needs a receipt for THIS mutation, not an empty or idle terminal.
|
|
89
|
+
for (let delivery = 0; delivery < 2; delivery++) {
|
|
90
|
+
try {
|
|
91
|
+
await driver.run(s, command);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
if (!isTransportTimeout(error))
|
|
96
|
+
throw error;
|
|
97
|
+
let notAccepted = false;
|
|
98
|
+
for (let retry = 0; retry <= delays.length; retry++) {
|
|
99
|
+
held({ operation: "run", retry, delivery, state: "held", dispatchId, error: String(error), backoffMs: delays[retry] });
|
|
100
|
+
try {
|
|
101
|
+
const receipt = await observer.observeDispatch?.(slot, command, dispatchId);
|
|
102
|
+
if (receipt?.authoritative === true && receipt.slotId === slot.id
|
|
103
|
+
&& receipt.command === command && receipt.dispatchId === dispatchId) {
|
|
104
|
+
if (receipt.outcome === "accepted")
|
|
105
|
+
return;
|
|
106
|
+
if (receipt.outcome === "not-accepted") {
|
|
107
|
+
notAccepted = true;
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const text = await driver.read(slot, PANE_READ_ROWS);
|
|
112
|
+
if (text.split(/\r?\n/).some((line) => line === `TICKMARKR_DISPATCH_${dispatchId}`))
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
catch { /* failed observations prove neither acceptance nor nonacceptance */ }
|
|
116
|
+
if (retry < delays.length)
|
|
117
|
+
await sleep(delays[retry]);
|
|
118
|
+
}
|
|
119
|
+
if (!notAccepted)
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
throw new HeldProbeExhausted("dispatch outcome remained uncertain; no delivery authorized");
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
// A fixed attempt ceiling is independent of the rolling inactivity clock.
|
|
128
|
+
let attemptHardTimeoutMs;
|
|
129
|
+
export function setAttemptHardTimeoutMsForTests(ms) { attemptHardTimeoutMs = ms; }
|
|
130
|
+
export function resetAttemptHardTimeoutMsForTests() { attemptHardTimeoutMs = undefined; }
|
|
51
131
|
// Compatibility exports: gate dispatch can import stall.ts without importing the daemon.
|
|
52
132
|
export { harvestCpuFlatWindowMs, resetHarvestCpuFlatMsForTests, setHarvestCpuFlatMsForTests, workerTreeCpuMs, } from "./stall.js";
|
|
53
133
|
const MODE_RANK = { "staff-led": 0, "risk-based": 1, "partner-led": 2 };
|
|
@@ -261,6 +341,22 @@ export function decisiveReviewRounds(events) {
|
|
|
261
341
|
close(taskId); // Map iteration tolerates deleting the current key
|
|
262
342
|
return events.filter((e) => !spent.has(e));
|
|
263
343
|
}
|
|
344
|
+
/** OBS-1034: the files a byte cap drops from a repair diff — every file whose header starts at or past the cap, plus the one the cut lands inside.
|
|
345
|
+
* Offsets are UTF-8 BYTES (the cap's unit), never UTF-16 code units. `names` is the NUL-delimited `git diff --name-only` list in
|
|
346
|
+
* header order, so quoted headers (tabs, quotes, non-ASCII) still name their file. */
|
|
347
|
+
function repairDiffFilesCut(diff, cap, names) {
|
|
348
|
+
const files = [];
|
|
349
|
+
const header = Buffer.from("\ndiff --git ");
|
|
350
|
+
const starts = diff.subarray(0, header.length - 1).equals(header.subarray(1)) ? [0] : [];
|
|
351
|
+
for (let at = diff.indexOf(header); at !== -1; at = diff.indexOf(header, at + 1))
|
|
352
|
+
starts.push(at + 1);
|
|
353
|
+
starts.forEach((start, i) => {
|
|
354
|
+
const end = starts[i + 1] === undefined ? diff.length : starts[i + 1] - 1;
|
|
355
|
+
if (start >= cap || end > cap)
|
|
356
|
+
files.push(names[i] ?? "(unnamed)");
|
|
357
|
+
});
|
|
358
|
+
return files;
|
|
359
|
+
}
|
|
264
360
|
/** The fix-only contract: the findings verbatim, then the diff content of the work already landed. */
|
|
265
361
|
function repairBrief(findings, diff, baseRef) {
|
|
266
362
|
const repair = { findings, diff };
|
|
@@ -451,7 +547,9 @@ async function captureWorkerStream(journalDir, taskId, attempt, driver, slot, fa
|
|
|
451
547
|
try {
|
|
452
548
|
captured = await driver.read(slot, PANE_READ_ROWS);
|
|
453
549
|
}
|
|
454
|
-
catch {
|
|
550
|
+
catch (error) {
|
|
551
|
+
if (error instanceof HeldProbeExhausted)
|
|
552
|
+
throw error;
|
|
455
553
|
// Earlier successful reads are still a captured stream. Persist them rather than losing the
|
|
456
554
|
// only failure evidence because the pane vanished between its terminal read and this snapshot.
|
|
457
555
|
}
|
|
@@ -781,6 +879,12 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
781
879
|
}
|
|
782
880
|
return tipFailed;
|
|
783
881
|
}
|
|
882
|
+
async function commitsAheadOfRef(base, head, cwd) {
|
|
883
|
+
if (head === base)
|
|
884
|
+
return [];
|
|
885
|
+
const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, cwd);
|
|
886
|
+
return r.code === 0 ? r.stdout.trim().split("\n").filter(Boolean) : [];
|
|
887
|
+
}
|
|
784
888
|
async function commitsAheadOf(base, wt) {
|
|
785
889
|
const head = await gitHead(wt);
|
|
786
890
|
if (head === base)
|
|
@@ -1376,6 +1480,12 @@ function endedRunIdsForReconcile(repoRoot) {
|
|
|
1376
1480
|
return ended;
|
|
1377
1481
|
}
|
|
1378
1482
|
export async function runDaemon(repoRoot, opts = {}) {
|
|
1483
|
+
// OBS-1045: a daemon launched from inside a leased command (a nested runner, an operator's test
|
|
1484
|
+
// shell) inherits its ancestor's lease token through the environment, and every gate it ran would
|
|
1485
|
+
// ride that lease instead of taking its own. Clear it at run-start so the first gate leases itself.
|
|
1486
|
+
if (process.env[COMMAND_LEASE_TOKEN_ENV]) {
|
|
1487
|
+
delete process.env[COMMAND_LEASE_TOKEN_ENV];
|
|
1488
|
+
}
|
|
1379
1489
|
// v1.51 T2 / OBS-89 (v1.60): retired --quality env seam. Mode resolution owns premium routing;
|
|
1380
1490
|
// route() no longer reads the retired env at all, so the old entrypoint scrub is gone with it.
|
|
1381
1491
|
const adapters = opts.adapters ?? allAdapters();
|
|
@@ -1442,6 +1552,21 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1442
1552
|
// > repo > global > default. The resolved cfg carries mode-compiled floors; route() never sees the mode.
|
|
1443
1553
|
const rm = resolveRunMode(repoRoot, { flag: opts.mode, spec: graph.mode, globalDir: opts.globalDir });
|
|
1444
1554
|
const cfg = rm.cfg;
|
|
1555
|
+
if (opts.resume) {
|
|
1556
|
+
const start = Journal.open(repoRoot, runId).read().find((e) => e.event === "run-start");
|
|
1557
|
+
const snapshot = start?.data.effectivePolicy;
|
|
1558
|
+
if (snapshot !== undefined && (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)
|
|
1559
|
+
|| !("config" in snapshot) || !snapshot.config || typeof snapshot.config !== "object" || Array.isArray(snapshot.config))) {
|
|
1560
|
+
throw new Error("cannot restore malformed recorded execution policy");
|
|
1561
|
+
}
|
|
1562
|
+
const recorded = snapshot;
|
|
1563
|
+
// Historical runs did not opt in; current settings cannot grant them a new allowance.
|
|
1564
|
+
cfg.executionPolicy = recorded?.config?.executionPolicy === undefined
|
|
1565
|
+
? undefined : ExecutionPolicySchema.parse(recorded.config.executionPolicy);
|
|
1566
|
+
}
|
|
1567
|
+
if (cfg.executionPolicy && driver.id !== "subprocess") {
|
|
1568
|
+
throw new Error("experimental executionPolicy currently requires the subprocess driver");
|
|
1569
|
+
}
|
|
1445
1570
|
// Resolve both budgets once: worker fan-out divides by dispatch concurrency; verification
|
|
1446
1571
|
// divides by one runner command holding a lease. Explicit operator caps still win.
|
|
1447
1572
|
const concurrency = opts.concurrency ?? cfg.concurrency;
|
|
@@ -1472,14 +1597,58 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1472
1597
|
// opens/closes keep the ledger exact. Termination then closes exactly what is still live — and a
|
|
1473
1598
|
// slot closed once (task-done, quota reroute, gate self-clean) can never be closed twice.
|
|
1474
1599
|
const liveSlots = new Set();
|
|
1475
|
-
|
|
1600
|
+
// A driver handle identifies the physical slot even if a retry returns a new Slot object.
|
|
1601
|
+
const trustAnsweredSlots = new Set();
|
|
1602
|
+
const workerOwners = new Map();
|
|
1603
|
+
const reapReports = new Map();
|
|
1604
|
+
const reapWorker = async (slot) => {
|
|
1605
|
+
const owner = workerOwners.get(slot);
|
|
1606
|
+
if (owner) {
|
|
1607
|
+
const processGroup = readOwnedProcessGroup(owner.groupFile);
|
|
1608
|
+
let survivors;
|
|
1609
|
+
try {
|
|
1610
|
+
const shared = processGroup !== undefined && [...workerOwners].some(([other, otherOwner]) => other !== slot && liveSlots.has(other) && readOwnedProcessGroup(otherOwner.groupFile) === processGroup);
|
|
1611
|
+
if (shared)
|
|
1612
|
+
throw new Error(`worker group ${processGroup} is shared with another live attempt`);
|
|
1613
|
+
survivors = await reapOwnedProcessGroup(processGroup, slot.cwd);
|
|
1614
|
+
}
|
|
1615
|
+
catch (error) {
|
|
1616
|
+
journal.append("worker-process-reaped", owner.taskId, { slot: slot.name, attempt: owner.attempt,
|
|
1617
|
+
processGroup: processGroup ?? null, survivors: null, error: String(error) });
|
|
1618
|
+
throw error;
|
|
1619
|
+
}
|
|
1620
|
+
reapReports.set(slot, { processGroup: processGroup ?? null, survivors });
|
|
1621
|
+
journal.append("worker-process-reaped", owner.taskId, {
|
|
1622
|
+
slot: slot.name, attempt: owner.attempt, processGroup: processGroup ?? null, survivors,
|
|
1623
|
+
});
|
|
1624
|
+
if (survivors && survivors.length > 0)
|
|
1625
|
+
throw new Error(`worker group ${processGroup} survivors: ${survivors.join(", ")}`);
|
|
1626
|
+
}
|
|
1627
|
+
};
|
|
1628
|
+
const closeSlot = (s) => closeLiveSlot(liveSlots, { close: async (slot) => {
|
|
1629
|
+
await reapWorker(slot);
|
|
1630
|
+
await driver.close(slot);
|
|
1631
|
+
} }, s);
|
|
1632
|
+
const budgetSlots = new Map();
|
|
1476
1633
|
const trackedDriver = {
|
|
1477
1634
|
id: driver.id,
|
|
1478
1635
|
interactive: driver.interactive,
|
|
1479
1636
|
...(driver.readSource ? { readSource: driver.readSource } : {}),
|
|
1480
1637
|
...(driver.describe ? { describe: driver.describe.bind(driver) } : {}),
|
|
1481
1638
|
...(driver.focus ? { focus: driver.focus.bind(driver) } : {}),
|
|
1482
|
-
slot: async (cwd, name, o) => {
|
|
1639
|
+
slot: async (cwd, name, o) => {
|
|
1640
|
+
const signal = executionSignal();
|
|
1641
|
+
signal?.throwIfAborted();
|
|
1642
|
+
const s = await driver.slot(cwd, name, o);
|
|
1643
|
+
liveSlots.add(s);
|
|
1644
|
+
if (signal)
|
|
1645
|
+
budgetSlots.get(signal)?.add(s);
|
|
1646
|
+
if (signal?.aborted) {
|
|
1647
|
+
await withoutExecutionBudget(() => closeSlot(s));
|
|
1648
|
+
signal.throwIfAborted();
|
|
1649
|
+
}
|
|
1650
|
+
return s;
|
|
1651
|
+
},
|
|
1483
1652
|
run: (s, cmd) => driver.run(s, cmd),
|
|
1484
1653
|
waitOutput: (s, p, ms, o) => driver.waitOutput(s, p, ms, o),
|
|
1485
1654
|
waitAgentStatus: (s, st, ms) => driver.waitAgentStatus(s, st, ms),
|
|
@@ -1540,6 +1709,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1540
1709
|
activeTipVerify.controller.abort(termination);
|
|
1541
1710
|
await activeTipVerify.settled;
|
|
1542
1711
|
}
|
|
1712
|
+
if (cfg.visibility.keepPanes === "forever") {
|
|
1713
|
+
for (const s of Array.from(liveSlots)) {
|
|
1714
|
+
try {
|
|
1715
|
+
await reapWorker(s);
|
|
1716
|
+
}
|
|
1717
|
+
catch { /* survivor evidence is retained by the reaper */ }
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1543
1720
|
if (cfg.visibility.keepPanes !== "forever") {
|
|
1544
1721
|
for (const s of Array.from(liveSlots)) { // snapshot: failed closes restore membership for the next sweep
|
|
1545
1722
|
try {
|
|
@@ -1563,6 +1740,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1563
1740
|
const demotedReviewers = new Set(journal.read()
|
|
1564
1741
|
.filter((event) => event.event === "review-pool-demotion" && typeof event.data.reviewer === "string")
|
|
1565
1742
|
.map((event) => event.data.reviewer));
|
|
1743
|
+
// OBS-1010: a demotion the JOURNAL carries into this session left the rotation for the run — a
|
|
1744
|
+
// resumed daemon excludes that seat from every review pick (reviewer role only; the worker channel
|
|
1745
|
+
// is untouched — channel-demotion is the worker fold, replayExcludedChannels). In-session demotions
|
|
1746
|
+
// keep their soft ordering above so a still-live seat can be re-tried at the end of the rotation.
|
|
1747
|
+
const replayedReviewerExclusions = opts.resume ? [...demotedReviewers] : [];
|
|
1748
|
+
// OBS-1025 add.2 / OBS-1052 (Leg-2): the run-scoped no-verdict tally run-gates retires a seat from at two
|
|
1749
|
+
// strikes. execTask's badReviewers already excludes a seat after ONE strike within a task, so this map
|
|
1750
|
+
// is what bounds a seat that flakes once per task (or once per session) across the run. Seeded from the
|
|
1751
|
+
// journal's review-no-verdict rows — the same notes noteReviewEvent appends — so a resumed daemon keeps
|
|
1752
|
+
// the count; run-gates appends to it in place, exactly as it does demotedReviewers.
|
|
1753
|
+
const reviewNoVerdicts = new Map();
|
|
1754
|
+
for (const event of journal.read()) {
|
|
1755
|
+
if (event.event !== "review-no-verdict" || event.data.noVerdict !== true || typeof event.data.reviewer !== "string")
|
|
1756
|
+
continue;
|
|
1757
|
+
reviewNoVerdicts.set(event.data.reviewer, [...(reviewNoVerdicts.get(event.data.reviewer) ?? []), String(event.data.cause)]);
|
|
1758
|
+
}
|
|
1566
1759
|
const reviewHistory = journal.read()
|
|
1567
1760
|
.filter((event) => event.event === "gate-result" && event.data.gate === "review" && typeof event.data.reviewer === "string")
|
|
1568
1761
|
.map((event) => event.data.reviewer);
|
|
@@ -1702,11 +1895,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1702
1895
|
};
|
|
1703
1896
|
// Context carries attribution through gates and remote inference; only shell commands acquire.
|
|
1704
1897
|
const commandLeases = new CommandLeases();
|
|
1705
|
-
const withCommandContext = (taskId, run, signal) => runWithCommandLease((_command, execute) => commandLeases.run(async () => {
|
|
1898
|
+
const withCommandContext = (taskId, run, signal = executionSignal()) => runWithCommandLease((_command, execute) => commandLeases.run(async () => {
|
|
1706
1899
|
let lastCount = -1;
|
|
1707
1900
|
const startedAt = Date.now();
|
|
1708
1901
|
for (;;) {
|
|
1709
1902
|
signal?.throwIfAborted();
|
|
1903
|
+
executionSignal()?.throwIfAborted();
|
|
1710
1904
|
const count = await liveSuiteCount(repoRoot);
|
|
1711
1905
|
if (count === 0)
|
|
1712
1906
|
break;
|
|
@@ -1730,7 +1924,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1730
1924
|
const waitForBaseline = async (taskId) => {
|
|
1731
1925
|
if (baselinePending)
|
|
1732
1926
|
journal.append("baseline-wait", taskId, { baseRef });
|
|
1733
|
-
|
|
1927
|
+
const signal = executionSignal();
|
|
1928
|
+
if (!signal) {
|
|
1929
|
+
await baselineCapture;
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
signal.throwIfAborted();
|
|
1933
|
+
// Capture is run-owned; stop this task waiting without cancelling another task's baseline.
|
|
1934
|
+
await new Promise((resolve, reject) => {
|
|
1935
|
+
const abort = () => reject(signal.reason);
|
|
1936
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
1937
|
+
baselineCapture.then(() => { signal.removeEventListener("abort", abort); resolve(); }, (error) => { signal.removeEventListener("abort", abort); reject(error); });
|
|
1938
|
+
});
|
|
1734
1939
|
};
|
|
1735
1940
|
// Phase 46 (RES-01/RES-02): the resume-state map is built ONCE here so execTask closes over it.
|
|
1736
1941
|
// Empty Map on fresh runs — every seed below conditions on resume.get(t.id), never on opts.resume (the
|
|
@@ -1745,7 +1950,23 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1745
1950
|
// every shell this run spawns will divide the machine by. Recorded evidence from another session
|
|
1746
1951
|
// is only reusable against this.
|
|
1747
1952
|
const sessionCapacity = resolvedCapacity();
|
|
1953
|
+
// R41: run-start/run-resume provenance — the protocol + effective lifecycle policy measured at
|
|
1954
|
+
// the repository root when this session began. Gate-result rows do NOT carry this value: each row
|
|
1955
|
+
// carries what run-gates measured for the task worktree the gate ran in (meta.verification), and
|
|
1956
|
+
// the replay guard below compares against a fresh measurement for the recreated worktree.
|
|
1957
|
+
const sessionVerification = verificationProtocol(process.env, repoRoot);
|
|
1748
1958
|
const replayedExclusions = opts.resume ? journal.replayExcludedChannels() : new Set();
|
|
1959
|
+
// Stamp each daemon session, including resumes with changed configuration or CLI overrides.
|
|
1960
|
+
// Keep the loaded-config hash's existing meaning; resolved execution inputs are explicit beside it.
|
|
1961
|
+
// Journal.append applies the normal credential redaction to the readable policy snapshot.
|
|
1962
|
+
const sessionMetadata = {
|
|
1963
|
+
environment: runEnvironment(cfg, channels, health),
|
|
1964
|
+
distFingerprint: distFingerprint(),
|
|
1965
|
+
verification: sessionVerification,
|
|
1966
|
+
effectivePolicy: { config: cfg, concurrency, driver: driver.id, commands },
|
|
1967
|
+
mode: rm.mode.mode,
|
|
1968
|
+
modeSource: rm.source,
|
|
1969
|
+
};
|
|
1749
1970
|
if (opts.resume) {
|
|
1750
1971
|
// v1.53 T5: a superseded run is dead — resuming it beside its successor is the exact
|
|
1751
1972
|
// two-concurrent-runs hazard supersession exists to prevent. Fail closed, naming the successor.
|
|
@@ -1761,7 +1982,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1761
1982
|
// The SAME comparator status uses (engagementComparable); one decision, two consumers. Fail closed:
|
|
1762
1983
|
// no resume path silently accepts a mismatched or unbound journal. --graph-changed is the operator's
|
|
1763
1984
|
// audited release for the stop-amend-resume workflow, journaling a graph-rehash naming both hashes.
|
|
1764
|
-
graph = applyScopeAmendments(graph, journal, true);
|
|
1985
|
+
graph = applyScopeAmendments(graph, journal, true, opts.graphChanged === true);
|
|
1765
1986
|
const loadedHash = graphDefinitionHash(graph);
|
|
1766
1987
|
const cmp = engagementComparable(journal.read(), loadedHash);
|
|
1767
1988
|
if (!cmp.comparable) {
|
|
@@ -1794,9 +2015,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1794
2015
|
graph = setStatus(graph, id, st);
|
|
1795
2016
|
}
|
|
1796
2017
|
journal.append("run-resume", undefined, {
|
|
2018
|
+
...sessionMetadata,
|
|
1797
2019
|
pid: process.pid, // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
1798
2020
|
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
1799
2021
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
2022
|
+
...(opts.graphChanged ? { graphChanged: true } : {}), // OBS-1073: the release is the engagement's
|
|
1800
2023
|
});
|
|
1801
2024
|
await placeBoard();
|
|
1802
2025
|
}
|
|
@@ -1820,11 +2043,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1820
2043
|
// Attach a rejection handler immediately; gate and run-close awaits still propagate the error.
|
|
1821
2044
|
void baselineCapture.catch(() => { baselineFailed = true; });
|
|
1822
2045
|
writeFileSync(join(journal.dir, "graph.json"), readFileSync(join(tickmarkrDir(repoRoot), "graph.json")));
|
|
1823
|
-
// v1.70 T2: environment identity beside the graph/branch identity — running tickmarkr version,
|
|
1824
|
-
// loaded-config hash, and the probed CLI version of each adapter holding a channel in the run,
|
|
1825
|
-
// gathered through the existing probe/config-load paths (no second mechanism).
|
|
1826
|
-
const environment = runEnvironment(cfg, channels, health);
|
|
1827
2046
|
journal.append("run-start", undefined, {
|
|
2047
|
+
...sessionMetadata,
|
|
1828
2048
|
pid: process.pid, baseRef, commands, channels: channels.map(channelKey),
|
|
1829
2049
|
channelsByRole: {
|
|
1830
2050
|
worker: pools.worker.map(channelKey),
|
|
@@ -1834,9 +2054,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1834
2054
|
},
|
|
1835
2055
|
driver: driver.id,
|
|
1836
2056
|
driverEvidence: driverEvidence(cfg, driver, opts.driverOverride),
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
environment, ...(prior ? { supersedes: prior.runId } : {}),
|
|
2057
|
+
branch, graphDefinitionHash: graphDefinitionHash(graph),
|
|
2058
|
+
...(prior ? { supersedes: prior.runId } : {}),
|
|
1840
2059
|
}); // graphDefinitionHash: T3 engagement identity (status+resume share it); pid: v1.13 (VIS-11) liveness; mode/modeSource: v1.51 T2; supersedes: v1.53 T5
|
|
1841
2060
|
runStarted = true;
|
|
1842
2061
|
// v1.53 T5: mark the prior run AFTER this run's run-start exists, so the prior journal never
|
|
@@ -1923,6 +2142,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1923
2142
|
let mergeChain = Promise.resolve();
|
|
1924
2143
|
const mergeSerial = (taskBranch, t, gated) => {
|
|
1925
2144
|
const next = mergeChain.then(() => {
|
|
2145
|
+
executionSignal()?.throwIfAborted();
|
|
1926
2146
|
journal.phaseStart(t.id, "merge");
|
|
1927
2147
|
return mergeTask(intWt, taskBranch, `tickmarkr: merge ${t.id} ${t.title}`, gated);
|
|
1928
2148
|
});
|
|
@@ -1939,17 +2159,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1939
2159
|
];
|
|
1940
2160
|
return `${identity} — release with ${commands.map((command) => `\`${command}\``).join(" or ")}`;
|
|
1941
2161
|
};
|
|
1942
|
-
|
|
2162
|
+
// Lexing never starts in the middle of a word. Resolution is separate so missing hints
|
|
2163
|
+
// remain visible evidence without becoming executable approval advice.
|
|
2164
|
+
const namedPaths = (text) => [...text.matchAll(/(?:^|[\s`'"])((?:[A-Za-z0-9_@.()[\]-]+\/)+[A-Za-z0-9_@.[\]-]+|[A-Za-z0-9_@-]+(?:\.[A-Za-z0-9_-]+)+)(?=$|[\s`'"),:;.!?])/g)]
|
|
1943
2165
|
.map((match) => match[1].replace(/^\.\//, "").replace(/\.$/, ""))
|
|
1944
|
-
.filter((path) => !path.split("/").includes("..")
|
|
1945
|
-
|
|
2166
|
+
.filter((path) => !path.split("/").includes(".."));
|
|
2167
|
+
const diffPaths = async (base, wt) => {
|
|
2168
|
+
const out = await shGit(`git diff --name-only -z ${shq(base)}..HEAD`, wt);
|
|
2169
|
+
return new Set(out.code === 0 ? out.stdout.split("\0").filter(Boolean) : []);
|
|
2170
|
+
};
|
|
2171
|
+
/** The paths a gate diagnostic may name: the gated worktree's tracked tree plus its diff — never the main checkout. */
|
|
2172
|
+
const treeOrDiffPaths = async (base, wt) => {
|
|
2173
|
+
const tree = await shGit("git ls-files -z", wt);
|
|
2174
|
+
return new Set([...(tree.code === 0 ? tree.stdout.split("\0").filter(Boolean) : []), ...await diffPaths(base, wt)]);
|
|
2175
|
+
};
|
|
1946
2176
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
1947
2177
|
// OBS-979: a worker refusal can identify the missing authoring scope even when gates
|
|
1948
2178
|
// subsequently supply the park's disposition. Keep that actionable path on the park itself.
|
|
1949
2179
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
1950
|
-
if (t.files.length > 0 && worker?.data.ok === false && typeof worker.data.summary === "string") {
|
|
2180
|
+
if (!details.approveCommand && t.files.length > 0 && worker?.data.ok === false && typeof worker.data.summary === "string") {
|
|
1951
2181
|
const allowed = filesGlob(t.files);
|
|
1952
|
-
const paths = namedPaths(worker.data.summary).filter((path) => !allowed(path));
|
|
2182
|
+
const paths = namedPaths(worker.data.summary).filter((path) => !allowed(path) && existsSync(join(repoRoot, path)));
|
|
1953
2183
|
if (paths.length)
|
|
1954
2184
|
reason += ` — files[] repair hint: ${[...new Set(paths)].join(", ")}`;
|
|
1955
2185
|
}
|
|
@@ -1979,20 +2209,26 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1979
2209
|
// count. The two callers arrive at it differently (the attempt loop's index already excludes the
|
|
1980
2210
|
// dispatch in flight; a gate replay's rs.attempts already includes it), and the ordinal journaled
|
|
1981
2211
|
// below is derived from it here so both paths number the same dispatch identically.
|
|
1982
|
-
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode) => {
|
|
2212
|
+
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode, changed = new Set()) => {
|
|
1983
2213
|
const scopeRed = results.find((g) => g.gate === "scope" && gateFailed(g));
|
|
1984
2214
|
const verdict = scopeRed?.meta?.collateral;
|
|
1985
2215
|
if (!verdict && t.files.length > 0) {
|
|
1986
2216
|
const allowed = filesGlob(t.files);
|
|
1987
2217
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
1988
|
-
const
|
|
2218
|
+
const requested = worker?.data.ok === false && typeof worker.data.summary === "string"
|
|
1989
2219
|
&& /outside|out.of.scope|allowlist|scope expansion|not (?:in|own)|unowned/i.test(worker.data.summary)
|
|
1990
2220
|
? namedPaths(worker.data.summary).filter((path) => !allowed(path)) : [];
|
|
1991
2221
|
const reds = results.filter(gateFailed);
|
|
1992
2222
|
// A scope verdict already attributes actual out-of-scope EDITS using the collateral map.
|
|
1993
2223
|
// Path-only diagnostics from the other gates name code that needs fixing, not an edit the
|
|
1994
2224
|
// worker made. Do not replace the scope gate's stronger attribution with that inference.
|
|
1995
|
-
const
|
|
2225
|
+
const hints = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details));
|
|
2226
|
+
const unresolved = [...new Set([...requested, ...hints].filter((path) => !changed.has(path)))];
|
|
2227
|
+
if (unresolved.length)
|
|
2228
|
+
journal.append("scope-hint-unresolved", t.id, { paths: unresolved,
|
|
2229
|
+
reason: `scope hint resolves to nothing in the task tree: ${unresolved.join(", ")}`, attempt: attempts + 1 });
|
|
2230
|
+
const refusal = requested.filter((path) => changed.has(path));
|
|
2231
|
+
const paths = hints.filter((path) => changed.has(path));
|
|
1996
2232
|
if (refusal.length || (paths.length > 0 && paths.every((path) => !allowed(path)))) {
|
|
1997
2233
|
const classification = {
|
|
1998
2234
|
gate: refusal.length ? "worker" : reds[0]?.gate,
|
|
@@ -2000,14 +2236,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2000
2236
|
repair: `files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}`,
|
|
2001
2237
|
attempt: attempts + 1, chargeable: false, source: "diagnostic",
|
|
2002
2238
|
};
|
|
2003
|
-
|
|
2239
|
+
const scopeRequest = refusal.length > 0 || reds.some((g) => g.gate === "review");
|
|
2240
|
+
if (scopeRequest)
|
|
2004
2241
|
journal.append("scope-request", t.id, classification);
|
|
2005
2242
|
else
|
|
2006
2243
|
journal.append("scope-authoring", t.id, classification);
|
|
2007
2244
|
// OBS-547 (Leg-2 T3 material): unchargeable ⇒ metered 0, exactly as the predicted path below —
|
|
2008
2245
|
// passing the physical count would write `meteredAttempts: 1` beside `attempts: 0`.
|
|
2009
|
-
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${[...new Set(refusal)].map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2010
|
-
await park(t, `${
|
|
2246
|
+
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${[...new Set(refusal.length ? refusal : paths)].map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2247
|
+
await park(t, `${scopeRequest ? "scope request" : "authoring defect"} — files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}; current files[]: ${t.files.join(", ")}${scopeRequest ? `\n${approveCommand}` : ""}`, scopeRequest ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, scopeRequest ? { paths: [...new Set(refusal.length ? refusal : paths)], approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
2011
2248
|
return true;
|
|
2012
2249
|
}
|
|
2013
2250
|
}
|
|
@@ -2047,7 +2284,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2047
2284
|
approvalSweepCursor = events.length;
|
|
2048
2285
|
if (approvals.length === 0)
|
|
2049
2286
|
return;
|
|
2050
|
-
graph = applyScopeAmendments(graph, journal);
|
|
2287
|
+
graph = applyScopeAmendments(graph, journal, false, opts.graphChanged === true); // OBS-1073: same release as launch
|
|
2051
2288
|
resume = journal.replayResumeState();
|
|
2052
2289
|
satisfiedGates = journal.replaySatisfiedGates();
|
|
2053
2290
|
replayedGateResults = resumeLifecycleOpen
|
|
@@ -2075,6 +2312,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2075
2312
|
};
|
|
2076
2313
|
const execTask = async (t) => {
|
|
2077
2314
|
const startMs = Date.now();
|
|
2315
|
+
let stallSeat;
|
|
2316
|
+
let stallReaps = 0;
|
|
2317
|
+
let reapedWorktreeRef;
|
|
2078
2318
|
const taskTimeoutMinutes = t.timeoutMinutes ?? cfg.taskTimeoutMinutes;
|
|
2079
2319
|
if (t.humanGate && !approved.has(t.id)) {
|
|
2080
2320
|
// GATE-08: the condition is the APPROVAL, never the code path. `!opts.resume` (or any run-phase
|
|
@@ -2151,10 +2391,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2151
2391
|
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
2152
2392
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
2153
2393
|
}
|
|
2154
|
-
else if (!hintsChanged && rs && rs.tried.length) {
|
|
2394
|
+
else if (!hintsChanged && rs && rs.tried.length && !t.routingHints?.pin) {
|
|
2155
2395
|
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
2156
2396
|
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
2157
2397
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
2398
|
+
// OBS-1034: a PIN is exempt — route() already returned it, and a review-upheld repair of the diff
|
|
2399
|
+
// the pin produced belongs on the pin, not on the next seat its own tried[] entry would pick.
|
|
2158
2400
|
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
|
|
2159
2401
|
if (next)
|
|
2160
2402
|
assignment = next;
|
|
@@ -2180,7 +2422,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2180
2422
|
// stays unblocked); inert to replayStatuses (unknown events ignored, pinned at journal.test.ts:70-80).
|
|
2181
2423
|
if (rs)
|
|
2182
2424
|
journal.append("resume-restore", t.id, { attempts: rs.attempts, tried: [...tried], assignment });
|
|
2183
|
-
|
|
2425
|
+
// Keep one live list: recovery retries must see exclusions added by onGate during the round.
|
|
2426
|
+
const badReviewers = [...replayedReviewerExclusions];
|
|
2184
2427
|
const noteReviewEvent = (e) => {
|
|
2185
2428
|
journal.append(e.name, t.id, e.payload);
|
|
2186
2429
|
if (e.name === "review-no-verdict" && typeof e.payload.reviewer === "string") {
|
|
@@ -2190,7 +2433,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2190
2433
|
};
|
|
2191
2434
|
// Keep the measured siblings, including a red judge, while replacing only a missing review.
|
|
2192
2435
|
// Every dispatched seat is excluded for this task, so the loop is bounded by the eligible pool.
|
|
2193
|
-
const runReviewRecovery = async (task, ctx) => {
|
|
2436
|
+
const runReviewRecovery = async (task, ctx, allowSelection = true) => {
|
|
2437
|
+
ctx.authorizeInfraRetry = cfg.executionPolicy?.boundedInfrastructure
|
|
2438
|
+
? (subject, cause) => reserveInfrastructureRetry(journal.read(), task.id, subject, journal.append.bind(journal), cause) : undefined;
|
|
2439
|
+
if (cfg.executionPolicy?.repairSelection && allowSelection) {
|
|
2440
|
+
const decision = repairSelectionDecision(journal.read(), task.id, true);
|
|
2441
|
+
ctx.selectTests = decision.selectTests;
|
|
2442
|
+
ctx.requiredRepairTests = decision.requiredFiles;
|
|
2443
|
+
ctx.selectionReason = decision.reason;
|
|
2444
|
+
}
|
|
2445
|
+
// Resume and ordinary verification share this boundary. Count persisted rounds so a
|
|
2446
|
+
// resumed daemon cannot reuse a previous round's identity within the same attempt.
|
|
2447
|
+
ctx.buildReceiptIdentity = {
|
|
2448
|
+
runId, taskId: task.id, attempt: gateSubject?.attempt ?? 0,
|
|
2449
|
+
gateRound: journal.read().filter((row) => row.taskId === task.id
|
|
2450
|
+
&& row.event === "phase-start" && row.data.phase === "gates").length,
|
|
2451
|
+
};
|
|
2194
2452
|
const round = await runGates(task, ctx);
|
|
2195
2453
|
let review = round.results.find((g) => g.gate === "review");
|
|
2196
2454
|
while (review?.meta?.noVerdict === true) {
|
|
@@ -2263,6 +2521,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2263
2521
|
const unverdicted = (g.meta?.skipped === true && !g.pass) || noVerdictReview;
|
|
2264
2522
|
journal.append("gate-result", t.id, {
|
|
2265
2523
|
gate: g.gate, ...(unverdicted ? {} : { pass: g.pass }), details: g.details,
|
|
2524
|
+
// R41: the protocol + effective lifecycle policy the gate MEASURED under, stamped by run-gates
|
|
2525
|
+
// for the worktree it ran in; absent only for a row that never went through the battery.
|
|
2526
|
+
...(g.meta?.verification !== undefined ? { verification: g.meta.verification } : {}),
|
|
2266
2527
|
...(gateSubject ? { commit: gateSubject.commit, attempt: gateSubject.attempt } : {}),
|
|
2267
2528
|
...(gateSubject?.replayMeasurement ? { replayMeasurement: true } : {}),
|
|
2268
2529
|
...(gateSubject?.replayedFromAttempt !== undefined ? { replayedFromAttempt: gateSubject.replayedFromAttempt } : {}),
|
|
@@ -2271,6 +2532,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2271
2532
|
// be able to tell "the suite found a defect" from "the runner never ran", and the merge
|
|
2272
2533
|
// predicate's reason for refusing has to be legible in the ledger it refused from.
|
|
2273
2534
|
...(g.meta?.infra === true ? { infra: true } : {}),
|
|
2535
|
+
// OBS-1030: a dirty-tree refusal names its litter, its culprit command and the recovery ref ON
|
|
2536
|
+
// THE ROW — the ledger is where an operator looks for the ref before a recheck, not the prose.
|
|
2537
|
+
...(g.meta?.dirtyWorktree === true ? {
|
|
2538
|
+
dirtyWorktree: true, dirtyPaths: g.meta.paths,
|
|
2539
|
+
...(typeof g.meta.culprit === "string" ? { culprit: g.meta.culprit } : {}),
|
|
2540
|
+
...(typeof g.meta.preservedRef === "string" ? { preservedRef: g.meta.preservedRef } : {}),
|
|
2541
|
+
} : {}),
|
|
2542
|
+
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
2543
|
+
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision"]
|
|
2544
|
+
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
2274
2545
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
2275
2546
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
2276
2547
|
// oracle's recorded verdict when any journal-backed reader reconstructs it.
|
|
@@ -2440,7 +2711,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2440
2711
|
// FROM THE JOURNAL, not from process memory: a park, a `resume`, or a daemon restart must not
|
|
2441
2712
|
// hand the selector a clean slate it did not earn, and the journal is the only state that
|
|
2442
2713
|
// survives all three (same read as upheldFeedbackByTask above).
|
|
2443
|
-
let testGateFailed = journal.read().some((e) => e.event === "gate-result" && e.taskId === t.id && e.data.gate === "test" && e.data.pass === false
|
|
2714
|
+
let testGateFailed = journal.read().some((e) => e.event === "gate-result" && e.taskId === t.id && e.data.gate === "test" && e.data.pass === false
|
|
2715
|
+
&& !(cfg.executionPolicy?.boundedInfrastructure && e.data.disposition === "infrastructure"));
|
|
2444
2716
|
let retryMode = "fresh";
|
|
2445
2717
|
let lastContextTokens; // v1.23 reset signal, including stalled/quota attempts
|
|
2446
2718
|
// v1.29: only a gate-failed attempt can seed same-session retry. The next attempt consumes this
|
|
@@ -2550,7 +2822,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2550
2822
|
const taskBase = await integrationHead(intWt);
|
|
2551
2823
|
const taskBranch = `${branch}--${t.id}`;
|
|
2552
2824
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
2553
|
-
|
|
2825
|
+
// OBS-1022: the ref `approve --recheck` validated (commits ahead of the base) is the tree the
|
|
2826
|
+
// battery gates — read from the pending approval row, never re-selected here, so the tree
|
|
2827
|
+
// approve counted and the tree the daemon gates are one and the same, checkout present or not.
|
|
2828
|
+
const preservedRef = recheck
|
|
2829
|
+
? [...journal.read()].reverse().find((e) => e.taskId === t.id && e.event === "task-approved" && e.data.release === RECHECK_RELEASE)?.data.recheckedRef
|
|
2830
|
+
: undefined;
|
|
2831
|
+
if (!existsSync(priorWt) && !preservedRef) {
|
|
2554
2832
|
if (satisfiedGate || recheck)
|
|
2555
2833
|
throw new Error(`${recheck ? "recheck" : `approved gate ${satisfiedGate}`} cannot resume: task worktree is missing`);
|
|
2556
2834
|
// Observed passes are an optimization, never authority: without the task worktree there is no
|
|
@@ -2563,9 +2841,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2563
2841
|
: satisfiedGate
|
|
2564
2842
|
? `approved gate ${satisfiedGate}`
|
|
2565
2843
|
: `recorded gates on ${replayedGates.commit.slice(0, 10)}`;
|
|
2566
|
-
const priorTaskTip = await gitHead(priorWt);
|
|
2567
|
-
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, priorWt);
|
|
2568
|
-
const commitsToCarry = await commitsAheadOf(taskBase, priorWt);
|
|
2844
|
+
const priorTaskTip = preservedRef ? (await shGit(`git rev-parse ${shq(preservedRef)}`, repoRoot)).stdout.trim() : await gitHead(priorWt);
|
|
2845
|
+
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, preservedRef ? repoRoot : priorWt);
|
|
2846
|
+
const commitsToCarry = preservedRef ? await commitsAheadOfRef(taskBase, priorTaskTip, repoRoot) : await commitsAheadOf(taskBase, priorWt);
|
|
2569
2847
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
2570
2848
|
const carriedCommits = await cherryPickCommits(wt, commitsToCarry);
|
|
2571
2849
|
// Reuse is about the tree the gates will actually inspect. The integration tip may have moved
|
|
@@ -2604,16 +2882,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2604
2882
|
}
|
|
2605
2883
|
const workerEvent = [...journal.read()].reverse()
|
|
2606
2884
|
.find((e) => e.event === "worker-result" && e.taskId === t.id);
|
|
2607
|
-
|
|
2885
|
+
// OBS-1022: a task that FAILED before its harvest has no worker-result; a recheck judges the
|
|
2886
|
+
// committed tree on its own evidence, so the missing claim is recorded as exactly that.
|
|
2887
|
+
if (!workerEvent && !recheck)
|
|
2608
2888
|
throw new Error(`${resumeReason} cannot resume: worker result is missing`);
|
|
2609
|
-
const priorResult = {
|
|
2889
|
+
const priorResult = workerEvent ? {
|
|
2610
2890
|
ok: workerEvent.data.ok === true,
|
|
2611
2891
|
summary: typeof workerEvent.data.summary === "string" ? workerEvent.data.summary : "",
|
|
2612
2892
|
deviations: Array.isArray(workerEvent.data.deviations)
|
|
2613
2893
|
? workerEvent.data.deviations.filter((d) => typeof d === "string")
|
|
2614
2894
|
: [],
|
|
2615
2895
|
raw: "",
|
|
2616
|
-
};
|
|
2896
|
+
} : { ok: false, summary: "no worker result recorded (task failed before harvest); recheck gates the committed tree", deviations: [], raw: "" };
|
|
2617
2897
|
const parkedAuthor = recheck
|
|
2618
2898
|
? [...journal.read()].reverse().find((e) => e.event === "task-dispatch" && e.taskId === t.id)?.data.assignment
|
|
2619
2899
|
: undefined;
|
|
@@ -2679,7 +2959,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2679
2959
|
commit: replayedGates.commit, recorded: replayedCapacities, resolved: sessionCapacity,
|
|
2680
2960
|
});
|
|
2681
2961
|
}
|
|
2682
|
-
|
|
2962
|
+
// R41: the same rule for the verification protocol, measured for the RECREATED worktree
|
|
2963
|
+
// the re-run would gate in (its project npmrc included) — but a row with NO stamp is an OLDER
|
|
2964
|
+
// row from a different (pre-stamp) implementation, never "unchanged": it is re-run.
|
|
2965
|
+
const replayVerification = verificationProtocol(process.env, wt);
|
|
2966
|
+
const replayedVerifications = priorEvents
|
|
2967
|
+
.filter((e) => e.event === "gate-result" && e.taskId === t.id && e.data.commit === replayedGates.commit)
|
|
2968
|
+
.map((e) => e.data.verification);
|
|
2969
|
+
const sameProtocol = replayedVerifications.every((recorded) => sameVerification(recorded, replayVerification));
|
|
2970
|
+
if (!sameProtocol) {
|
|
2971
|
+
journal.append("gate-replay-verification-changed", t.id, {
|
|
2972
|
+
commit: replayedGates.commit, recorded: replayedVerifications, resolved: replayVerification,
|
|
2973
|
+
});
|
|
2974
|
+
}
|
|
2975
|
+
let reusable = sameWorld && sameProtocol && (exactCurrentCommit || canonicalCurrentCommit || recreatedLegacyCommit);
|
|
2683
2976
|
const reused = [];
|
|
2684
2977
|
const declaredGates = GATE_NAMES.filter((gate) => t.gates.includes(gate));
|
|
2685
2978
|
for (const gate of declaredGates) {
|
|
@@ -2688,13 +2981,42 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2688
2981
|
else
|
|
2689
2982
|
reused.push(gate);
|
|
2690
2983
|
}
|
|
2984
|
+
// OBS-1049: build outputs are not commits. A replayed build verdict says the tree WAS green;
|
|
2985
|
+
// it says nothing about whether the RECREATED checkout holds what that build produced (dist),
|
|
2986
|
+
// and the host's npm lifecycle (ignore-scripts) may never rebuild it before the suite runs.
|
|
2987
|
+
// So a replayed build re-runs its own command here as provisioning, journaled by name; a red
|
|
2988
|
+
// provisioning is not reused — build (and everything behind it) re-enters the battery as a gate.
|
|
2989
|
+
// Every resume on this path recreates the checkout (recreateTaskWorktree above), so "reused
|
|
2990
|
+
// build" is exactly "build replayed onto a recreated tree".
|
|
2991
|
+
// Journal order is the criterion's order: a GREEN provisioning is recorded AFTER the reuse
|
|
2992
|
+
// rows it belongs to (gate-reused build, then gate-provisioned build); a RED provisioning is
|
|
2993
|
+
// recorded alone, and no reuse row follows it.
|
|
2994
|
+
let provisionedRow;
|
|
2995
|
+
if (reused.includes("build") && commands.build !== undefined) {
|
|
2996
|
+
// Provisioning is a verification command like any gate: it waits for the run's baseline and
|
|
2997
|
+
// takes the command lease (suite admission), never a bare shell beside sibling gates.
|
|
2998
|
+
await waitForBaseline(t.id);
|
|
2999
|
+
const startedAt = Date.now();
|
|
3000
|
+
const provisioned = await withCommandContext(t.id, () => sh(commands.build, wt));
|
|
3001
|
+
provisionedRow = {
|
|
3002
|
+
gate: "build", commit: replayedGates.commit, exitCode: provisioned.code, durationMs: Date.now() - startedAt,
|
|
3003
|
+
};
|
|
3004
|
+
if (provisioned.code !== 0) {
|
|
3005
|
+
reused.splice(reused.indexOf("build"));
|
|
3006
|
+
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
3007
|
+
provisionedRow = undefined;
|
|
3008
|
+
}
|
|
3009
|
+
}
|
|
2691
3010
|
for (const gate of reused) {
|
|
2692
3011
|
journal.append("gate-reused", t.id, { gate, commit: replayedGates.commit });
|
|
2693
3012
|
}
|
|
3013
|
+
if (provisionedRow !== undefined)
|
|
3014
|
+
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
2694
3015
|
remainingGates = declaredGates.slice(reused.length);
|
|
2695
3016
|
}
|
|
2696
3017
|
const resumedTask = { ...t, gates: remainingGates };
|
|
2697
3018
|
gateLoop: while (true) {
|
|
3019
|
+
executionSignal()?.throwIfAborted();
|
|
2698
3020
|
const gated = await gitHead(wt);
|
|
2699
3021
|
gateSubject = {
|
|
2700
3022
|
commit: await gateCommitSubject(taskBase, gated, wt),
|
|
@@ -2713,7 +3035,6 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2713
3035
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
2714
3036
|
collateral: collateral.get(t.id) ?? [],
|
|
2715
3037
|
// a recheck re-verifies a human's release: it never selects tests down, it runs the suite.
|
|
2716
|
-
pipeline: "v185",
|
|
2717
3038
|
via: cfg.visibility.llm === "pane"
|
|
2718
3039
|
? {
|
|
2719
3040
|
driver: trackedDriver,
|
|
@@ -2725,6 +3046,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2725
3046
|
: undefined,
|
|
2726
3047
|
excludeReviewers: badReviewers,
|
|
2727
3048
|
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
3049
|
+
// Leg-2 (OBS-1052): the run-scoped two-strike tally — without it retirement is inert and a
|
|
3050
|
+
// flaking seat is re-asked on every task. (carriedAuthors is held back: see SURGEON-LEG2-REPORT.)
|
|
3051
|
+
reviewNoVerdicts,
|
|
3052
|
+
recheck, // OBS-1055: a recheck discards cached reds — the battery re-measures what the operator questioned
|
|
2728
3053
|
onGate: async (e) => {
|
|
2729
3054
|
if (e.phase === "start") {
|
|
2730
3055
|
notePhaseStart(e);
|
|
@@ -2746,7 +3071,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2746
3071
|
}
|
|
2747
3072
|
});
|
|
2748
3073
|
},
|
|
2749
|
-
}));
|
|
3074
|
+
}, false));
|
|
2750
3075
|
results.forEach(classifySignalOnlyTest);
|
|
2751
3076
|
if (pendingRechecks(journal.read()).has(t.id)) {
|
|
2752
3077
|
journal.append("recheck-battery", t.id, {
|
|
@@ -2764,6 +3089,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2764
3089
|
await park(t, `${infra.gate}: ${infra.details}`, "infra", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2765
3090
|
return;
|
|
2766
3091
|
}
|
|
3092
|
+
const capTrip = results.find(isDiffCapPark);
|
|
3093
|
+
if (capTrip) {
|
|
3094
|
+
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3095
|
+
return;
|
|
3096
|
+
}
|
|
2767
3097
|
// OBS-547: disposition FIRST, and through the same helper the ordinary attempt path uses. A
|
|
2768
3098
|
// crash between a journaled scope red and its classification must not turn a predicted red
|
|
2769
3099
|
// into a charged one just because a resume is what observed it.
|
|
@@ -2776,14 +3106,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2776
3106
|
// restored. Ask the journal which state this is, and take the classified dispatch's own
|
|
2777
3107
|
// assignment back with it.
|
|
2778
3108
|
const classified = journal.classifiedDispatch(t.id);
|
|
2779
|
-
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode))
|
|
3109
|
+
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
2780
3110
|
return;
|
|
2781
3111
|
gateFails++;
|
|
2782
3112
|
// Observed green gates are only measurements. If the resumed suffix is red, preserve that
|
|
2783
3113
|
// result in the journal and return to the ordinary attempt/consult ladder, which rebuilds
|
|
2784
3114
|
// feedback from those rows. Only an operator-authorized gate release parks on a new red.
|
|
2785
|
-
if (!satisfiedGate)
|
|
3115
|
+
if (!satisfiedGate) {
|
|
3116
|
+
// OBS-1055: a recheck red funds a repair of the pin's own work. A pinned task's repair sits
|
|
3117
|
+
// on the pin (OBS-1034's exemption keeps the tried list from excluding it); a pin the fleet
|
|
3118
|
+
// cannot seat parks naming the pin — never the ladder.
|
|
3119
|
+
const pin = recheck ? t.routingHints?.pin : undefined;
|
|
3120
|
+
if (pin) {
|
|
3121
|
+
const seat = channels.find((c) => c.adapter === pin.via && c.model === pin.model && !demotedChannels.has(channelKey(c)));
|
|
3122
|
+
if (!seat) {
|
|
3123
|
+
await park(t, `recheck red: pinned ${pin.via}:${pin.model} is unavailable to host the repair — refusing the ladder`, "gate-fail", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3124
|
+
return;
|
|
3125
|
+
}
|
|
3126
|
+
assignment = { adapter: seat.adapter, model: seat.model, channel: seat.channel, tier: seat.tier };
|
|
3127
|
+
}
|
|
2786
3128
|
break gateLoop;
|
|
3129
|
+
}
|
|
2787
3130
|
await park(t, gateFailApprovalReason(t.id, "post-approval gate failed", results.some((g) => g.gate === "review" && gateFailed(g))), "gate-fail", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2788
3131
|
return;
|
|
2789
3132
|
}
|
|
@@ -2827,6 +3170,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2827
3170
|
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
2828
3171
|
let supersededWorkerSlot;
|
|
2829
3172
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
3173
|
+
executionSignal()?.throwIfAborted();
|
|
2830
3174
|
if (attempt !== providerDeathAttempt) {
|
|
2831
3175
|
providerDeathRequeues = 0;
|
|
2832
3176
|
providerDeathAttempt = attempt;
|
|
@@ -3021,7 +3365,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3021
3365
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
3022
3366
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
3023
3367
|
const recreating = existsSync(priorWt);
|
|
3368
|
+
// Carry the interrupted worker's snapshot, never incidental files created by later gates.
|
|
3369
|
+
const preserved = recreating ? reapedWorktreeRef : undefined;
|
|
3370
|
+
reapedWorktreeRef = undefined;
|
|
3024
3371
|
const commitsToCarry = recreating ? await commitsAheadOf(taskBase, priorWt) : [];
|
|
3372
|
+
if (preserved)
|
|
3373
|
+
commitsToCarry.push(preserved);
|
|
3025
3374
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
3026
3375
|
// OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
|
|
3027
3376
|
// cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
|
|
@@ -3090,14 +3439,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3090
3439
|
if (repairFindings) {
|
|
3091
3440
|
const raw = await shGit(`git diff ${shq(taskBase)}..HEAD`, wt);
|
|
3092
3441
|
const cap = cfg.gates.diffCap ?? DEFAULT_DIFF_CAP; // same fallback the measuring gates use
|
|
3093
|
-
|
|
3094
|
-
|
|
3442
|
+
// OBS-1034: a capped brief says WHAT it cut — the files whose hunks start at or past the cap
|
|
3443
|
+
// (the file the cut lands inside is partial too) — on the row and in the worker's prompt.
|
|
3444
|
+
const rawBytes = Buffer.from(raw.stdout, "utf8"); // the cap is BYTES: measure and cut in UTF-8, not code units
|
|
3445
|
+
const capped = rawBytes.length > cap;
|
|
3446
|
+
const droppedFiles = capped ? repairDiffFilesCut(rawBytes, cap, [...await diffPaths(taskBase, wt)]) : [];
|
|
3447
|
+
const diff = capped
|
|
3448
|
+
? `${new TextDecoder().decode(rawBytes.subarray(0, cap))}\n… diff truncated at gates.diffCap (${cap} bytes)`
|
|
3449
|
+
+ `\nPARTIAL DIFF: the diff above is incomplete; files cut by the cap: ${droppedFiles.join(", ") || "(unknown)"}`
|
|
3095
3450
|
: raw.stdout;
|
|
3096
3451
|
const brief = repairBrief(repairFindings, diff, taskBase);
|
|
3097
3452
|
// anything the live brief holds beyond the journaled findings (a consult's guidance) is kept:
|
|
3098
3453
|
// a repair adds the diff and the fix-only contract, it never subtracts what was already known.
|
|
3099
3454
|
feedback = feedback && !brief.includes(feedback) ? `${brief}\n\n${feedback}` : brief;
|
|
3100
|
-
journal.append("repair-dispatch", t.id, { diffBytes: diff
|
|
3455
|
+
journal.append("repair-dispatch", t.id, { diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
3101
3456
|
}
|
|
3102
3457
|
if (feedback || priorNamed.length > 0) {
|
|
3103
3458
|
feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
|
|
@@ -3159,12 +3514,21 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3159
3514
|
? (hasSeed ? ":" : icmd)
|
|
3160
3515
|
: adapter.invoke(t, wt, assignment, { promptFile }).command;
|
|
3161
3516
|
const dispatchScript = promptFile.replace(/\.md$/, ".sh");
|
|
3517
|
+
// Approval can reset the attempt counter while the old pane remains retained.
|
|
3518
|
+
// Its ownership claim must survive a new engagement reusing the script path.
|
|
3519
|
+
const groupFile = `${dispatchScript}.${nonce}.pgid`;
|
|
3520
|
+
workerOwners.set(slot, { taskId: t.id, attempt, groupFile });
|
|
3162
3521
|
writeFileSync(dispatchScript, [
|
|
3522
|
+
// A driver may launch inside the daemon's group. That group is never worker-owned.
|
|
3523
|
+
`worker_pgid=$(ps -o pgid= -p $$ 2>/dev/null); daemon_pgid=$(ps -o pgid= -p ${process.pid} 2>/dev/null)`,
|
|
3524
|
+
`if [ -n "$worker_pgid" ] && [ -n "$daemon_pgid" ] && [ "$worker_pgid" != "$daemon_pgid" ]; then printf '%s\\n' "$worker_pgid" > ${shq(groupFile)}; fi`,
|
|
3163
3525
|
"export BASH_SILENCE_DEPRECATION_WARNING=1",
|
|
3164
3526
|
bannerShell(),
|
|
3527
|
+
`printf '%s\\n' 'TICKMARKR_DISPATCH_${nonce}'`,
|
|
3165
3528
|
workerCmd,
|
|
3166
3529
|
exitMarkerCmd,
|
|
3167
3530
|
].join("\n"));
|
|
3531
|
+
const workerTransport = heldWorkerTransport(driver, slot, nonce, (data) => journal.append("held-probe", t.id, { ...data, slot: slot.id, attempt }));
|
|
3168
3532
|
// T2 (OBS-264): the liveness triad, shared by BOTH wait loops (a headless worker stalls on
|
|
3169
3533
|
// finished work exactly as a visible one does — and rode the whole window before this). It
|
|
3170
3534
|
// sits ABOVE the fast-kill and the nudge because the population it governs is the opposite
|
|
@@ -3226,7 +3590,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3226
3590
|
}
|
|
3227
3591
|
if (hasSeed || cpuAccountant !== undefined)
|
|
3228
3592
|
return;
|
|
3229
|
-
cpuAccountant = new WorkerTreeCpuAccountant(dispatchScript, wt);
|
|
3593
|
+
cpuAccountant = new WorkerTreeCpuAccountant(dispatchScript, wt, () => readOwnedProcessGroup(groupFile));
|
|
3230
3594
|
await cpuAccountant.start();
|
|
3231
3595
|
};
|
|
3232
3596
|
const readCpuLeg = () => {
|
|
@@ -3320,6 +3684,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3320
3684
|
// window that could not have killed anything and briefed the next worker to fix a stall that
|
|
3321
3685
|
// never happened. Fourth conflation in the taxonomy OBS-53 opened.
|
|
3322
3686
|
let deadChannelKilled = false;
|
|
3687
|
+
let hardTimedOut = false;
|
|
3688
|
+
let quotaBannerKilled = false;
|
|
3689
|
+
let driverProbeFailed = false;
|
|
3690
|
+
let heldLegs = [];
|
|
3691
|
+
const noteDriverUnreadable = (error) => {
|
|
3692
|
+
if (error instanceof HeldProbeExhausted)
|
|
3693
|
+
throw error;
|
|
3694
|
+
driverProbeFailed = true;
|
|
3695
|
+
heldLegs = ["quota:driver-unreadable", "stall:driver-unreadable"];
|
|
3696
|
+
journal.append("contact-unreadable", t.id, { slot: slot.name, attempt, source: "driver", concludes: false,
|
|
3697
|
+
error: error instanceof Error ? error.message : String(error) });
|
|
3698
|
+
};
|
|
3323
3699
|
// T2 review: print mode's "the exit marker appeared". Kept apart from `finished` (the
|
|
3324
3700
|
// trailer) but still needed by the keepPanes decision below, whose contract is about a
|
|
3325
3701
|
// subprocess tree that REACHED its exit marker, not about what the worker claimed.
|
|
@@ -3337,7 +3713,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3337
3713
|
const trailerFrames = interactive && adapter.busyFrameMarkers
|
|
3338
3714
|
? new SettledTrailerTracker(adapter.busyFrameMarkers) : undefined;
|
|
3339
3715
|
const sampleTrailer = (frame) => {
|
|
3340
|
-
const
|
|
3716
|
+
const parsed = adapter.parse(frame, nonce);
|
|
3717
|
+
const trailer = new RegExp(trailerPattern(nonce)).test(frame)
|
|
3718
|
+
&& parsed.summary !== NO_TRAILER_SUMMARY && parsed.summary !== UNPARSEABLE_TRAILER_SUMMARY;
|
|
3341
3719
|
return trailerFrames ? trailerFrames.sample(frame, trailer) : trailer;
|
|
3342
3720
|
};
|
|
3343
3721
|
let seedResult;
|
|
@@ -3351,11 +3729,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3351
3729
|
// DeliveryReadinessError, and that catch continues or returns with no result to read, so a
|
|
3352
3730
|
// latch built from one would start false with the key already sent, and the next modal on
|
|
3353
3731
|
// this slot would get the free Enter this contract exists to prevent.
|
|
3354
|
-
let trustAnswered =
|
|
3732
|
+
let trustAnswered = trustAnsweredSlots.has(slot.id);
|
|
3355
3733
|
const noteSeedTrustAnswered = () => {
|
|
3356
3734
|
if (trustAnswered)
|
|
3357
3735
|
return;
|
|
3358
3736
|
trustAnswered = true;
|
|
3737
|
+
trustAnsweredSlots.add(slot.id);
|
|
3359
3738
|
// Same audit line the loop below writes, so a live run shows the answer wherever it happened.
|
|
3360
3739
|
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id, phase: "seed" });
|
|
3361
3740
|
};
|
|
@@ -3409,7 +3788,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3409
3788
|
// failure rather than allowed to race the trailer wait.
|
|
3410
3789
|
try {
|
|
3411
3790
|
seedResult = await runInteractiveSeed({
|
|
3412
|
-
driver
|
|
3791
|
+
driver: trustAnswered ? {
|
|
3792
|
+
run: workerTransport.run.bind(workerTransport), waitOutput: workerTransport.waitOutput.bind(workerTransport), read: workerTransport.read.bind(workerTransport),
|
|
3793
|
+
} : { ...trackedDriver, ...workerTransport },
|
|
3794
|
+
slot, adapter, assignment, promptFile, taskTimeoutMinutes,
|
|
3413
3795
|
onTrustAnswered: noteSeedTrustAnswered,
|
|
3414
3796
|
});
|
|
3415
3797
|
}
|
|
@@ -3425,7 +3807,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3425
3807
|
}
|
|
3426
3808
|
else {
|
|
3427
3809
|
try {
|
|
3428
|
-
await
|
|
3810
|
+
await workerTransport.run(slot, paneDispatchCommand(dispatchScript));
|
|
3429
3811
|
}
|
|
3430
3812
|
catch (error) {
|
|
3431
3813
|
if (!(error instanceof DeliveryReadinessError))
|
|
@@ -3435,7 +3817,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3435
3817
|
return;
|
|
3436
3818
|
}
|
|
3437
3819
|
await noteLaunched();
|
|
3438
|
-
output = await
|
|
3820
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3439
3821
|
}
|
|
3440
3822
|
// The returning paths report the same fact on the result; both callbacks land on the one
|
|
3441
3823
|
// latch, and the second is a no-op. A seed that answered is never re-answered by the loop.
|
|
@@ -3456,6 +3838,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3456
3838
|
// pageRepeatMs elapses, so an operator who missed the first one is paged again.
|
|
3457
3839
|
let lastPagedStatus;
|
|
3458
3840
|
let lastPagedAt = 0;
|
|
3841
|
+
let pagesSuppressed = 0; // OBS-1006: pageable slices swallowed by the cadence since the last row
|
|
3459
3842
|
// T1 review: worker-status is journaled on CHANGE, not per slice — every journal append
|
|
3460
3843
|
// is narrated to the run's live surface (cli/commands/run.ts) and feeds activity.ts's
|
|
3461
3844
|
// `now:` cell, so a per-slice append wrote a status line per worker per ~30s slice and
|
|
@@ -3481,9 +3864,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3481
3864
|
let cpuHeld = false; // likewise for the CPU leg's stand-down (OBS-548)
|
|
3482
3865
|
let paneReadHeld = false;
|
|
3483
3866
|
let paneStatusHeld = false;
|
|
3484
|
-
|
|
3867
|
+
const hardDeadline = attemptStart + (attemptHardTimeoutMs ?? stallWindowMs * 4);
|
|
3868
|
+
await armCpuLeg(true);
|
|
3869
|
+
while (Date.now() < hardDeadline) {
|
|
3870
|
+
if (termSignal)
|
|
3871
|
+
throw new Error(`terminated by ${termSignal}`);
|
|
3872
|
+
executionSignal()?.throwIfAborted();
|
|
3873
|
+
driverProbeFailed = false;
|
|
3485
3874
|
const sliceStart = Date.now();
|
|
3486
|
-
const remaining = stallWindowMs - (sliceStart - lastProgressAt);
|
|
3875
|
+
const remaining = Math.max(100, Math.min(stallWindowMs - (sliceStart - lastProgressAt), hardDeadline - sliceStart));
|
|
3487
3876
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3488
3877
|
const startupLeft = workerLaunchedAt === undefined
|
|
3489
3878
|
? 0
|
|
@@ -3500,11 +3889,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3500
3889
|
// otherwise cost a minute per pair of CPU samples. At the shipped 5m gate this leaves
|
|
3501
3890
|
// every slice before the gate exactly as long as it already was.
|
|
3502
3891
|
slice = Math.min(slice, harvestSliceMs(sliceStart - lastProgressAt));
|
|
3503
|
-
|
|
3892
|
+
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
3893
|
+
if (await workerTransport.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true }).catch((error) => { noteDriverUnreadable(error); return false; })) {
|
|
3504
3894
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
3505
3895
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
3506
3896
|
// parseable trailer or a digit-suffixed exit marker in the harvest is completion.
|
|
3507
|
-
output = await
|
|
3897
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; }); // TUI transcripts carry chrome — read deeper than print's 500
|
|
3508
3898
|
finished = sampleTrailer(output);
|
|
3509
3899
|
const exit = exitRe.exec(output);
|
|
3510
3900
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3513,6 +3903,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3513
3903
|
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
3514
3904
|
break;
|
|
3515
3905
|
}
|
|
3906
|
+
if (adapter.parse(output, nonce).summary === UNPARSEABLE_TRAILER_SUMMARY)
|
|
3907
|
+
break;
|
|
3516
3908
|
if (trailerFrames && new RegExp(trailerPattern(nonce)).test(output)) {
|
|
3517
3909
|
// A matching busy frame is still a live turn. Preserve the rolling progress budget.
|
|
3518
3910
|
await sampleContext();
|
|
@@ -3526,22 +3918,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3526
3918
|
// rolling taskTimeoutMinutes window as the backstop and name the held probe once.
|
|
3527
3919
|
let paneText;
|
|
3528
3920
|
try {
|
|
3529
|
-
paneText = await
|
|
3921
|
+
paneText = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3530
3922
|
}
|
|
3531
3923
|
catch (error) {
|
|
3532
|
-
|
|
3533
|
-
// sees a pane. The outer attempt finally owns accountant cleanup on that path. Only
|
|
3534
|
-
// an undetectable status makes the read failure relevant to the death detector, and
|
|
3535
|
-
// that genuinely unmeasurable pair fails open to the rolling timeout.
|
|
3536
|
-
let paneUndetectable = true;
|
|
3537
|
-
try {
|
|
3538
|
-
paneUndetectable = await driver.status(slot) === "unknown";
|
|
3539
|
-
}
|
|
3540
|
-
catch {
|
|
3541
|
-
// Two unreadable pane probes are still unmeasurable, never proof of death.
|
|
3542
|
-
}
|
|
3543
|
-
if (!paneUndetectable)
|
|
3544
|
-
throw error;
|
|
3924
|
+
noteDriverUnreadable(error);
|
|
3545
3925
|
if (!paneReadHeld) {
|
|
3546
3926
|
paneReadHeld = true;
|
|
3547
3927
|
noteWorkerLiveness("worker-dead-held", {
|
|
@@ -3549,14 +3929,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3549
3929
|
error: error instanceof Error ? error.message : String(error),
|
|
3550
3930
|
});
|
|
3551
3931
|
}
|
|
3552
|
-
await armCpuLeg(
|
|
3932
|
+
await armCpuLeg(true);
|
|
3553
3933
|
const spent = Date.now() - sliceStart;
|
|
3554
3934
|
if (spent < slice)
|
|
3555
3935
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
3556
3936
|
continue;
|
|
3557
3937
|
}
|
|
3558
3938
|
// waitOutput is only a wake hint; a driver may miss a trailer already painted.
|
|
3559
|
-
if (
|
|
3939
|
+
if (sampleTrailer(paneText)) {
|
|
3560
3940
|
output = paneText;
|
|
3561
3941
|
finished = true;
|
|
3562
3942
|
const exit = exitRe.exec(output);
|
|
@@ -3565,6 +3945,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3565
3945
|
await sampleContext();
|
|
3566
3946
|
break;
|
|
3567
3947
|
}
|
|
3948
|
+
if (adapter.parse(paneText, nonce).summary === UNPARSEABLE_TRAILER_SUMMARY) {
|
|
3949
|
+
output = paneText;
|
|
3950
|
+
break;
|
|
3951
|
+
}
|
|
3952
|
+
if (driverProbeFailed) {
|
|
3953
|
+
const spent = Date.now() - sliceStart;
|
|
3954
|
+
if (spent < slice)
|
|
3955
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(slice - spent, 1_000)));
|
|
3956
|
+
continue;
|
|
3957
|
+
}
|
|
3568
3958
|
if (paneText.length > 0)
|
|
3569
3959
|
everHadOutput = true;
|
|
3570
3960
|
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
@@ -3623,14 +4013,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3623
4013
|
// filtered by identity, never by novelty — a banner already on screen at launch
|
|
3624
4014
|
// classifies exactly like one printed mid-attempt (T1 review: a novelty baseline
|
|
3625
4015
|
// exculpated the launch-throttle case forever).
|
|
3626
|
-
|
|
4016
|
+
const bannerMatch = QUOTA_RE.exec(stallSnapshotBannerRows(paneText));
|
|
4017
|
+
if (bannerMatch)
|
|
3627
4018
|
quotaStreak++;
|
|
3628
4019
|
else
|
|
3629
4020
|
quotaStreak = 0;
|
|
3630
|
-
if (
|
|
4021
|
+
if (bannerMatch && !stallProgress.rowSignalSaturated && !nudgeFailed
|
|
4022
|
+
&& !(driver.nudge && NUDGEABLE_ADAPTERS.has(adapter.id) && (!nudged || nudgeDeadline !== undefined))
|
|
4023
|
+
&& readCpuLeg().state === "flat"
|
|
4024
|
+
&& quotaStreak >= 2 && sliceNow - lastProgressAt >= quotaBannerSilentMs) {
|
|
3631
4025
|
// no `output =` here: the post-loop no-trailer tail re-reads the pane anyway, so an
|
|
3632
4026
|
// assignment would only split the classification read from the verdict read.
|
|
3633
|
-
|
|
4027
|
+
quotaBannerKilled = true;
|
|
4028
|
+
journal.append("quota-banner", t.id, { slot: slot.name, attempt, silentMs: sliceNow - lastProgressAt, matched: bannerMatch[0], excerpt: bannerMatch.input, regex: QUOTA_RE.source });
|
|
3634
4029
|
break;
|
|
3635
4030
|
}
|
|
3636
4031
|
// T1 (OBS-262): the `paged` latch is deleted — status is sampled EVERY slice (and
|
|
@@ -3640,9 +4035,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3640
4035
|
// "unknown"/"working" never page.
|
|
3641
4036
|
let st;
|
|
3642
4037
|
try {
|
|
3643
|
-
st = await
|
|
4038
|
+
st = await workerTransport.status(slot);
|
|
3644
4039
|
}
|
|
3645
4040
|
catch (error) {
|
|
4041
|
+
noteDriverUnreadable(error);
|
|
3646
4042
|
if (!paneStatusHeld) {
|
|
3647
4043
|
paneStatusHeld = true;
|
|
3648
4044
|
noteWorkerLiveness("worker-dead-held", {
|
|
@@ -3650,7 +4046,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3650
4046
|
error: error instanceof Error ? error.message : String(error),
|
|
3651
4047
|
});
|
|
3652
4048
|
}
|
|
3653
|
-
await armCpuLeg(
|
|
4049
|
+
await armCpuLeg(true);
|
|
3654
4050
|
const spent = Date.now() - sliceStart;
|
|
3655
4051
|
if (spent < slice)
|
|
3656
4052
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
@@ -3665,9 +4061,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3665
4061
|
// text matches. tickmarkr created the worktree from the operator's own repo — safe by construction.
|
|
3666
4062
|
if (!trustAnswered && adapter.trustDialog && driver.sendKey) {
|
|
3667
4063
|
try {
|
|
3668
|
-
const paneText = await
|
|
4064
|
+
const paneText = await workerTransport.read(slot, 80);
|
|
3669
4065
|
if (matchesTrustDialog(paneText, adapter.trustDialog)) {
|
|
3670
4066
|
trustAnswered = true;
|
|
4067
|
+
trustAnsweredSlots.add(slot.id);
|
|
3671
4068
|
// v1.25 T1: audit trail for live runs — prove the dialog appeared and was answered.
|
|
3672
4069
|
// Latch + sendKey + no-page continue stay byte-identical; this append is additive only.
|
|
3673
4070
|
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id });
|
|
@@ -3678,7 +4075,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3678
4075
|
continue; // do not page — keep waiting for the trailer
|
|
3679
4076
|
}
|
|
3680
4077
|
}
|
|
3681
|
-
catch {
|
|
4078
|
+
catch (error) {
|
|
4079
|
+
if (error instanceof HeldProbeExhausted)
|
|
4080
|
+
throw error;
|
|
3682
4081
|
/* read/send failed — fall through to page the operator */
|
|
3683
4082
|
}
|
|
3684
4083
|
}
|
|
@@ -3710,7 +4109,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3710
4109
|
let paneAbsent = paneAbsentCandidate;
|
|
3711
4110
|
if (paneAbsent) {
|
|
3712
4111
|
try {
|
|
3713
|
-
const confirmation = await
|
|
4112
|
+
const confirmation = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3714
4113
|
paneAbsent = confirmation.trim().length === 0;
|
|
3715
4114
|
if (!paneAbsent) {
|
|
3716
4115
|
everHadOutput = true;
|
|
@@ -3719,6 +4118,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3719
4118
|
}
|
|
3720
4119
|
}
|
|
3721
4120
|
catch (error) {
|
|
4121
|
+
if (error instanceof HeldProbeExhausted)
|
|
4122
|
+
throw error;
|
|
3722
4123
|
paneAbsent = false;
|
|
3723
4124
|
if (!paneReadHeld) {
|
|
3724
4125
|
paneReadHeld = true;
|
|
@@ -3734,7 +4135,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3734
4135
|
: "unmeasurable";
|
|
3735
4136
|
if (processTree === "empty") {
|
|
3736
4137
|
try {
|
|
3737
|
-
const confirmation = await
|
|
4138
|
+
const confirmation = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3738
4139
|
paneAbsent = confirmation.trim().length === 0;
|
|
3739
4140
|
if (!paneAbsent) {
|
|
3740
4141
|
everHadOutput = true;
|
|
@@ -3743,6 +4144,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3743
4144
|
}
|
|
3744
4145
|
}
|
|
3745
4146
|
catch (error) {
|
|
4147
|
+
if (error instanceof HeldProbeExhausted)
|
|
4148
|
+
throw error;
|
|
3746
4149
|
paneAbsent = false;
|
|
3747
4150
|
if (!paneReadHeld) {
|
|
3748
4151
|
paneReadHeld = true;
|
|
@@ -3793,20 +4196,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3793
4196
|
// lastProgressAt. Either one advancing is output growth.
|
|
3794
4197
|
const lastOutputGrowthAt = Math.max(stallProgress.lastRowGrowthAt ?? 0, lastProgressAt);
|
|
3795
4198
|
// Everything the fast-kill can decide from the CHANNEL, evaluated before the CPU leg is
|
|
3796
|
-
// asked for
|
|
3797
|
-
// slice must not pay for it. The CPU reading is consulted at the kill itself, below.
|
|
4199
|
+
// asked for a conclusion. The shared CPU reading is consulted at the kill below.
|
|
3798
4200
|
const fastKillEligible = !nudgePending && !nudgeFailed
|
|
3799
4201
|
&& sliceNow - lastOutputGrowthAt >= deadChannelFastKillMs
|
|
3800
4202
|
&& worktreeSinceLaunch === "unchanged";
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
// slice from BOTH windows. Arming per-reader let the harvest's own "not silent enough
|
|
3805
|
-
// yet" branch stop the sampler the fast-kill had just started, and a restarted
|
|
3806
|
-
// accountant re-bases its monotonic total — the leg could then never read flat. The
|
|
3807
|
-
// accountant also carries the readers' nudge holds: when neither reader may conclude,
|
|
3808
|
-
// there is no reason to fork its 10 Hz process snapshots through the remaining window.
|
|
3809
|
-
await armCpuLeg(harvestCpuEligible || fastKillEligible);
|
|
4203
|
+
// All silent conclusions share the attempt's continuous CPU ledger. A nudge
|
|
4204
|
+
// hold or a saturated pane must not reset its history before the stall boundary.
|
|
4205
|
+
await armCpuLeg(true);
|
|
3810
4206
|
// T2 (OBS-264): the liveness triad CONCLUDES the wait on finished work — commits ahead
|
|
3811
4207
|
// of the task base (this attempt's own AND any carried forward — see the eligibility
|
|
3812
4208
|
// comment at the dispatch site), a flat worker-tree CPU delta, and >= harvestSilentMs
|
|
@@ -3864,7 +4260,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3864
4260
|
// concluded here". A live CPU delta is a live worker, by construction; an UNMEASURABLE
|
|
3865
4261
|
// reading keeps the fail-open contract the probe already states (undefined is never
|
|
3866
4262
|
// evidence a worker stopped), so on a host whose `ps` cannot be read the fast-kill stands
|
|
3867
|
-
// down and the
|
|
4263
|
+
// down and the fixed attempt ceiling bounds the hold.
|
|
3868
4264
|
if (fastKillEligible) {
|
|
3869
4265
|
const leg = readCpuLeg();
|
|
3870
4266
|
if (leg.state === "flat") {
|
|
@@ -3883,6 +4279,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3883
4279
|
});
|
|
3884
4280
|
}
|
|
3885
4281
|
}
|
|
4282
|
+
const stallCpu = readCpuLeg();
|
|
4283
|
+
heldLegs = [
|
|
4284
|
+
...(stallProgress.rowSignalSaturated ? ["quota:row-signal-saturated", "stall:row-signal-saturated"] : []),
|
|
4285
|
+
...(stallCpu.state !== "flat" ? [`stall:cpu-${stallCpu.state}`] : []),
|
|
4286
|
+
...(nudgeFailed || nudgePending ? ["quota:nudge", "stall:nudge"] : []),
|
|
4287
|
+
...(worktreeSincePriorRead === "unreadable" ? ["stall:worktree-unreadable"] : []),
|
|
4288
|
+
];
|
|
4289
|
+
if (sliceNow - lastProgressAt >= stallWindowMs && heldLegs.length === 0) {
|
|
4290
|
+
timedOut = true;
|
|
4291
|
+
break;
|
|
4292
|
+
}
|
|
3886
4293
|
if (nudgeable && !nudged && sliceNow - lastProgressAt >= nudgeAfterSilentMs) {
|
|
3887
4294
|
nudged = true;
|
|
3888
4295
|
// T1 review: a false return is a driver-delivery outcome (missing pin, readiness
|
|
@@ -3890,16 +4297,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3890
4297
|
// one failure is a flake class, retried once in-slice after a short settle. Only a
|
|
3891
4298
|
// failed RETRY condemns the channel. Both failures happen inside this slice, so the
|
|
3892
4299
|
// latch stays immediate and exactly one failure is journaled per attempt.
|
|
3893
|
-
let delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE);
|
|
4300
|
+
let delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE).catch((error) => { noteDriverUnreadable(error); return false; });
|
|
3894
4301
|
if (!delivered) {
|
|
3895
4302
|
await new Promise((r) => setTimeout(r, NUDGE_REDELIVER_MS));
|
|
3896
|
-
delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE);
|
|
4303
|
+
delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE).catch((error) => { noteDriverUnreadable(error); return false; });
|
|
3897
4304
|
}
|
|
3898
4305
|
if (delivered) {
|
|
3899
4306
|
// absorb the nudge's own echo BEFORE arming the grace timer — post-nudge progress
|
|
3900
4307
|
// is measured against this baseline, not against the echo.
|
|
3901
|
-
const echo = await
|
|
3902
|
-
|
|
4308
|
+
const echo = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4309
|
+
if (echo !== undefined)
|
|
4310
|
+
stallProgress.observe({ paneText: echo, contextTokens });
|
|
3903
4311
|
nudgeDeadline = Date.now() + workerNudgeGraceMs;
|
|
3904
4312
|
journal.append("worker-nudge", t.id, { slot: slot.name, attempt });
|
|
3905
4313
|
}
|
|
@@ -3907,15 +4315,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3907
4315
|
// Delivery failure is journal evidence about the daemon's contact attempt only.
|
|
3908
4316
|
// The fast-kill's worktree clause remains wholly filesystem/git-derived above.
|
|
3909
4317
|
nudgeFailed = true;
|
|
3910
|
-
journal.append("worker-nudge-failed", t.id, { slot: slot.name, attempt });
|
|
4318
|
+
journal.append("worker-nudge-failed", t.id, { slot: slot.name, attempt, cause: "delivery-refused-or-unconfirmed" });
|
|
3911
4319
|
}
|
|
3912
4320
|
}
|
|
3913
|
-
if (nudged && nudgeDeadline !== undefined && sliceNow >= nudgeDeadline && sliceNow - lastProgressAt >= workerNudgeGraceMs) {
|
|
4321
|
+
if (!stallProgress.rowSignalSaturated && readCpuLeg().state === "flat" && nudged && nudgeDeadline !== undefined && sliceNow >= nudgeDeadline && sliceNow - lastProgressAt >= workerNudgeGraceMs) {
|
|
3914
4322
|
// grace spent, still no post-nudge progress: re-harvest once (the trailer may have
|
|
3915
4323
|
// landed between polls), then conclude the wait as a stall NOW — the consult sees the
|
|
3916
4324
|
// un-answered nudge instead of the remainder of the window.
|
|
4325
|
+
const finalPane = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4326
|
+
if (finalPane === undefined)
|
|
4327
|
+
continue;
|
|
3917
4328
|
nudgeDeadline = undefined;
|
|
3918
|
-
output =
|
|
4329
|
+
output = finalPane;
|
|
3919
4330
|
finished = sampleTrailer(output);
|
|
3920
4331
|
const exit = exitRe.exec(output);
|
|
3921
4332
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3925,8 +4336,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3925
4336
|
break;
|
|
3926
4337
|
}
|
|
3927
4338
|
journal.append("worker-nudge-expired", t.id, { slot: slot.name, attempt, graceMs: workerNudgeGraceMs });
|
|
3928
|
-
|
|
3929
|
-
|
|
4339
|
+
timedOut = true;
|
|
4340
|
+
break;
|
|
3930
4341
|
}
|
|
3931
4342
|
// Unlatched page (T1): the page DECISION fires and is journaled every slice the
|
|
3932
4343
|
// operator is the right actor — i.e. the nudge path doesn't own it (non-allowlisted
|
|
@@ -3939,38 +4350,51 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3939
4350
|
// once pageRepeatMs elapses, so a missed first page is not the last one.
|
|
3940
4351
|
const pageable = (st === "blocked" || st === "idle")
|
|
3941
4352
|
&& (!nudgePending || nudgeFailed);
|
|
4353
|
+
// OBS-1006: the row is appended INSIDE the delivery guard — one row per page delivered, never
|
|
4354
|
+
// one per slice — and it carries how many pageable slices were suppressed since the last one.
|
|
3942
4355
|
if (pageable) {
|
|
3943
|
-
journal.append("operator-page", t.id, { slot: slot.name, attempt, status: st });
|
|
3944
4356
|
if (st !== lastPagedStatus || sliceNow - lastPagedAt >= pageRepeatMs) {
|
|
4357
|
+
journal.append("operator-page", t.id, { slot: slot.name, attempt, status: st, suppressed: pagesSuppressed });
|
|
4358
|
+
pagesSuppressed = 0;
|
|
3945
4359
|
lastPagedStatus = st;
|
|
3946
4360
|
lastPagedAt = sliceNow;
|
|
3947
4361
|
const why = st === "blocked" ? "is blocked on a prompt — approve in its pane" : "looks idle without finishing — check its pane";
|
|
3948
4362
|
await driver.notify(`tickmarkr ${runId}: ${slot.name} ${why}`, { tier: "attention" });
|
|
3949
4363
|
}
|
|
4364
|
+
else {
|
|
4365
|
+
pagesSuppressed++;
|
|
4366
|
+
}
|
|
3950
4367
|
}
|
|
3951
4368
|
// a dead pane or a false-positive marker display returns fast — sleep the unspent slice, never hot-spin
|
|
3952
4369
|
const spent = Date.now() - sliceStart;
|
|
3953
4370
|
if (spent < slice)
|
|
3954
4371
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
3955
4372
|
}
|
|
3956
|
-
if (!finished && exitCode === null) {
|
|
4373
|
+
if (!finished && exitCode === null && adapter.parse(output, nonce).summary !== UNPARSEABLE_TRAILER_SUMMARY) {
|
|
3957
4374
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
3958
|
-
|
|
4375
|
+
hardTimedOut = Date.now() >= hardDeadline;
|
|
4376
|
+
timedOut ||= hardTimedOut;
|
|
4377
|
+
if (hardTimedOut)
|
|
4378
|
+
journal.append("worker-hard-timeout", t.id, { slot: slot.name, attempt, heldLegs });
|
|
3959
4379
|
try {
|
|
3960
|
-
output = await
|
|
4380
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3961
4381
|
}
|
|
3962
|
-
catch {
|
|
4382
|
+
catch (error) {
|
|
4383
|
+
if (error instanceof HeldProbeExhausted)
|
|
4384
|
+
throw error;
|
|
3963
4385
|
// The poll loop already recorded the unreadable pane. Retain the last readable bytes
|
|
3964
4386
|
// so this ambiguous path still reaches the ordinary timeout/consult backstop.
|
|
3965
4387
|
}
|
|
3966
|
-
finished =
|
|
4388
|
+
finished = sampleTrailer(output);
|
|
3967
4389
|
const exit = exitRe.exec(output);
|
|
3968
4390
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3969
4391
|
processExited = exit !== null;
|
|
3970
4392
|
}
|
|
3971
4393
|
if (finished && !trailerFrames) {
|
|
3972
|
-
await
|
|
3973
|
-
|
|
4394
|
+
await workerTransport.waitAgentStatus(slot, "idle", 5_000).catch(noteDriverUnreadable);
|
|
4395
|
+
const settled = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; });
|
|
4396
|
+
if (sampleTrailer(settled))
|
|
4397
|
+
output = settled;
|
|
3974
4398
|
}
|
|
3975
4399
|
// T5 / OBS-111: an interactive harvest can race the TUI's final paint. When the pane
|
|
3976
4400
|
// contains the nonce token but the JSON hasn't balanced yet, settle and re-read through
|
|
@@ -3987,7 +4411,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3987
4411
|
if (remaining <= 0)
|
|
3988
4412
|
break;
|
|
3989
4413
|
await new Promise((r) => setTimeout(r, Math.min(settleDelayMs, remaining)));
|
|
3990
|
-
output = await
|
|
4414
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS);
|
|
3991
4415
|
trailerFrames?.sample(output, new RegExp(trailerPattern(nonce)).test(output));
|
|
3992
4416
|
settleParsed = adapter.parse(output, nonce);
|
|
3993
4417
|
settleTries++;
|
|
@@ -4000,7 +4424,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4000
4424
|
}
|
|
4001
4425
|
else {
|
|
4002
4426
|
try {
|
|
4003
|
-
await
|
|
4427
|
+
await workerTransport.run(slot, paneDispatchCommand(dispatchScript));
|
|
4004
4428
|
}
|
|
4005
4429
|
catch (error) {
|
|
4006
4430
|
if (!(error instanceof DeliveryReadinessError))
|
|
@@ -4013,7 +4437,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4013
4437
|
// OBS-54: headless workers have the same output-inactivity budget as visible panes.
|
|
4014
4438
|
// v1.76: same monotonic-progress measure as the interactive site; harvest stays raw.
|
|
4015
4439
|
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
4016
|
-
const initialPane = await
|
|
4440
|
+
const initialPane = await workerTransport.read(slot, 500);
|
|
4017
4441
|
startupFailure = startupFailureInWindow(initialPane, workerLaunchedAt);
|
|
4018
4442
|
let everHadOutput = initialPane.length > 0;
|
|
4019
4443
|
const stallProgress = new StallProgressTracker();
|
|
@@ -4028,6 +4452,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4028
4452
|
// there is the trailer regex; the exit marker only sets exitCode), and the cause taxonomy
|
|
4029
4453
|
// already names this shape "clean-exit-no-trailer" — unreachable in print mode until now.
|
|
4030
4454
|
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
4455
|
+
executionSignal()?.throwIfAborted();
|
|
4031
4456
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
4032
4457
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
4033
4458
|
const startupLeft = workerLaunchedAt === undefined
|
|
@@ -4042,11 +4467,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4042
4467
|
}
|
|
4043
4468
|
// T2 (OBS-264): same probe cadence the interactive loop uses — see harvestSliceMs.
|
|
4044
4469
|
slice = Math.min(slice, harvestSliceMs(Date.now() - lastProgressAt));
|
|
4045
|
-
|
|
4470
|
+
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
4471
|
+
if (await workerTransport.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
4046
4472
|
processExited = true;
|
|
4047
4473
|
break;
|
|
4048
4474
|
}
|
|
4049
|
-
const paneText = await
|
|
4475
|
+
const paneText = await workerTransport.read(slot, 500);
|
|
4050
4476
|
if (paneText.length > 0)
|
|
4051
4477
|
everHadOutput = true;
|
|
4052
4478
|
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
@@ -4071,7 +4497,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4071
4497
|
if (await harvestConcludes(Date.now() - lastProgressAt))
|
|
4072
4498
|
break;
|
|
4073
4499
|
}
|
|
4074
|
-
output = await
|
|
4500
|
+
output = await workerTransport.read(slot, 500);
|
|
4075
4501
|
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
4076
4502
|
// Completion is the trailer, exactly as in the interactive loop. A process that exited
|
|
4077
4503
|
// without one is finished:false with a non-null exitCode — the harvest synthesis then owns
|
|
@@ -4089,13 +4515,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4089
4515
|
// interactive attempts to the same exit marker before close and the post-hoc usage disk read
|
|
4090
4516
|
// so a writer never races the reader (real CLIs can flush usage asynchronously after the trailer).
|
|
4091
4517
|
if (interactive && finished && !exitRe.test(output)) {
|
|
4092
|
-
await
|
|
4518
|
+
await workerTransport.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true }).catch(noteDriverUnreadable);
|
|
4093
4519
|
}
|
|
4094
4520
|
// Free availability reroutes deliberately reuse the charged-attempt ordinal. Stream artifacts
|
|
4095
4521
|
// cannot: every dispatch owns bytes that must remain independently recoverable, so derive the
|
|
4096
4522
|
// append-only dispatch ordinal from the journal instead of overwriting an earlier a<n>.out.
|
|
4097
4523
|
const streamAttempt = journal.read().filter((event) => event.event === "task-dispatch" && event.taskId === t.id).length - 1;
|
|
4098
|
-
const capturedStream = await captureWorkerStream(journal.dir, t.id, streamAttempt,
|
|
4524
|
+
const capturedStream = await captureWorkerStream(journal.dir, t.id, streamAttempt, { ...trackedDriver, ...workerTransport }, slot, output);
|
|
4099
4525
|
// An interactive settle already selected the bounded parse it is allowed to use. Every other
|
|
4100
4526
|
// path parses the common final stream snapshot — notably envelope adapters whose trailer is
|
|
4101
4527
|
// encoded and therefore invisible to the raw wait regex. Keep `output` bounded: every text
|
|
@@ -4126,8 +4552,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4126
4552
|
try {
|
|
4127
4553
|
await closeSlot(slot);
|
|
4128
4554
|
if (!workerFinished) {
|
|
4555
|
+
const ref = await preserveWorktree(wt);
|
|
4556
|
+
if (ref) {
|
|
4557
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
4558
|
+
reapedWorktreeRef = ref;
|
|
4559
|
+
}
|
|
4129
4560
|
journal.append("worker-reaped-before-harvest", t.id, {
|
|
4130
|
-
slot: slot.name, attempt, cause: workerCause ?? "stall-timeout",
|
|
4561
|
+
slot: slot.name, attempt, cause: hardTimedOut ? "hard-timeout" : workerCause ?? "stall-timeout",
|
|
4562
|
+
...reapReports.get(slot),
|
|
4563
|
+
...(hardTimedOut ? { heldLegs } : {}),
|
|
4131
4564
|
...(startupEvidence ? { evidence: startupEvidence } : {}),
|
|
4132
4565
|
});
|
|
4133
4566
|
}
|
|
@@ -4173,8 +4606,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4173
4606
|
// (provider outage, quota banner, dead CLI) must still route; the harvest synthesis only
|
|
4174
4607
|
// decides whether THIS attempt's worktree goes to gates, and when routing wins instead, the
|
|
4175
4608
|
// commits survive via the existing commitsToCarry/cherryPickCommits carry-forward.
|
|
4176
|
-
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4609
|
+
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
4177
4610
|
return;
|
|
4611
|
+
if (!workerFinished && !processExited && (timedOut || deadChannelKilled || quotaBannerKilled)) {
|
|
4612
|
+
const seat = channelKey(assignment);
|
|
4613
|
+
stallReaps = stallSeat === seat ? stallReaps + 1 : 1;
|
|
4614
|
+
stallSeat = seat;
|
|
4615
|
+
if (stallReaps >= 2) {
|
|
4616
|
+
const ref = await preserveWorktree(wt);
|
|
4617
|
+
if (ref)
|
|
4618
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
4619
|
+
await park(t, `two consecutive stall reaps without a gate on seat ${seat}`, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { seat, stallReaps });
|
|
4620
|
+
return;
|
|
4621
|
+
}
|
|
4622
|
+
}
|
|
4623
|
+
else {
|
|
4624
|
+
stallReaps = 0;
|
|
4625
|
+
stallSeat = undefined;
|
|
4626
|
+
}
|
|
4178
4627
|
const preHarvestResult = result;
|
|
4179
4628
|
// T2 (OBS-264): recognize committed no-trailer work BEFORE any no-trailer streak, provider,
|
|
4180
4629
|
// quota or dead-channel routing. Gates never trusted the trailer, so this successful synthesis
|
|
@@ -4366,6 +4815,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4366
4815
|
}
|
|
4367
4816
|
// Provisioning can mutate the shared dependency tree the pristine capture is still using.
|
|
4368
4817
|
await waitForBaseline(t.id);
|
|
4818
|
+
stallReaps = 0;
|
|
4819
|
+
stallSeat = undefined;
|
|
4369
4820
|
graph = setStatus(graph, t.id, "gated");
|
|
4370
4821
|
saveGraph(repoRoot, graph);
|
|
4371
4822
|
// OBS-47: re-assert the node_modules link BEFORE gates run on any attempt. A worker may have
|
|
@@ -4430,6 +4881,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4430
4881
|
let results = [];
|
|
4431
4882
|
let commits = [];
|
|
4432
4883
|
gateLoop: while (true) {
|
|
4884
|
+
executionSignal()?.throwIfAborted();
|
|
4433
4885
|
const gated = await gitHead(wt);
|
|
4434
4886
|
gateSubject = { commit: await gateCommitSubject(taskBase, gated, wt), attempt };
|
|
4435
4887
|
await trackedDriver.project?.(t.id, "in-review");
|
|
@@ -4469,7 +4921,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4469
4921
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
4470
4922
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4471
4923
|
collateral: collateral.get(t.id) ?? [],
|
|
4472
|
-
|
|
4924
|
+
selectTests: !testGateFailed,
|
|
4473
4925
|
via: cfg.visibility.llm === "pane"
|
|
4474
4926
|
? {
|
|
4475
4927
|
driver: trackedDriver,
|
|
@@ -4488,13 +4940,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4488
4940
|
: undefined,
|
|
4489
4941
|
excludeReviewers: badReviewers,
|
|
4490
4942
|
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
4943
|
+
reviewNoVerdicts, // Leg-2 (OBS-1052): see the recheck site above
|
|
4491
4944
|
onGate,
|
|
4492
4945
|
})));
|
|
4493
4946
|
}
|
|
4494
4947
|
results.forEach(classifySignalOnlyTest);
|
|
4495
4948
|
graph = addEvidence(graph, t.id, { commits, gateResults: results, artifacts: [promptFile] });
|
|
4496
4949
|
saveGraph(repoRoot, graph);
|
|
4497
|
-
if (results.some((g) => g.gate === "test" && !g.pass
|
|
4950
|
+
if (results.some((g) => g.gate === "test" && !g.pass
|
|
4951
|
+
&& !(cfg.executionPolicy?.boundedInfrastructure && failureDisposition(g) === "infrastructure")))
|
|
4498
4952
|
testGateFailed = true;
|
|
4499
4953
|
if (results.every(gateSatisfied)) {
|
|
4500
4954
|
// T6: every gate — the review included — is satisfied on this commit, so the failure brief
|
|
@@ -4506,6 +4960,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4506
4960
|
// gate-fail brief is, so prior-RUN evidence (retired by its own rule, not by this reviewer)
|
|
4507
4961
|
// survives and only this run's settled findings go.
|
|
4508
4962
|
feedback = withCarriedEvidence("");
|
|
4963
|
+
executionSignal()?.throwIfAborted();
|
|
4509
4964
|
const m = await mergeSerial(taskBranch, t, gated);
|
|
4510
4965
|
if (m.tipMoved) {
|
|
4511
4966
|
journal.append("tip-moved", t.id, m.tipMoved);
|
|
@@ -4555,12 +5010,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4555
5010
|
// flowing through the chargeable quality path below.
|
|
4556
5011
|
const infraFailure = results.find((g) => gateFailed(g) && g.meta?.infra === true);
|
|
4557
5012
|
if (infraFailure) {
|
|
4558
|
-
await park(t, `${infraFailure.gate}: ${infraFailure.details}`, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5013
|
+
await park(t, `${infraFailure.gate}: ${infraFailure.details}${infraFailure.meta?.recoveryBlocked ? ` — ${infraFailure.meta.recoveryBlocked}` : ""}`, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5014
|
+
return;
|
|
5015
|
+
}
|
|
5016
|
+
// OBS-1007: a cap trip is a typed park of its own — the diff cannot shrink by retrying and the
|
|
5017
|
+
// only honest verb is a recheck under a raised cap. Decided BEFORE the authoring classifier, which
|
|
5018
|
+
// used to lift `gates.diffCap` out of the cap prose as a files[] hint.
|
|
5019
|
+
const capTrip = results.find(isDiffCapPark);
|
|
5020
|
+
if (capTrip) {
|
|
5021
|
+
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4559
5022
|
return;
|
|
4560
5023
|
}
|
|
4561
5024
|
// OBS-547: who pays for this red is decided by the run's collateral prediction (see
|
|
4562
5025
|
// dispositionScopeRed) — an authoring defect parks unchargeable before any accounting below.
|
|
4563
|
-
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
5026
|
+
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
4564
5027
|
return;
|
|
4565
5028
|
gateFails++; // this attempt's gates failed — the one place quality degradation is verified (never inferred from attempts)
|
|
4566
5029
|
// v1.53 T3: prefer the CLI's own session id captured from this attempt's output (kimi's resume
|
|
@@ -4720,6 +5183,39 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4720
5183
|
return;
|
|
4721
5184
|
}
|
|
4722
5185
|
};
|
|
5186
|
+
const executeWithPolicy = async (t) => {
|
|
5187
|
+
const policy = cfg.executionPolicy;
|
|
5188
|
+
if (!policy || (t.humanGate && !approved.has(t.id)))
|
|
5189
|
+
return execTask(t);
|
|
5190
|
+
await withExecutionBudget({ limitMs: policy.taskExecutionLimitMs, taskId: t.id,
|
|
5191
|
+
readEvents: () => journal.read(), append: journal.append.bind(journal) }, async () => {
|
|
5192
|
+
const signal = executionSignal();
|
|
5193
|
+
const slots = new Set();
|
|
5194
|
+
budgetSlots.set(signal, slots);
|
|
5195
|
+
let closing;
|
|
5196
|
+
const closeOwned = () => withoutExecutionBudget(async () => {
|
|
5197
|
+
for (const slot of slots)
|
|
5198
|
+
await closeSlot(slot);
|
|
5199
|
+
});
|
|
5200
|
+
const stop = () => {
|
|
5201
|
+
closing = closeOwned();
|
|
5202
|
+
// Awaited again below; prevent an unhandled rejection while the worker unwinds.
|
|
5203
|
+
void closing.catch(() => { });
|
|
5204
|
+
};
|
|
5205
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
5206
|
+
try {
|
|
5207
|
+
await execTask(t);
|
|
5208
|
+
}
|
|
5209
|
+
finally {
|
|
5210
|
+
signal.removeEventListener("abort", stop);
|
|
5211
|
+
if (signal.aborted) {
|
|
5212
|
+
await closing;
|
|
5213
|
+
await closeOwned();
|
|
5214
|
+
}
|
|
5215
|
+
budgetSlots.delete(signal);
|
|
5216
|
+
}
|
|
5217
|
+
});
|
|
5218
|
+
};
|
|
4723
5219
|
taskLoopStarted = true;
|
|
4724
5220
|
const inflight = new Map();
|
|
4725
5221
|
// A settled worker can release a dependency or an approval in the same poll tick.
|
|
@@ -4752,8 +5248,47 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4752
5248
|
.filter((t) => !inflight.has(t.id))
|
|
4753
5249
|
.slice(0, Math.max(0, concurrency - inflight.size));
|
|
4754
5250
|
for (const t of ready) {
|
|
4755
|
-
const p =
|
|
5251
|
+
const p = executeWithPolicy(t)
|
|
4756
5252
|
.catch(async (err) => {
|
|
5253
|
+
const cleanupErrors = [];
|
|
5254
|
+
for (const [slot, owner] of workerOwners) {
|
|
5255
|
+
if (owner.taskId !== t.id || !liveSlots.has(slot))
|
|
5256
|
+
continue;
|
|
5257
|
+
try {
|
|
5258
|
+
await withoutExecutionBudget(() => closeSlot(slot));
|
|
5259
|
+
}
|
|
5260
|
+
catch (error) {
|
|
5261
|
+
// Cleanup evidence must not replace the task's original failure or abort siblings.
|
|
5262
|
+
cleanupErrors.push({ slot: slot.name, attempt: owner.attempt, error: String(error) });
|
|
5263
|
+
}
|
|
5264
|
+
}
|
|
5265
|
+
const cleanupEvidence = cleanupErrors.length ? { cleanupErrors } : {};
|
|
5266
|
+
if (err instanceof HeldProbeExhausted) {
|
|
5267
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
5268
|
+
let ref = await withoutExecutionBudget(() => preserveWorktree(wt));
|
|
5269
|
+
if (!ref) {
|
|
5270
|
+
const head = await gitHead(wt);
|
|
5271
|
+
ref = `refs/tickmarkr/preserved/${head}`;
|
|
5272
|
+
const saved = await shGit(`git update-ref ${shq(ref)} ${shq(head)}`, wt);
|
|
5273
|
+
if (saved.code !== 0)
|
|
5274
|
+
throw new Error(`could not preserve ${head}: ${saved.stderr}`);
|
|
5275
|
+
}
|
|
5276
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
5277
|
+
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: "transport-uncertain", ref, ...cleanupEvidence });
|
|
5278
|
+
return;
|
|
5279
|
+
}
|
|
5280
|
+
if (err instanceof ExecutionBudgetExceeded) {
|
|
5281
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
5282
|
+
const ref = await preserveWorktree(wt);
|
|
5283
|
+
if (ref)
|
|
5284
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
5285
|
+
const dispatch = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "task-dispatch");
|
|
5286
|
+
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: "execution-budget-exhausted", limitMs: cfg.executionPolicy.taskExecutionLimitMs,
|
|
5287
|
+
...cleanupEvidence,
|
|
5288
|
+
...(Number.isInteger(dispatch?.data.attempt) ? { attempt: dispatch.data.attempt } : {}),
|
|
5289
|
+
...(dispatch?.data.assignment ? { assignment: dispatch.data.assignment } : {}) });
|
|
5290
|
+
return;
|
|
5291
|
+
}
|
|
4757
5292
|
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
4758
5293
|
const dispatch = [...taskEvents].reverse().find((e) => e.event === "task-dispatch");
|
|
4759
5294
|
// OBS-206: shared rule with `resume --retry-failed` — see classifyTaskFailure.
|
|
@@ -4761,7 +5296,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4761
5296
|
const attempts = dispatch && Number.isInteger(dispatch.data.attempt) ? dispatch.data.attempt : 0;
|
|
4762
5297
|
graph = setStatus(graph, t.id, "failed");
|
|
4763
5298
|
saveGraph(repoRoot, graph);
|
|
4764
|
-
journal.append("task-failed", t.id, { error: String(err), kind, attempts });
|
|
5299
|
+
journal.append("task-failed", t.id, { error: String(err), kind, attempts, ...cleanupEvidence });
|
|
4765
5300
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
4766
5301
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
4767
5302
|
})
|