tickmarkr 2.6.1 → 2.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -3
- package/dist/adapters/catalog-remote.js +89 -47
- package/dist/adapters/claude-code.js +9 -6
- package/dist/adapters/codex.js +7 -4
- package/dist/adapters/prompt.d.ts +1 -0
- package/dist/adapters/prompt.js +14 -6
- package/dist/adapters/registry.js +3 -3
- package/dist/adapters/types.d.ts +12 -4
- package/dist/adapters/types.js +6 -0
- package/dist/cli/commands/approve.d.ts +11 -4
- package/dist/cli/commands/approve.js +82 -27
- package/dist/cli/commands/compile.js +13 -3
- package/dist/cli/commands/doctor.d.ts +8 -2
- package/dist/cli/commands/doctor.js +11 -3
- package/dist/cli/commands/fleet.js +87 -11
- package/dist/cli/commands/plan.js +13 -8
- package/dist/cli/commands/report.d.ts +2 -1
- package/dist/cli/commands/report.js +74 -8
- package/dist/cli/commands/resume.js +4 -2
- package/dist/cli/commands/status.js +43 -20
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.js +9 -2
- package/dist/compile/native.js +7 -0
- package/dist/config/config.d.ts +35 -2
- package/dist/config/config.js +86 -10
- package/dist/config/fleet-overlay.d.ts +13 -2
- package/dist/config/fleet-overlay.js +60 -0
- package/dist/drivers/herdr.d.ts +12 -0
- package/dist/drivers/herdr.js +51 -0
- package/dist/drivers/orca.d.ts +35 -2
- package/dist/drivers/orca.js +222 -67
- package/dist/drivers/types.d.ts +2 -0
- package/dist/drivers/types.js +2 -2
- package/dist/eval/canary.d.ts +2 -1
- package/dist/eval/canary.js +2 -2
- package/dist/eval/dispatch.js +1 -0
- package/dist/gates/acceptance.d.ts +9 -1
- package/dist/gates/acceptance.js +31 -4
- package/dist/gates/baseline.d.ts +32 -2
- package/dist/gates/baseline.js +111 -24
- package/dist/gates/cache.d.ts +8 -0
- package/dist/gates/cache.js +12 -2
- package/dist/gates/llm.d.ts +11 -4
- package/dist/gates/llm.js +40 -21
- package/dist/gates/review.d.ts +14 -1
- package/dist/gates/review.js +160 -34
- package/dist/gates/run-gates.d.ts +56 -4
- package/dist/gates/run-gates.js +358 -58
- package/dist/gates/test-manifest.d.ts +45 -1
- package/dist/gates/test-manifest.js +78 -12
- package/dist/graph/schema.d.ts +2 -0
- package/dist/graph/schema.js +2 -0
- package/dist/plan/scope.js +2 -2
- package/dist/route/preference.d.ts +20 -2
- package/dist/route/preference.js +48 -13
- package/dist/route/router.d.ts +12 -1
- package/dist/route/router.js +56 -24
- package/dist/run/consult.d.ts +15 -1
- package/dist/run/consult.js +18 -7
- package/dist/run/daemon.d.ts +38 -2
- package/dist/run/daemon.js +895 -192
- package/dist/run/git.d.ts +8 -0
- package/dist/run/git.js +14 -0
- package/dist/run/interactive-seed.d.ts +4 -0
- package/dist/run/interactive-seed.js +35 -9
- package/dist/run/journal.d.ts +152 -3
- package/dist/run/journal.js +551 -50
- package/dist/run/lease.d.ts +13 -0
- package/dist/run/lease.js +45 -0
- package/dist/run/merge.d.ts +3 -1
- package/dist/run/merge.js +3 -2
- package/dist/run/operator-summary.d.ts +3 -0
- package/dist/run/operator-summary.js +3 -1
- package/dist/run/protocol.d.ts +46 -1
- package/dist/run/protocol.js +14 -2
- package/dist/run/receipt-resolver.d.ts +22 -0
- package/dist/run/receipt-resolver.js +40 -1
- package/dist/run/repair-selection.d.ts +11 -1
- package/dist/run/repair-selection.js +17 -9
- package/dist/run/supervision.d.ts +7 -1
- package/dist/run/supervision.js +5 -2
- package/dist/run/wall-budget.d.ts +48 -0
- package/dist/run/wall-budget.js +280 -0
- package/dist/tui/cockpit/board.js +3 -3
- package/dist/tui/cockpit/decision-actions.d.ts +8 -5
- package/dist/tui/cockpit/decision-actions.js +55 -32
- package/dist/tui/cockpit/derive.js +13 -2
- package/dist/tui/cockpit/live-runtime.d.ts +10 -0
- package/dist/tui/cockpit/live-runtime.js +50 -3
- package/dist/tui/cockpit/run-cockpit.d.ts +3 -0
- package/dist/tui/cockpit/run-cockpit.js +27 -2
- package/dist/tui/cockpit/run-view.d.ts +9 -2
- package/dist/tui/cockpit/run-view.js +66 -9
- package/dist/tui/cockpit/setup-cockpit.d.ts +6 -0
- package/dist/tui/cockpit/setup-cockpit.js +10 -3
- package/dist/tui/ink/fleet-app.d.ts +15 -3
- package/dist/tui/ink/fleet-app.js +91 -22
- package/package.json +3 -1
- package/schema/config.schema.json +825 -0
- package/skills/tickmarkr-loop/SKILL.md +15 -3
- package/skills/tickmarkr-overseer/SKILL.md +42 -0
- package/skills/tickmarkr-overseer/scripts/classify-vitest-log.sh +91 -0
- package/skills/tickmarkr-overseer/scripts/context-statusline.sh +81 -0
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +36 -34
- package/skills/tickmarkr-overseer/scripts/watch-journal.sh +6 -4
package/dist/run/daemon.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HOST_PROBE_SAMPLE_MS, HOST_PROBE_SAMPLES, HostDegradedError, hostDegraded, observeHost } from "./host-health.js";
|
|
2
|
-
import { VITEST_CACHE_ENV, worktreeVitestCache } from "../gates/test-manifest.js";
|
|
2
|
+
import { runWithInterruptionSink, VITEST_CACHE_ENV, worktreeVitestCache } from "../gates/test-manifest.js";
|
|
3
3
|
import { COMMAND_LEASE_TOKEN_ENV, commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
4
4
|
import { execFileSync, spawn } from "node:child_process";
|
|
5
5
|
import { createHash, randomBytes } from "node:crypto";
|
|
@@ -10,7 +10,7 @@ import { tmpdir } from "node:os";
|
|
|
10
10
|
import { basename, dirname, isAbsolute, join, posix, relative, resolve, sep } from "node:path";
|
|
11
11
|
import { fileURLToPath } from "node:url";
|
|
12
12
|
import { stringify } from "yaml";
|
|
13
|
-
import { classifyDeadChannel, classifyTransientCapacity,
|
|
13
|
+
import { BOOTSTRAP_FAILURE_RE, classifyDeadChannel, classifyTransientCapacity, trailerPattern, writePrompt } from "../adapters/prompt.js";
|
|
14
14
|
import { allAdapters, getAdapter, probeAll, readDoctor, rolePools } from "../adapters/registry.js";
|
|
15
15
|
import { SettledTrailerTracker, addUsage, CAPACITY_RE, channelKey, matchesInputBox, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
16
16
|
import { bannerShell, paneDispatchCommand } from "../brand.js";
|
|
@@ -21,7 +21,7 @@ import { driverEvidence } from "../drivers/index.js";
|
|
|
21
21
|
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
22
22
|
import { formatOwnedName } from "../drivers/types.js";
|
|
23
23
|
import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../gates/baseline.js";
|
|
24
|
-
import { runGates } from "../gates/run-gates.js";
|
|
24
|
+
import { reobserveTestFiles, runGates } from "../gates/run-gates.js";
|
|
25
25
|
import { isInfraResult } from "../gates/cache.js";
|
|
26
26
|
import { filesGlob } from "../graph/files-glob.js";
|
|
27
27
|
import { addEvidence, attributeBlocked, batteryPriority, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus, taskContentDigest, tickmarkrDir } from "../graph/graph.js";
|
|
@@ -29,19 +29,19 @@ import { GATE_NAMES } from "../graph/schema.js";
|
|
|
29
29
|
import { distFingerprint } from "../cli/commands/version.js";
|
|
30
30
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
31
31
|
import { executionSignal, remainingExecutionMs, withExecutionBudget, withoutExecutionBudget, ExecutionBudgetExceeded } from "./execution-budget.js";
|
|
32
|
-
import { repairSelectionDecision } from "./repair-selection.js";
|
|
32
|
+
import { repairSelectionDecision, repairSelectionEnabled } from "./repair-selection.js";
|
|
33
33
|
import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
34
34
|
import { runEnvironment } from "./environment.js";
|
|
35
|
-
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, PRESERVE_COMMIT_SUBJECT, PRESERVE_PRODUCER_TRAILER, preserveWorktree, producerFields, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
35
|
+
import { changeRepresented, cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, PRESERVE_COMMIT_SUBJECT, PRESERVE_PRODUCER_TRAILER, preserveWorktree, producerFields, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
36
36
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
37
37
|
import { classifyRepairDisposition, resolveScopeHints } from "./repair-disposition.js";
|
|
38
|
-
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingApprovalActions, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
39
|
-
import { gateReviewerFloor, isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
38
|
+
import { applyScopeAmendments, activeRetryBan, interruptedAttempt, RESUME_HARVEST_SOURCE, resumeHarvestAuthor, approvalAction, APPROVAL_REFUSED, bindingToken, effectiveEvents, foldDecisions, physicalLine, staleApprovals, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, readableExcerpt, receiptOrigin, outstandingReviewFindings, pendingApprovalActions, pendingRechecks, pendingRepairFindings, phaseForGate, priorJudgments, readPriorRunEvidence, recordedTaskFailureKind, reraisedReviewChains, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, standingRulings, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
39
|
+
import { gateReviewerFloor, isDiffCapPark, isGarbageReview, pickReviewer } from "../gates/review.js";
|
|
40
40
|
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
41
41
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, reusedTipEvidence, verifyIntegrationTip } from "./merge.js";
|
|
42
|
-
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
42
|
+
import { climbChannel, inTaskPool, marginalCostRank, nextChannel, poolExhaustion, route, taskPool } from "../route/router.js";
|
|
43
43
|
import { desiredPanes } from "./reconcile.js";
|
|
44
|
-
import { readTierLiveness, readWatchBoard,
|
|
44
|
+
import { readTierLiveness, readWatchBoard, supervisionPresencePath } from "./supervision.js";
|
|
45
45
|
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, reapOwnedProcessGroup, readOwnedProcessGroup, } from "./stall.js";
|
|
46
46
|
// The live set is also the ownership claim shared by task cleanup and termination.
|
|
47
47
|
export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
@@ -58,6 +58,19 @@ export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
|
58
58
|
// A transport timeout says nothing about whether a mutation reached the terminal.
|
|
59
59
|
class HeldProbeExhausted extends Error {
|
|
60
60
|
}
|
|
61
|
+
// OBS-1108: an owned slot whose contact stayed latched unreadable past its deadline. Same terminal
|
|
62
|
+
// path as an exhausted transport probe — worktree preserved, one infra park (recheck-able) — with
|
|
63
|
+
// its own disposition, so the park names what was actually lost.
|
|
64
|
+
class ContactUnreadableExhausted extends HeldProbeExhausted {
|
|
65
|
+
}
|
|
66
|
+
// OBS-1108: how long one owned slot's contact may stay latched unreadable before its attempt parks.
|
|
67
|
+
// Default: the attempt's own stall window — the latch holds the stall kill down, so without this
|
|
68
|
+
// only the 4x hard deadline bounded a blind wait (2.5.8: 22k rows; 2.6.1: ~12k rows/h per worker).
|
|
69
|
+
let contactUnreadableDeadlineMs;
|
|
70
|
+
export function setContactUnreadableDeadlineMsForTests(ms) { contactUnreadableDeadlineMs = ms; }
|
|
71
|
+
export function resetContactUnreadableDeadlineMsForTests() { contactUnreadableDeadlineMs = undefined; }
|
|
72
|
+
const CONTACT_BACKOFF_BASE_MS = 1_000; // the pre-latch retry pause, doubled per latched retry
|
|
73
|
+
const CONTACT_BACKOFF_DOUBLINGS = 5; // ponytail: 32 s ceiling; the poll slice caps it lower anyway
|
|
61
74
|
function isTransportTimeout(error) {
|
|
62
75
|
if (!(error instanceof Error))
|
|
63
76
|
return false;
|
|
@@ -167,26 +180,53 @@ export function resolveRunMode(repoRoot, opts = {}) {
|
|
|
167
180
|
: undefined;
|
|
168
181
|
return { cfg: resolved.cfg, mode: resolved.mode, source, ...(conflict ? { conflict } : {}) };
|
|
169
182
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
183
|
+
/** Journal rows are untrusted input: a malformed field reads as not recorded, never as a value. */
|
|
184
|
+
export const fingerprintsOf = (v) => Array.isArray(v) && v.every((x) => typeof x === "string") ? v : undefined;
|
|
185
|
+
export const baselineProvenanceOf = (v) => {
|
|
186
|
+
const { baseRef, capturedAt } = (typeof v === "object" && v !== null ? v : {});
|
|
187
|
+
return typeof baseRef === "string" && typeof capturedAt === "string" ? { baseRef, capturedAt } : undefined;
|
|
188
|
+
};
|
|
189
|
+
/** A forgiven battery green: the structured field, or the battery's own "(forgiven)" wording on a legacy row. */
|
|
190
|
+
export const forgivenGateRow = (data) => data.pass === true && (Array.isArray(data.forgivenFingerprints) || /\(forgiven\)/u.test(String(data.details ?? "")));
|
|
191
|
+
/**
|
|
192
|
+
* OBS-1123: the forgiveness standing at this point in the journal — each task gate's LATEST verdict
|
|
193
|
+
* where it is a forgiven green, then the latest verification cycle's forgiven tip rows. A legacy row
|
|
194
|
+
* keeps what it never recorded absent, so the reader says "unknown" instead of borrowing a date.
|
|
195
|
+
*/
|
|
196
|
+
export function forgivenFingerprints(events) {
|
|
197
|
+
const latest = new Map();
|
|
198
|
+
for (const e of events) {
|
|
199
|
+
if (e.event === "gate-result" && e.taskId && typeof e.data.gate === "string")
|
|
200
|
+
latest.set(`${e.taskId}\0${e.data.gate}`, e);
|
|
188
201
|
}
|
|
189
|
-
|
|
202
|
+
const tasks = [...latest.values()].filter((e) => forgivenGateRow(e.data));
|
|
203
|
+
const start = events.map((e) => e.event).lastIndexOf("tip-verify-start");
|
|
204
|
+
const tip = events.slice(start + 1).filter((e) => e.event === "tip-verify" && e.data.pass === true
|
|
205
|
+
&& e.data.forgiven === true && typeof e.data.gate === "string");
|
|
206
|
+
return [...tasks, ...tip].map((e) => {
|
|
207
|
+
const fingerprints = fingerprintsOf(e.event === "tip-verify" ? e.data.fingerprints : e.data.forgivenFingerprints);
|
|
208
|
+
const baseline = baselineProvenanceOf(e.data.baselineProvenance);
|
|
209
|
+
return { ...(e.taskId ? { taskId: e.taskId } : {}), gate: e.data.gate,
|
|
210
|
+
...(fingerprints ? { fingerprints } : {}), ...(baseline ? { baseline } : {}) };
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
/** The capture a forgiveness rests on, or an explicit unknown — never a time read off another clock. */
|
|
214
|
+
export function formatBaselineProvenance(p) {
|
|
215
|
+
return p ? `baseline ${p.baseRef.slice(0, 12)} captured ${p.capturedAt}`
|
|
216
|
+
: "baseline provenance unknown (legacy: no capture identity or time recorded)";
|
|
217
|
+
}
|
|
218
|
+
export function formatFingerprints(fingerprints) {
|
|
219
|
+
return fingerprints === undefined ? "fingerprints not recorded"
|
|
220
|
+
: fingerprints.length ? fingerprints.join(" | ") : "no fingerprint recognized";
|
|
221
|
+
}
|
|
222
|
+
export function formatForgiven(f) {
|
|
223
|
+
return `${f.taskId ?? "tip"} ${f.gate}: ${formatFingerprints(f.fingerprints)} — ${formatBaselineProvenance(f.baseline)}`;
|
|
224
|
+
}
|
|
225
|
+
// OBS-1150: the reviewer reads the same standing rulings the worker brief carries, oldest first. A
|
|
226
|
+
// later approval, with or without a reason, adds to them and never supersedes an earlier one.
|
|
227
|
+
function approvalReviewContext(events, taskId) {
|
|
228
|
+
const rulings = standingRulings(events, taskId);
|
|
229
|
+
return rulings.length ? rulings.map((ruling) => `- ${ruling}`).join("\n") : undefined;
|
|
190
230
|
}
|
|
191
231
|
// T14: the events that prove an approval was ENACTED — narrowly causal, never merely subsequent.
|
|
192
232
|
// Ordinary, attempt-cap and review-upheld approvals buy a WORKER, so their proof is a dispatch;
|
|
@@ -225,7 +265,10 @@ export function pendingDaemonApprovalActions(events) {
|
|
|
225
265
|
*/
|
|
226
266
|
export function outstandingApprovals(events) {
|
|
227
267
|
const newest = new Map();
|
|
228
|
-
|
|
268
|
+
// OBS-1178: this reports ANSWERS, not effects — a refused row was answered; an unsound one the daemon
|
|
269
|
+
// has not yet refused (approved, then failed before any dispatch) is still an unanswered decision.
|
|
270
|
+
const { refused } = foldDecisions(events);
|
|
271
|
+
events.forEach((e, i) => { if (e.event === "task-approved" && e.taskId && !refused.has(physicalLine(events, i)))
|
|
229
272
|
newest.set(e.taskId, i); });
|
|
230
273
|
return [...newest]
|
|
231
274
|
.filter(([taskId, i]) => {
|
|
@@ -262,7 +305,9 @@ export function formatSummary(s) {
|
|
|
262
305
|
+ (resumable.length ? `; \`tickmarkr resume ${s.runId}\` enacts ${resumable.join(", ")}` : "")
|
|
263
306
|
+ (stalled.length ? `; ${stalled.join(", ")} failed before any dispatch — neither resume nor \`--retry-failed\` re-dispatches that` : "")
|
|
264
307
|
: "";
|
|
265
|
-
|
|
308
|
+
// OBS-1123: a green that carried baseline reds names each one and the capture that recorded it.
|
|
309
|
+
const forgiven = (s.forgiven ?? []).map((f) => `\nforgiven vs baseline — ${formatForgiven(f)}`).join("");
|
|
310
|
+
return `done: ${s.done.length}, failed: ${s.failed.length}, human: ${s.human.length}, blocked: ${s.blocked.length}, pending: ${s.pending.length}\nintegration branch: ${s.branch}${tip}${outstanding}${forgiven}`;
|
|
266
311
|
}
|
|
267
312
|
/** The narrator enters the production Run cockpit for this exact run. The
|
|
268
313
|
* completed static/growing four-hour records precede this default cutover;
|
|
@@ -325,6 +370,30 @@ function classifyInfraResult(g) {
|
|
|
325
370
|
return;
|
|
326
371
|
g.meta = { ...g.meta, classification: "infra", infra: true };
|
|
327
372
|
}
|
|
373
|
+
// OBS-1106 residual: a red whose assertion is a timeout or a wall-clock budget, and the runner-level
|
|
374
|
+
// diagnostics (never-started files, a worker RPC timeout) that make such a red infrastructure-SHAPED.
|
|
375
|
+
// Shape alone never parks anything: it only buys one isolated re-observation before a charge. A
|
|
376
|
+
// fresh-fingerprint section masks digits to `#`, so both spellings of a number are read.
|
|
377
|
+
const TIMEOUT_SHAPED_RE = /\b(?:Test|Hook) timed out in (?:\d+|#)\s*ms\b|\bexpected (?:\d+|#)(?:\.(?:\d+|#))? to be (?:less than|below)(?: or equal to)? (?:\d+|#)/i;
|
|
378
|
+
const RUNNER_INFRA_DIAGNOSTIC_RE = /\bnever-started (?:[1-9]\d*|#)(?![\d#])|\[vitest-worker\]: Timeout calling\b/;
|
|
379
|
+
const timeoutShaped = (details) => TIMEOUT_SHAPED_RE.test(readableExcerpt(details));
|
|
380
|
+
const ATTRIBUTED_FILE_RE = /^\s*(?:FAIL|×|✗)\s+(?:\|[^|\n]*\|\s+)?(\S+\.(?:test|spec)\.[cm]?[jt]sx?)\b/gm;
|
|
381
|
+
/** The failing files a red attributes to itself: the runner's own list, else its FAIL headlines. */
|
|
382
|
+
const attributedFailingFiles = (g) => [...new Set(Array.isArray(g.meta?.failingFiles) && g.meta.failingFiles.length
|
|
383
|
+
? g.meta.failingFiles.filter((f) => typeof f === "string")
|
|
384
|
+
: [...readableExcerpt(g.details).matchAll(ATTRIBUTED_FILE_RE)].map((m) => m[1].replace(/^\.\//, "")))].sort();
|
|
385
|
+
const metaFiles = (v) => Array.isArray(v) ? v.filter((f) => typeof f === "string") : undefined;
|
|
386
|
+
/** D-607: the files a rerun OBSERVABLY ran — the runner's manifest, the only complete selection
|
|
387
|
+
* evidence. Never the requested selection, and never the per-file lines a runner printed: a runner that
|
|
388
|
+
* ignores file arguments and reports only its failures looks isolated while it ran the whole suite.
|
|
389
|
+
* Undefined without a manifest, and the rerun is then ambiguous. */
|
|
390
|
+
const observedSelection = (g) => {
|
|
391
|
+
const ran = metaFiles(g.meta?.manifest);
|
|
392
|
+
return ran?.length ? [...new Set(ran)].sort() : undefined;
|
|
393
|
+
};
|
|
394
|
+
/** D-607: on the manifest path only the reporter's own attribution counts — a `FAIL path` line inside
|
|
395
|
+
* captured stdout is text, not attribution. Headline lexing is for runners without a reporter. */
|
|
396
|
+
const rerunFailingFiles = (g) => metaFiles(g.meta?.manifest) ? [...new Set(metaFiles(g.meta?.failingFiles) ?? [])].sort() : attributedFailingFiles(g);
|
|
328
397
|
// v1.85 T3: the gates whose failure IS a deterministic measurement — a machine re-ran a command over a
|
|
329
398
|
// tree and printed the same bytes. Those are the failures the fingerprint cap governs (the ruling names
|
|
330
399
|
// it a "deterministic-gate" cap): a third identical answer to a question already answered twice is the
|
|
@@ -353,6 +422,12 @@ function narrowRepairBattery(failing) {
|
|
|
353
422
|
return isOracleFailure(g) && g.meta?.unparseable !== true;
|
|
354
423
|
return false;
|
|
355
424
|
}
|
|
425
|
+
// OBS-1182: re-seating copies the channel's identity AND its launch effort — the four-field copy
|
|
426
|
+
// this replaced silently reverted a climbed or pinned seat to the CLI's default effort.
|
|
427
|
+
const seatAssignment = (c) => ({
|
|
428
|
+
adapter: c.adapter, model: c.model, channel: c.channel, tier: c.tier,
|
|
429
|
+
...(c.effort ? { effort: c.effort } : {}),
|
|
430
|
+
});
|
|
356
431
|
// v2.0 T2 (OBS-554): the measurement keys run-gates stamps on a GateResult, lifted verbatim onto the
|
|
357
432
|
// gate row. One list, one lift — both onGate sites record through the same helper.
|
|
358
433
|
const GATE_TELEMETRY_KEYS = ["durationMs", "load1Start", "load1End", "load1Max", "load1Mean", "selectedDurationMs", "fullDurationMs", "invocations"];
|
|
@@ -409,18 +484,36 @@ function repairDiffFilesCut(diff, cap, names) {
|
|
|
409
484
|
function repairFindingsBrief(results) {
|
|
410
485
|
return results.filter(gateFailed).map((g) => {
|
|
411
486
|
const evidence = g.meta?.failureEvidence;
|
|
412
|
-
return `${g.gate}: ${g.details}` + (Array.isArray(evidence) && evidence.length
|
|
487
|
+
return `${g.gate}: ${readableExcerpt(g.details)}` + (Array.isArray(evidence) && evidence.length
|
|
413
488
|
? `\nAssertion evidence:\n${JSON.stringify(evidence, null, 2)}` : "");
|
|
414
489
|
}).join("\n\n");
|
|
415
490
|
}
|
|
416
|
-
|
|
417
|
-
|
|
491
|
+
// OBS-1019 add.2: the order a repeated review chain earns in place of the smallest-edit order.
|
|
492
|
+
const REPAIR_CLASS_SENTENCE = "Before editing a chain reraised more than once, enumerate the in-scope consumers,"
|
|
493
|
+
+ " bridges, operations and event sequences as a closed case table; repair every implicated member and verify"
|
|
494
|
+
+ " the accumulated diff against that table.";
|
|
495
|
+
/**
|
|
496
|
+
* The fix-only contract: the findings verbatim, then the diff content of the work already landed.
|
|
497
|
+
* OBS-1019 add.2: a first finding stays bounded to the smallest edit. A chain a review has re-raised
|
|
498
|
+
* more than once was chased edge by edge while every round re-judged the accumulated diff, so it owes
|
|
499
|
+
* the whole class instead: a closed case table inside the declared bounds, never beyond them.
|
|
500
|
+
*/
|
|
501
|
+
function repairBrief(findings, diff, baseRef, repeated = []) {
|
|
418
502
|
const repair = { findings, diff };
|
|
419
503
|
return [
|
|
420
|
-
"## Repair attempt — fix ONLY what these findings name",
|
|
504
|
+
repeated.length ? "## Repair attempt — repair the whole class these findings name" : "## Repair attempt — fix ONLY what these findings name",
|
|
421
505
|
"The commits from your prior attempt are already in this worktree and their diff is reproduced"
|
|
422
|
-
+ " below. Do NOT re-implement that work, do not start over, and do not revert it:
|
|
423
|
-
+ " smallest change that resolves every finding, then commit.",
|
|
506
|
+
+ " below. Do NOT re-implement that work, do not start over, and do not revert it: "
|
|
507
|
+
+ (repeated.length ? "resolve every member of the case table below, then commit." : "make the smallest change that resolves every finding, then commit."),
|
|
508
|
+
...(repeated.length ? [
|
|
509
|
+
"",
|
|
510
|
+
"### Repeated review chains — repair the class, not the next edge",
|
|
511
|
+
// Fingerprint and count only: the finding's prose is already quoted once, below or under its heading.
|
|
512
|
+
...repeated.map(({ finding, reraises }) => `- \`${finding.fingerprint}\` re-raised ${reraises} times`),
|
|
513
|
+
REPAIR_CLASS_SENTENCE,
|
|
514
|
+
"The table stays inside the task's declared write scope (files[]) and never narrows its goal or an"
|
|
515
|
+
+ " acceptance criterion; the reviewer judges the accumulated diff, not the table.",
|
|
516
|
+
] : []),
|
|
424
517
|
"",
|
|
425
518
|
"### Failing gate findings (verbatim)",
|
|
426
519
|
repair.findings,
|
|
@@ -441,7 +534,8 @@ const DEFERRED_FINDINGS_HEADING = "## Deferred review findings — a reviewer AC
|
|
|
441
534
|
// OBS-419: the newest approval starts the current engagement, so it is also the sole authority for
|
|
442
535
|
// that engagement's optional ceiling. Stop at the newest approval even when the field is absent: a
|
|
443
536
|
// later ordinary release restores the module default instead of inheriting an older operator limit.
|
|
444
|
-
function approvedReviewRoundCeiling(
|
|
537
|
+
function approvedReviewRoundCeiling(journaled, taskId) {
|
|
538
|
+
const events = effectiveEvents(journaled); // OBS-1178: a refused or unsound approval sets no ceiling
|
|
445
539
|
for (let i = events.length - 1; i >= 0; i--) {
|
|
446
540
|
const event = events[i];
|
|
447
541
|
if (event.event !== "task-approved" || event.taskId !== taskId)
|
|
@@ -479,6 +573,14 @@ let capacityBackoffMs = CAPACITY_BACKOFF_MS;
|
|
|
479
573
|
/** Test seam — shrink the capacity backoff without minute-long sleeps. */
|
|
480
574
|
export function setCapacityBackoffMsForTests(ms) { capacityBackoffMs = ms; }
|
|
481
575
|
export function resetCapacityBackoffMsForTests() { capacityBackoffMs = CAPACITY_BACKOFF_MS; }
|
|
576
|
+
// OBS-1169: a CLI bootstrap failure buys ONE same-channel retry after a bounded wait — journal-counted
|
|
577
|
+
// per task and seat like capacity — and then the adapter is escalated away from, never retried forever.
|
|
578
|
+
const BOOTSTRAP_RETRY_CAP = 1;
|
|
579
|
+
const BOOTSTRAP_BACKOFF_MS = 5_000;
|
|
580
|
+
let bootstrapBackoffMs = BOOTSTRAP_BACKOFF_MS;
|
|
581
|
+
/** Test seam — shrink the bootstrap backoff. */
|
|
582
|
+
export function setBootstrapBackoffMsForTests(ms) { bootstrapBackoffMs = ms; }
|
|
583
|
+
export function resetBootstrapBackoffMsForTests() { bootstrapBackoffMs = BOOTSTRAP_BACKOFF_MS; }
|
|
482
584
|
const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
|
|
483
585
|
// OBS-117 (v1.71 T6): a worker pane that never prints a byte by T+60s after dispatch is a dead
|
|
484
586
|
// channel — don't burn the full stall window waiting for a silent launch failure. Checked on the
|
|
@@ -544,9 +646,16 @@ export function resetWorkerStartupWindowMsForTests() {
|
|
|
544
646
|
/** One seat, one startup prefix. Once execution or its input box is seen, later panes are ineligible. */
|
|
545
647
|
export class StartupFailureDetector {
|
|
546
648
|
adapter;
|
|
649
|
+
pattern;
|
|
650
|
+
wholePrefix;
|
|
547
651
|
closed = false;
|
|
548
|
-
constructor(adapter
|
|
652
|
+
constructor(adapter, pattern = WORKER_STARTUP_FAILURE_RE, // OBS-1169: or known bootstrap text
|
|
653
|
+
// OBS-1169: bootstrap evidence must own the WHOLE bounded prefix — a match followed by a tool frame
|
|
654
|
+
// or an input box anywhere later in the sample is a worker that ran, not a CLI that died starting.
|
|
655
|
+
wholePrefix = false) {
|
|
549
656
|
this.adapter = adapter;
|
|
657
|
+
this.pattern = pattern;
|
|
658
|
+
this.wholePrefix = wholePrefix;
|
|
550
659
|
}
|
|
551
660
|
sample(output, launchedAt) {
|
|
552
661
|
if (this.closed || launchedAt === undefined || Date.now() - launchedAt > workerStartupWindowMs)
|
|
@@ -583,6 +692,7 @@ export class StartupFailureDetector {
|
|
|
583
692
|
}
|
|
584
693
|
}
|
|
585
694
|
let offset = 0;
|
|
695
|
+
let found;
|
|
586
696
|
for (const [index, row] of rows.entries()) {
|
|
587
697
|
const cleanRow = row.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "");
|
|
588
698
|
// Structured tool frames and the terminal's tool headings close the prefix BEFORE their body.
|
|
@@ -591,16 +701,20 @@ export class StartupFailureDetector {
|
|
|
591
701
|
this.closed = true;
|
|
592
702
|
return;
|
|
593
703
|
}
|
|
594
|
-
if (!this.adapter.harnessBannerRows?.includes(cleanRow)) {
|
|
595
|
-
const match =
|
|
596
|
-
if (match)
|
|
597
|
-
|
|
704
|
+
if (!found && !this.adapter.harnessBannerRows?.includes(cleanRow)) {
|
|
705
|
+
const match = this.pattern.exec(row);
|
|
706
|
+
if (match) {
|
|
707
|
+
found = {
|
|
598
708
|
matchedBytes: match[0], offset: offset + Buffer.byteLength(row.slice(0, match.index)),
|
|
599
709
|
row, rowNumber: index + 1,
|
|
600
710
|
};
|
|
711
|
+
if (!this.wholePrefix)
|
|
712
|
+
return found;
|
|
713
|
+
}
|
|
601
714
|
}
|
|
602
715
|
offset += Buffer.byteLength(row) + 1;
|
|
603
716
|
}
|
|
717
|
+
return found;
|
|
604
718
|
}
|
|
605
719
|
}
|
|
606
720
|
// OBS-901/906: one daemon-owned writer for every execution surface. Drivers expose one retained
|
|
@@ -809,7 +923,7 @@ export function formatTipProof(p) {
|
|
|
809
923
|
}
|
|
810
924
|
// Isolate the battery so cancellation can stop its control flow as well as its shell children.
|
|
811
925
|
// The child uses the same verifier and capacity; only the daemon writes lifecycle verdict rows.
|
|
812
|
-
async function cancellableTipBattery(intWt, commands, runDir, baseline, signal) {
|
|
926
|
+
async function cancellableTipBattery(intWt, commands, runDir, baseline, signal, evidence = {}) {
|
|
813
927
|
signal.throwIfAborted();
|
|
814
928
|
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
|
|
815
929
|
const script = `
|
|
@@ -858,9 +972,9 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
858
972
|
finally { process.send({ type: 'lease-release', id }); }
|
|
859
973
|
};
|
|
860
974
|
let input = ''; for await (const chunk of process.stdin) input += chunk;
|
|
861
|
-
const { intWt, commands, runDir, baseline, capacity } = JSON.parse(input);
|
|
975
|
+
const { intWt, commands, runDir, baseline, capacity, evidence } = JSON.parse(input);
|
|
862
976
|
try {
|
|
863
|
-
const result = await runWithVerificationBudget(capacity, () => runWithCommandLease(lease, () => verifyIntegrationTip(intWt, commands, runDir, baseline)));
|
|
977
|
+
const result = await runWithVerificationBudget(capacity, () => runWithCommandLease(lease, () => verifyIntegrationTip(intWt, commands, runDir, baseline, evidence)));
|
|
864
978
|
if (!cancelled) process.stdout.write(JSON.stringify(result.map(row => ({ ...row, capacity: [...commandCapacities].reverse().find(([command]) => command === row.cmd || command.startsWith(row.cmd + " "))?.[1] ?? capacity }))));
|
|
865
979
|
} catch (error) { if (!cancelled) throw error; }
|
|
866
980
|
finally { process.disconnect(); }
|
|
@@ -911,7 +1025,7 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
911
1025
|
});
|
|
912
1026
|
signal.addEventListener("abort", cancel, { once: true });
|
|
913
1027
|
child.stdin.on("error", () => { });
|
|
914
|
-
child.stdin.end(JSON.stringify({ intWt, commands, runDir, baseline, capacity: resolvedCapacity() }));
|
|
1028
|
+
child.stdin.end(JSON.stringify({ intWt, commands, runDir, baseline, capacity: resolvedCapacity(), evidence }));
|
|
915
1029
|
try {
|
|
916
1030
|
if (signal.aborted)
|
|
917
1031
|
cancel();
|
|
@@ -972,9 +1086,9 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
972
1086
|
return false;
|
|
973
1087
|
}
|
|
974
1088
|
let tipFailed = false;
|
|
975
|
-
for (const r of await (opts.signal
|
|
976
|
-
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal)
|
|
977
|
-
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline))) {
|
|
1089
|
+
for (const r of await runWithInterruptionSink(journalInterruption(journal, undefined), () => opts.signal
|
|
1090
|
+
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal, opts.evidence)
|
|
1091
|
+
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline, opts.evidence))) {
|
|
978
1092
|
const measuredCapacity = r.capacity ?? capacity;
|
|
979
1093
|
const evidence = r.reused ? reusedTipEvidence(r) : {
|
|
980
1094
|
...(r.originRunRoot ? { originRunRoot: r.originRunRoot } : {}),
|
|
@@ -987,7 +1101,7 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
987
1101
|
};
|
|
988
1102
|
if (r.pass) {
|
|
989
1103
|
// Q121s: a forgiven pass journals its fingerprints — honest about what was carried, never a silent green.
|
|
990
|
-
journal.append("tip-verify", undefined, { ...evidence, gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.reused ? { cached: true } : {}), ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
1104
|
+
journal.append("tip-verify", undefined, { ...evidence, gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.reused ? { cached: true } : {}), ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints, ...(r.baselineProvenance ? { baselineProvenance: r.baselineProvenance } : {}) } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
991
1105
|
}
|
|
992
1106
|
else {
|
|
993
1107
|
journal.append("tip-verify-failed", undefined, {
|
|
@@ -1008,6 +1122,12 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
1008
1122
|
}
|
|
1009
1123
|
return tipFailed;
|
|
1010
1124
|
}
|
|
1125
|
+
/** OBS-953: a detected wall-over-monotonic discontinuity journals `host-suspend` (its offset was
|
|
1126
|
+
* subtracted from active test time); an overdue poll whose clocks advanced together journals
|
|
1127
|
+
* `host-poll-overdue` (recorded, never subtracted). */
|
|
1128
|
+
const journalInterruption = (journal, taskId) => (interruption) => {
|
|
1129
|
+
journal.append(interruption.kind === "host-suspend" ? "host-suspend" : "host-poll-overdue", taskId, { ...interruption });
|
|
1130
|
+
};
|
|
1011
1131
|
async function commitsAheadOfRef(base, head, cwd) {
|
|
1012
1132
|
if (head === base)
|
|
1013
1133
|
return [];
|
|
@@ -1500,17 +1620,24 @@ async function taskWithMaterializedContext(repoRoot, journal, worktree, task, at
|
|
|
1500
1620
|
}
|
|
1501
1621
|
return { ...task, context };
|
|
1502
1622
|
}
|
|
1623
|
+
// OBS-1107: a refused pick is not lost work by that fact alone. A commit whose own change the
|
|
1624
|
+
// destination already holds (content-empty, or a patch represented there under another hash) is
|
|
1625
|
+
// accounted for and skipped; the carry stops only at a nonempty change the destination lacks.
|
|
1503
1626
|
async function cherryPickCommits(wt, commits) {
|
|
1504
1627
|
const carried = [];
|
|
1628
|
+
const accounted = [];
|
|
1505
1629
|
for (const hash of commits) {
|
|
1506
1630
|
const r = await shGit(`git cherry-pick --no-gpg-sign ${shq(hash)}`, wt);
|
|
1507
|
-
if (r.code
|
|
1508
|
-
|
|
1509
|
-
|
|
1631
|
+
if (r.code === 0) {
|
|
1632
|
+
carried.push(hash);
|
|
1633
|
+
continue;
|
|
1510
1634
|
}
|
|
1511
|
-
|
|
1635
|
+
await shGit("git cherry-pick --abort", wt);
|
|
1636
|
+
if (!(await changeRepresented(wt, hash)))
|
|
1637
|
+
break;
|
|
1638
|
+
accounted.push(hash);
|
|
1512
1639
|
}
|
|
1513
|
-
return carried;
|
|
1640
|
+
return { carried, accounted };
|
|
1514
1641
|
}
|
|
1515
1642
|
const PRESERVED_REF_PREFIX = "refs/tickmarkr/preserved/";
|
|
1516
1643
|
const preservedRefOf = (data) => [data.ref, data.preservedRef].find((v) => typeof v === "string" && v.startsWith(PRESERVED_REF_PREFIX));
|
|
@@ -1662,6 +1789,7 @@ export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "s
|
|
|
1662
1789
|
const original = err instanceof Error ? err.message : String(err);
|
|
1663
1790
|
// A fatal close never finished a cycle: fresh/reused green is withheld, a recorded red still stands.
|
|
1664
1791
|
let tipProof = { kind: "incomplete" };
|
|
1792
|
+
let forgiven = [];
|
|
1665
1793
|
try {
|
|
1666
1794
|
// Only the latest engagement can already own this terminal outcome.
|
|
1667
1795
|
const events = journal.read();
|
|
@@ -1675,6 +1803,8 @@ export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "s
|
|
|
1675
1803
|
}
|
|
1676
1804
|
const latest = runEndTipProof(events.slice(from));
|
|
1677
1805
|
tipProof = { ...latest, kind: latest.kind === "failed" ? "failed" : "incomplete" };
|
|
1806
|
+
// OBS-1123: the same fold the normal close records, so a crash never drops what a green carried.
|
|
1807
|
+
forgiven = forgivenFingerprints(events);
|
|
1678
1808
|
}
|
|
1679
1809
|
catch (readErr) {
|
|
1680
1810
|
console.error(`tickmarkr ${runId}: journal read failed while recording the fatal run-end (${readErr instanceof Error ? readErr.message : String(readErr)}) — original error: ${original}`);
|
|
@@ -1691,6 +1821,7 @@ export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "s
|
|
|
1691
1821
|
fatal: true,
|
|
1692
1822
|
error: original,
|
|
1693
1823
|
tipProof,
|
|
1824
|
+
...(forgiven.length ? { forgiven } : {}),
|
|
1694
1825
|
};
|
|
1695
1826
|
try {
|
|
1696
1827
|
journal.append("run-end", undefined, record);
|
|
@@ -1821,6 +1952,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1821
1952
|
// > repo > global > default. The resolved cfg carries mode-compiled floors; route() never sees the mode.
|
|
1822
1953
|
const rm = resolveRunMode(repoRoot, { flag: opts.mode, spec: graph.mode, globalDir: opts.globalDir });
|
|
1823
1954
|
const cfg = rm.cfg;
|
|
1955
|
+
// OBS-1199: the effective repair-selection default is resolved apart from the optional execution
|
|
1956
|
+
// budget, so an absent policy selects on every driver; a resume keeps what the run recorded.
|
|
1957
|
+
let repairSelection = repairSelectionEnabled(cfg);
|
|
1824
1958
|
if (opts.resume) {
|
|
1825
1959
|
const start = Journal.open(repoRoot, runId).read().find((e) => e.event === "run-start");
|
|
1826
1960
|
const snapshot = start?.data.effectivePolicy;
|
|
@@ -1832,6 +1966,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1832
1966
|
// Historical runs did not opt in; current settings cannot grant them a new allowance.
|
|
1833
1967
|
cfg.executionPolicy = recorded?.config?.executionPolicy === undefined
|
|
1834
1968
|
? undefined : ExecutionPolicySchema.parse(recorded.config.executionPolicy);
|
|
1969
|
+
const recordedGates = recorded?.config?.gates;
|
|
1970
|
+
if (recordedGates?.repairSelection !== undefined && typeof recordedGates.repairSelection !== "boolean") {
|
|
1971
|
+
throw new Error("cannot restore malformed recorded repair selection");
|
|
1972
|
+
}
|
|
1973
|
+
repairSelection = repairSelectionEnabled({ executionPolicy: cfg.executionPolicy, gates: recordedGates });
|
|
1835
1974
|
}
|
|
1836
1975
|
if (cfg.executionPolicy && driver.id !== "subprocess") {
|
|
1837
1976
|
throw new Error("experimental executionPolicy currently requires the subprocess driver");
|
|
@@ -2107,11 +2246,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2107
2246
|
// GATE-08 (v1.12): the humanGate guard consults this run's journaled approvals, not just the compiled
|
|
2108
2247
|
// flag. Startup approvals seed the first scheduling pass; live approvals are folded at task
|
|
2109
2248
|
// boundaries below so a sibling can release parked work without waiting for run-end + resume.
|
|
2249
|
+
//
|
|
2250
|
+
// OBS-1178: revalidate every open approval before anything can enact it. The decision fold already
|
|
2251
|
+
// gives an unsound one no effect; the approval-refused row records that answer and consumes it, so
|
|
2252
|
+
// every fold and every operator reads the task as still parked — a stale waive never satisfies a newer gate.
|
|
2253
|
+
const refuseStaleApprovals = () => {
|
|
2254
|
+
for (const [taskId, stale] of staleApprovals(journal.read())) {
|
|
2255
|
+
// `lines` names the voided rows, so a sound decision beside a stale one survives the refusal.
|
|
2256
|
+
journal.append(APPROVAL_REFUSED, taskId, { reason: `stale or unbound decision refused before enactment — ${stale.reason}`, lines: stale.lines });
|
|
2257
|
+
}
|
|
2258
|
+
};
|
|
2259
|
+
refuseStaleApprovals();
|
|
2110
2260
|
const approvalStartupEvents = journal.read();
|
|
2111
|
-
// Continuing human-gate permission survives enactment; malformed releases grant none.
|
|
2112
|
-
const validApproval = (e) => e.event === "task-approved" && e.taskId
|
|
2113
|
-
&&
|
|
2114
|
-
const approved = new Set(approvalStartupEvents.filter(validApproval).map((e) => e.taskId));
|
|
2261
|
+
// Continuing human-gate permission survives enactment; malformed, refused or unsound releases grant none.
|
|
2262
|
+
const validApproval = (e) => e.event === "task-approved" && e.taskId !== undefined
|
|
2263
|
+
&& approvalAction(e.taskId, e).authority !== "inert";
|
|
2264
|
+
const approved = new Set(effectiveEvents(approvalStartupEvents).filter(validApproval).map((e) => e.taskId));
|
|
2115
2265
|
const startupActions = pendingDaemonApprovalActions(approvalStartupEvents);
|
|
2116
2266
|
let approvalSweepCursor = approvalStartupEvents.length;
|
|
2117
2267
|
const commands = detectGateCommands(repoRoot, cfg);
|
|
@@ -2150,11 +2300,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2150
2300
|
console.error(`tickmarkr: narrator not opened: ${placed.error}`);
|
|
2151
2301
|
return undefined;
|
|
2152
2302
|
};
|
|
2153
|
-
// WB-1 (OBS-988): the daemon watches its own cockpit. A board is lost when
|
|
2154
|
-
// aged past the supervision stale bound
|
|
2155
|
-
//
|
|
2156
|
-
//
|
|
2157
|
-
//
|
|
2303
|
+
// WB-1 (OBS-988): the daemon watches its own cockpit. A board is lost when its owner pid is dead, the
|
|
2304
|
+
// recorded arm has no presence, or its beat aged past the supervision stale bound with no owner pid
|
|
2305
|
+
// to consult. OBS-1110: a stale beat under a LIVE MATCHING OWNER — its pid answers and the arm it
|
|
2306
|
+
// claimed is present — is a slow board, not a lost one: journaled once per episode as
|
|
2307
|
+
// `watch-board-slow` and never spending a reopen. The beat and presence checks wait for the UI to
|
|
2308
|
+
// have armed (armId recorded) — a freshly reopened board that has not armed yet is not a second
|
|
2309
|
+
// loss. Reopens are bounded per run; past the bound the loss is journaled `boardless` and the
|
|
2310
|
+
// narrator is never called again.
|
|
2158
2311
|
// Leg-2 T7 M2: one `watch-board-lost` row per LOSS. A loss is identified by the pane and pid the
|
|
2159
2312
|
// owner record names; a reopen that fails leaves that record in place, so the next poll sees the
|
|
2160
2313
|
// same loss, journals only its own reopen attempt, and the bound retires the run boardless from the
|
|
@@ -2163,31 +2316,79 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2163
2316
|
let boardReopens = journal.read().filter((e) => e.event === "watch-board-reopened" || e.event === "watch-board-reopen-failed").length;
|
|
2164
2317
|
let boardless = false;
|
|
2165
2318
|
let journaledLoss;
|
|
2166
|
-
|
|
2319
|
+
let journaledSlow;
|
|
2320
|
+
// Per claim: the next adoption offer, backed off (doubling from the poll cadence to a small cap) but
|
|
2321
|
+
// never exhausted; and whether a reservation is unresolved, so the loop must keep waking to watch it.
|
|
2322
|
+
const MAX_HELD_OFFER_GAP_MS = 4 * APPROVAL_POLL_MS;
|
|
2323
|
+
let heldOffer;
|
|
2324
|
+
let heldRetry = false;
|
|
2325
|
+
const boardHealth = () => {
|
|
2167
2326
|
const owner = readWatchBoard(repoRoot, runId);
|
|
2168
2327
|
if (!owner)
|
|
2169
2328
|
return undefined;
|
|
2170
|
-
const
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
if (beat.state === "STALE")
|
|
2174
|
-
return { ...lost, beatAgeMs: beat.beatAgeMs };
|
|
2175
|
-
// ponytail: presence path math mirrors supervision.ts's private helper (`<tier>.live.<armId>`)
|
|
2176
|
-
if (!existsSync(join(dirname(supervisionBeatPath(repoRoot, "watch")), `watch.live.${owner.armId}`)))
|
|
2177
|
-
return lost;
|
|
2178
|
-
}
|
|
2329
|
+
const named = { pane: owner.pane, ...(owner.pid !== undefined ? { pid: owner.pid } : {}) };
|
|
2330
|
+
const beat = owner.armId !== undefined ? readTierLiveness(repoRoot, "watch") : undefined;
|
|
2331
|
+
const row = beat?.state === "STALE" ? { ...named, beatAgeMs: beat.beatAgeMs } : named;
|
|
2179
2332
|
if (owner.pid !== undefined && !isPidLive(owner.pid))
|
|
2180
|
-
return lost;
|
|
2333
|
+
return { lost: row };
|
|
2334
|
+
if (owner.armId !== undefined && !existsSync(supervisionPresencePath(repoRoot, "watch", owner.armId)))
|
|
2335
|
+
return { lost: row };
|
|
2336
|
+
if (beat?.state === "STALE")
|
|
2337
|
+
return owner.pid !== undefined ? { slow: row } : { lost: row };
|
|
2181
2338
|
return undefined;
|
|
2182
2339
|
};
|
|
2340
|
+
// OBS-1172: a placement the driver HELD after an indeterminate split receipt has no slot here. While
|
|
2341
|
+
// the reservation stays unresolved the loop keeps polling it — even with every worker slot busy — so
|
|
2342
|
+
// a claim landing mid-task is seen. Once a live observer has claimed, the narrator is offered it
|
|
2343
|
+
// again, backed off per claim (doubling from the poll cadence to MAX_HELD_OFFER_GAP_MS) but never
|
|
2344
|
+
// exhausted: a listing that is unavailable or not yet showing the handle is retried until it proves
|
|
2345
|
+
// the board or the run ends; only success latches (the slot). The driver adopts the proven pane —
|
|
2346
|
+
// tracked from then on like any board — or keeps holding it, and never splits a second board over it.
|
|
2347
|
+
// This spends no reopen: nothing was lost.
|
|
2348
|
+
const adoptHeldBoard = async () => {
|
|
2349
|
+
heldRetry = false;
|
|
2350
|
+
const owner = readWatchBoard(repoRoot, runId);
|
|
2351
|
+
if (watchSlot || owner?.pane !== "" || owner.retired)
|
|
2352
|
+
return;
|
|
2353
|
+
heldRetry = true;
|
|
2354
|
+
if (owner.pid === undefined || !isPidLive(owner.pid))
|
|
2355
|
+
return; // unclaimed, or claimant dead: keep watching
|
|
2356
|
+
const id = `${owner.token}:${owner.pid}`;
|
|
2357
|
+
const now = Date.now();
|
|
2358
|
+
const offer = heldOffer?.id === id ? heldOffer : { id, gapMs: APPROVAL_POLL_MS, nextAt: now };
|
|
2359
|
+
if (now < offer.nextAt) {
|
|
2360
|
+
heldOffer = offer;
|
|
2361
|
+
return;
|
|
2362
|
+
}
|
|
2363
|
+
const adopted = await openBoard();
|
|
2364
|
+
if (adopted.ok) {
|
|
2365
|
+
heldOffer = undefined;
|
|
2366
|
+
heldRetry = false;
|
|
2367
|
+
journal.append("watch-board-adopted", undefined, { pane: adopted.slot.id });
|
|
2368
|
+
return;
|
|
2369
|
+
}
|
|
2370
|
+
heldOffer = { id, gapMs: Math.min(offer.gapMs * 2, MAX_HELD_OFFER_GAP_MS), nextAt: Date.now() + offer.gapMs };
|
|
2371
|
+
};
|
|
2183
2372
|
const watchBoard = async () => {
|
|
2184
|
-
if (
|
|
2373
|
+
if (boardless || !trackedDriver.narrator)
|
|
2185
2374
|
return;
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2375
|
+
if (!boardOpened)
|
|
2376
|
+
return adoptHeldBoard();
|
|
2377
|
+
const health = boardHealth();
|
|
2378
|
+
if (health && "slow" in health) {
|
|
2379
|
+
const identity = `${health.slow.pane}:${health.slow.pid}`;
|
|
2380
|
+
if (identity !== journaledSlow) {
|
|
2381
|
+
journaledSlow = identity;
|
|
2382
|
+
journal.append("watch-board-slow", undefined, health.slow);
|
|
2383
|
+
}
|
|
2189
2384
|
return;
|
|
2190
2385
|
}
|
|
2386
|
+
journaledSlow = undefined;
|
|
2387
|
+
if (!health) {
|
|
2388
|
+
journaledLoss = undefined;
|
|
2389
|
+
return adoptHeldBoard();
|
|
2390
|
+
}
|
|
2391
|
+
const loss = health.lost;
|
|
2191
2392
|
const identity = `${loss.pane}:${loss.pid ?? ""}`;
|
|
2192
2393
|
if (identity !== journaledLoss) {
|
|
2193
2394
|
journaledLoss = identity;
|
|
@@ -2221,6 +2422,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2221
2422
|
void promise.finally(() => hostChecks.delete(promise)).catch(() => { });
|
|
2222
2423
|
return promise;
|
|
2223
2424
|
};
|
|
2425
|
+
// OBS-1190: every admission poll still probes (the probe gates admission), but the journal keeps only
|
|
2426
|
+
// the first observation and each change of state or reference — healthy, degraded, unreadable,
|
|
2427
|
+
// recovered — the way suite-wait already dedupes by count. A steady wait no longer floods the journal.
|
|
2428
|
+
let journaledHost;
|
|
2429
|
+
const journalObservation = (taskId, observation, resuming) => {
|
|
2430
|
+
const state = observation.medianMs === null ? "unreadable"
|
|
2431
|
+
: hostReferenceMs !== undefined && hostDegraded(observation, hostReferenceMs) ? "degraded" : "healthy";
|
|
2432
|
+
const key = `${state}:${hostReferenceMs ?? ""}`;
|
|
2433
|
+
if (key === journaledHost)
|
|
2434
|
+
return;
|
|
2435
|
+
journaledHost = key;
|
|
2436
|
+
journal.append("host-observation", taskId, { ...observation, referenceMs: hostReferenceMs ?? null, resuming, state });
|
|
2437
|
+
};
|
|
2224
2438
|
const recordReference = (observation) => {
|
|
2225
2439
|
if (observation.medianMs === null)
|
|
2226
2440
|
return;
|
|
@@ -2244,7 +2458,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2244
2458
|
const probeBudget = HOST_PROBE_SAMPLE_MS * HOST_PROBE_SAMPLES;
|
|
2245
2459
|
if (!observation || (!first && remaining >= probeBudget)) {
|
|
2246
2460
|
observation = await observeHost(signal, remaining > 0 ? Math.min(probeBudget, remaining) : undefined);
|
|
2247
|
-
|
|
2461
|
+
journalObservation(taskId, observation, resuming);
|
|
2248
2462
|
}
|
|
2249
2463
|
first = false;
|
|
2250
2464
|
if (hostReferenceMs === undefined && observation.medianMs !== null)
|
|
@@ -2288,7 +2502,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2288
2502
|
const initializeHost = () => trackHost((async () => {
|
|
2289
2503
|
const signal = hostSignal();
|
|
2290
2504
|
const observation = await observeHost(signal);
|
|
2291
|
-
|
|
2505
|
+
journalObservation(undefined, observation, !!opts.resume);
|
|
2292
2506
|
if (hostReferenceMs === undefined)
|
|
2293
2507
|
recordReference(observation);
|
|
2294
2508
|
if (opts.resume || observation.medianMs === null)
|
|
@@ -2334,7 +2548,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2334
2548
|
waited = true;
|
|
2335
2549
|
journal.append("suite-wait", taskId, { count, ...(gate ? { gate } : {}) });
|
|
2336
2550
|
}, SUITE_POLL_MS, signal);
|
|
2337
|
-
}, async () => {
|
|
2551
|
+
}, () => runWithInterruptionSink(journalInterruption(journal, taskId), async () => {
|
|
2338
2552
|
try {
|
|
2339
2553
|
const result = await run();
|
|
2340
2554
|
// Some command oracles turn launch errors into results. Admission failure still parks infra.
|
|
@@ -2346,7 +2560,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2346
2560
|
if (taskId)
|
|
2347
2561
|
activeGatePhases.delete(taskId);
|
|
2348
2562
|
}
|
|
2349
|
-
});
|
|
2563
|
+
}));
|
|
2350
2564
|
};
|
|
2351
2565
|
let baseRef;
|
|
2352
2566
|
let baseline;
|
|
@@ -2435,7 +2649,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2435
2649
|
// journal.replayStatuses predates typed releases and re-pends even inert rows. Keep that
|
|
2436
2650
|
// legacy reader intact; scheduling accepts only the fold's recognised authorities.
|
|
2437
2651
|
const statuses = new Map();
|
|
2438
|
-
for (const e of replayEvents) {
|
|
2652
|
+
for (const e of effectiveEvents(replayEvents)) { // OBS-1178: a refused or unsound decision released nothing
|
|
2439
2653
|
if (!e.taskId)
|
|
2440
2654
|
continue;
|
|
2441
2655
|
if (e.event === "task-dispatch")
|
|
@@ -2499,7 +2713,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2499
2713
|
if (emptyCapture)
|
|
2500
2714
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(emptyCapture, null, 2));
|
|
2501
2715
|
await initializeHost();
|
|
2502
|
-
|
|
2716
|
+
// OBS-1123: the capture's identity and publication time ride the measurement every forgiveness reads.
|
|
2717
|
+
const captured = { ...(emptyCapture ?? await captureBaseline(repoRoot, commands)),
|
|
2718
|
+
provenance: { baseRef, capturedAt: new Date().toISOString() } };
|
|
2503
2719
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(captured, null, 2));
|
|
2504
2720
|
baseline = captured;
|
|
2505
2721
|
for (const warning of captured.warnings ?? [])
|
|
@@ -2563,16 +2779,44 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2563
2779
|
// owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
|
|
2564
2780
|
// reconcile (optional chain → no-op), so gates and the oracle suite never feel this. keepPanes
|
|
2565
2781
|
// "forever" is the keep-everything debug override — it disables the sweep entirely.
|
|
2566
|
-
|
|
2782
|
+
// OBS-1109: an interrupted attempt's pane is evidence — its nonce-bound trailer decides whether
|
|
2783
|
+
// the attempt is harvested, declined as foreign, or recovered — so the fold's run-resume clear
|
|
2784
|
+
// must not reach it before harvestInterruptedAttempt has read it and reaped its processes. Only a
|
|
2785
|
+
// resumed daemon can hold such an attempt: a fresh run's sweep is the baseline sweep.
|
|
2786
|
+
// OBS-1204: the set is FROZEN here, once, from the rows up to this daemon's own run-resume row: an
|
|
2787
|
+
// attempt launched after it is this daemon's own and reconciles exactly as in a fresh run. Each
|
|
2788
|
+
// frozen spare lapses for good once its harvest, decline or a superseding dispatch is journaled.
|
|
2789
|
+
const resumeInterrupted = new Map();
|
|
2790
|
+
if (opts.resume) {
|
|
2791
|
+
const rows = journal.read();
|
|
2792
|
+
let resumedAt = rows.length - 1;
|
|
2793
|
+
while (resumedAt >= 0 && rows[resumedAt].event !== "run-resume")
|
|
2794
|
+
resumedAt--;
|
|
2795
|
+
const prior = rows.slice(0, resumedAt + 1);
|
|
2796
|
+
for (const t of graph.tasks) {
|
|
2797
|
+
const owned = interruptedAttempt(prior, t.id);
|
|
2798
|
+
if (owned?.launch)
|
|
2799
|
+
resumeInterrupted.set(t.id, { attempt: owned.attempt, nonce: owned.launch.nonce });
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
const reconcile = async (sweep) => {
|
|
2567
2803
|
if (keepForever)
|
|
2568
2804
|
return;
|
|
2569
2805
|
try {
|
|
2570
|
-
const
|
|
2806
|
+
const rows = journal.read();
|
|
2807
|
+
const desired = desiredPanes(rows, runId);
|
|
2808
|
+
for (const [taskId, frozen] of resumeInterrupted) {
|
|
2809
|
+
const owned = interruptedAttempt(rows, taskId);
|
|
2810
|
+
if (owned?.launch?.nonce !== frozen.nonce)
|
|
2811
|
+
resumeInterrupted.delete(taskId);
|
|
2812
|
+
else
|
|
2813
|
+
desired.add(formatOwnedName({ role: "worker", taskId, attempt: frozen.attempt, runId }));
|
|
2814
|
+
}
|
|
2571
2815
|
// The watch pane is never the DRIVER sweep's candidate (panesToClose spares role "watch":
|
|
2572
2816
|
// herdr's watches bookkeeping lives in close(), and a raw pane-close in the sweep would
|
|
2573
2817
|
// leave narrator() a stale cache) — the driver always sees it as desired; its lifecycle is
|
|
2574
2818
|
// decided here from the fold alone.
|
|
2575
|
-
await driver.reconcile?.(new Set([...desired, watchName]), runId, { ...
|
|
2819
|
+
await driver.reconcile?.(new Set([...desired, watchName]), runId, { ...sweep, endedRunIds });
|
|
2576
2820
|
// OBS-103: when the fold retires the watch (run-end boundary), close the narrator. The
|
|
2577
2821
|
// decision keys on the run identity in the pane name — narrator() adopts a prior daemon
|
|
2578
2822
|
// instance's pane under the same owned name, so a stop→resume cycle's leftover narrator
|
|
@@ -2662,6 +2906,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2662
2906
|
graph = setStatus(graph, t.id, "human");
|
|
2663
2907
|
saveGraph(repoRoot, graph);
|
|
2664
2908
|
journal.append("task-human", t.id, { ...details, reason, kind });
|
|
2909
|
+
// OBS-1178: the notice carries the park token an approval binds to (`approve --park <token>`).
|
|
2910
|
+
const parked = journal.newestBinding(t.id);
|
|
2911
|
+
const token = parked && bindingToken(parked);
|
|
2665
2912
|
if (assignment) {
|
|
2666
2913
|
// OBS-547: `metered` counts CHARGEABLE metered attempts, so an unchargeable dispatch passes 0 and
|
|
2667
2914
|
// the count is omitted rather than written as 0 or as `1` beside `attempts: 0` — a row claiming
|
|
@@ -2670,7 +2917,81 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2670
2917
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind, gateFails, consults, tokens, meteredAttempts: tokens && metered ? metered : undefined, retryMode });
|
|
2671
2918
|
}
|
|
2672
2919
|
await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
|
|
2673
|
-
await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}`, { tier: "attention" });
|
|
2920
|
+
await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}${token ? `\npark ${token} — bind the decision with \`--park ${token}\`` : ""}`, { tier: "attention" });
|
|
2921
|
+
};
|
|
2922
|
+
// OBS-1106 residual: a retry that landed nothing on a timeout-class red re-observes it ONCE fresh:
|
|
2923
|
+
// neither a journal replay nor the verdict cache may re-state that red. Owed when the prior round's
|
|
2924
|
+
// test red sits on this very subject, is timeout-shaped, and has not yet been observed twice
|
|
2925
|
+
// independently — once it has, the fingerprint cap owns the next move. Read from the journal alone,
|
|
2926
|
+
// so the live gate loop and a resume battery decide alike.
|
|
2927
|
+
const owesTimeoutRefresh = (t, priorRows, commit) => {
|
|
2928
|
+
const red = priorRows.find((e) => e.event === "gate-result" && e.data.gate === "test" && e.data.pass === false
|
|
2929
|
+
&& e.data.skipped !== true && e.data.commit === commit && typeof e.data.details === "string" && timeoutShaped(e.data.details));
|
|
2930
|
+
return red !== undefined
|
|
2931
|
+
&& identicalGateFailures(journal.read(), t.id, "test", normalizeGateFailure(String(red.data.details))) < GATE_FINGERPRINT_CAP;
|
|
2932
|
+
};
|
|
2933
|
+
// OBS-1106 residual: before a timeout or wall-budget red accompanied by runner infra diagnostics is
|
|
2934
|
+
// charged, its attributed failing files are re-run ONCE, on the same checkout, under the suite
|
|
2935
|
+
// lease. Only a reproduced assertion is charged. A pass, an infra rerun, a rerun naming none of the
|
|
2936
|
+
// files, or an unattributable red parks with the ORIGINAL red journaled and unresolved: the rerun is
|
|
2937
|
+
// diagnostic, never a verdict, so it can neither green the gate nor permit a merge. Returns true
|
|
2938
|
+
// when it parked.
|
|
2939
|
+
const adjudicateInfraShapedRed = async (t, results, wt, subject, attempt, replayed, parkInfra) => {
|
|
2940
|
+
const red = results.find((g) => g.gate === "test" && gateFailed(g) && !isInfraResult(g)
|
|
2941
|
+
&& timeoutShaped(g.details) && RUNNER_INFRA_DIAGNOSTIC_RE.test(g.details));
|
|
2942
|
+
if (!red || commands.test === undefined)
|
|
2943
|
+
return false;
|
|
2944
|
+
// The observation this red states: its subject, its fingerprint and, when it carries one, its
|
|
2945
|
+
// receipt's invocation. A copy (journal replay, cache hit) is never re-run when THAT observation's
|
|
2946
|
+
// adjudication is on the ledger — its recorded outcome decides, so a crash after the adjudication
|
|
2947
|
+
// but before its park still parks. A crash before the adjudication leaves none, so the copy is
|
|
2948
|
+
// adjudicated now. A fresh red always is.
|
|
2949
|
+
const fingerprint = normalizeGateFailure(red.details);
|
|
2950
|
+
const observation = receiptOrigin({ evidenceReceipt: red.evidenceReceipt ?? red.meta?.evidenceReceipt,
|
|
2951
|
+
evidenceReceipts: red.evidenceReceipts ?? red.meta?.evidenceReceipts });
|
|
2952
|
+
const recorded = replayed || red.meta?.reused === true
|
|
2953
|
+
? journal.read().reverse().find((e) => e.taskId === t.id && e.event === "gate-reobserved"
|
|
2954
|
+
&& e.data.commit === subject && e.data.fingerprint === fingerprint
|
|
2955
|
+
&& (observation === undefined || e.data.observation === undefined || e.data.observation === observation))
|
|
2956
|
+
: undefined;
|
|
2957
|
+
let files = [];
|
|
2958
|
+
let outcome = "unattributed";
|
|
2959
|
+
let selection;
|
|
2960
|
+
if (recorded) {
|
|
2961
|
+
files = Array.isArray(recorded.data.files) ? recorded.data.files.filter((f) => typeof f === "string") : [];
|
|
2962
|
+
outcome = ["reproduced", "passed", "unattributed"].includes(String(recorded.data.outcome)) ? recorded.data.outcome : "ambiguous";
|
|
2963
|
+
}
|
|
2964
|
+
else {
|
|
2965
|
+
for (const file of attributedFailingFiles(red)) {
|
|
2966
|
+
if (posix.normalize(file) === file && !file.startsWith("../") && !file.startsWith("/") && existsSync(join(wt, file)))
|
|
2967
|
+
files.push(file);
|
|
2968
|
+
}
|
|
2969
|
+
let rerun;
|
|
2970
|
+
if (files.length) {
|
|
2971
|
+
rerun = await withCommandContext(t.id, () => reobserveTestFiles(wt, commands.test, baseline, files, journal.dir))
|
|
2972
|
+
.catch((error) => ({ gate: "test", pass: false, details: `infra: rerun failed: ${error instanceof Error ? error.message : String(error)}`, meta: { infra: true } }));
|
|
2973
|
+
classifyInfraResult(rerun);
|
|
2974
|
+
// Reproduced only by failure evidence the rerun ATTRIBUTES to one of exactly these files, from an
|
|
2975
|
+
// OBSERVED selection that was exactly these files (D-607): a path merely printed in output, a
|
|
2976
|
+
// positional filter that also collected another file, or a runner that ignored the file
|
|
2977
|
+
// arguments and reported no manifest of its selection, says nothing about the original red.
|
|
2978
|
+
const failing = rerunFailingFiles(rerun);
|
|
2979
|
+
selection = observedSelection(rerun);
|
|
2980
|
+
const isolated = selection !== undefined && selection.length === files.length
|
|
2981
|
+
&& files.every((file) => selection.includes(file)) && failing.every((file) => files.includes(file));
|
|
2982
|
+
outcome = rerun.pass ? "passed" : isolated && !isInfraResult(rerun) && failing.length ? "reproduced" : "ambiguous";
|
|
2983
|
+
}
|
|
2984
|
+
journal.append("gate-reobserved", t.id, {
|
|
2985
|
+
gate: "test", attempt, commit: subject, fingerprint, ...(observation ? { observation } : {}), files, outcome,
|
|
2986
|
+
...(selection ? { selection } : {}),
|
|
2987
|
+
...(rerun ? { pass: rerun.pass, details: rerun.details } : {}),
|
|
2988
|
+
...(rerun?.evidenceReceipt ? { evidenceReceipt: rerun.evidenceReceipt } : {}),
|
|
2989
|
+
});
|
|
2990
|
+
}
|
|
2991
|
+
if (outcome === "reproduced")
|
|
2992
|
+
return false;
|
|
2993
|
+
await parkInfra(`test: timeout-shaped red with runner infra diagnostics was not reproduced by one isolated rerun of ${files.length ? files.join(", ") : "its attributed files (none attributable)"} (${outcome}); the original red stands unresolved — recheck to re-measure:\n${readableExcerpt(red.details)}`);
|
|
2994
|
+
return true;
|
|
2674
2995
|
};
|
|
2675
2996
|
// OBS-547: cross-reference a scope red against the prediction this run already computed. Every hard
|
|
2676
2997
|
// offender predicted ⇒ an AUTHORING defect whose repair is pre-written: journal the classification
|
|
@@ -2690,7 +3011,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2690
3011
|
const verdict = scopeRed?.meta?.collateral;
|
|
2691
3012
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
2692
3013
|
const disposition = classifyRepairDisposition({
|
|
2693
|
-
|
|
3014
|
+
// OBS-1072 add.2: hints are lexed from the readable excerpt, never from terminal control bytes.
|
|
3015
|
+
results: results.map((g) => ({ ...g, details: readableExcerpt(g.details) })), files: t.files, inventory: changed,
|
|
2694
3016
|
refusalSummary: worker?.data.ok === false && typeof worker.data.summary === "string" ? worker.data.summary : undefined,
|
|
2695
3017
|
});
|
|
2696
3018
|
for (const diagnostic of disposition.diagnostics) {
|
|
@@ -2749,7 +3071,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2749
3071
|
let admissionPriority = batteryPriority(startupActions.values());
|
|
2750
3072
|
const admissible = () => readyTasks(graph, admissionPriority);
|
|
2751
3073
|
const sweepLiveApprovals = () => {
|
|
2752
|
-
|
|
3074
|
+
let events = journal.read();
|
|
3075
|
+
if (events.slice(approvalSweepCursor).some((e) => e.event === "task-approved")) {
|
|
3076
|
+
refuseStaleApprovals();
|
|
3077
|
+
events = journal.read();
|
|
3078
|
+
}
|
|
2753
3079
|
admissionPriority = batteryPriority(pendingDaemonApprovalActions(events).values());
|
|
2754
3080
|
const approvals = events.slice(approvalSweepCursor)
|
|
2755
3081
|
.filter((e) => e.event === "task-approved" && e.taskId);
|
|
@@ -2798,7 +3124,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2798
3124
|
if (changed)
|
|
2799
3125
|
saveGraph(repoRoot, graph);
|
|
2800
3126
|
};
|
|
3127
|
+
// OBS-1168(c): every gate callback is bound to the engagement that asked for it. Once the scheduler
|
|
3128
|
+
// settles that engagement (done, parked or failed), a late sibling's event is dropped, never journaled.
|
|
3129
|
+
const engagements = new Map();
|
|
2801
3130
|
const execTask = async (t) => {
|
|
3131
|
+
const engagement = {};
|
|
3132
|
+
engagements.set(t.id, engagement);
|
|
3133
|
+
const liveEngagement = () => engagements.get(t.id) === engagement;
|
|
2802
3134
|
const approvalAction = pendingDaemonApprovalActions(journal.read()).get(t.id);
|
|
2803
3135
|
const fundedRerun = approvalAction?.authority === "worker";
|
|
2804
3136
|
const startMs = Date.now();
|
|
@@ -2861,6 +3193,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2861
3193
|
// fresh-budget release, prefer nextChannel over the surviving tried-list so burned channels are
|
|
2862
3194
|
// not re-tried first (consult bans / prior failovers survive the release).
|
|
2863
3195
|
const rs = resume.get(t.id);
|
|
3196
|
+
let bootstrapExhausted;
|
|
2864
3197
|
const contentDigest = taskContentDigest(t);
|
|
2865
3198
|
const previousDispatch = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "task-dispatch");
|
|
2866
3199
|
const recordedGraphPath = join(journal.dir, "graph.json");
|
|
@@ -2881,8 +3214,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2881
3214
|
// OBS-1161: no `attempts > 0` guard — every release already clears lastAssignment in the replay,
|
|
2882
3215
|
// and a lastAssignment at zero attempts is a first dispatch whose capacity requeue was taken back:
|
|
2883
3216
|
// the seat is still in force, so restore it instead of failing over its own tried[] entry early.
|
|
3217
|
+
// OBS-1187: a restored seat must still be a member of the task's pool (the operator may have closed it).
|
|
2884
3218
|
if (!hintsChanged && rs?.lastAssignment
|
|
2885
3219
|
&& channels.some((c) => channelKey(c) === channelKey(rs.lastAssignment))
|
|
3220
|
+
&& inTaskPool(t, cfg, channelKey(rs.lastAssignment))
|
|
2886
3221
|
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
2887
3222
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
2888
3223
|
}
|
|
@@ -2892,29 +3227,60 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2892
3227
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
2893
3228
|
// OBS-1034: a PIN is exempt — route() already returned it, and a review-upheld repair of the diff
|
|
2894
3229
|
// the pin produced belongs on the pin, not on the next seat its own tried[] entry would pick.
|
|
2895
|
-
|
|
3230
|
+
// OBS-1169: an adapter a bootstrap failover escalated away from stays out, sibling or none.
|
|
3231
|
+
const escalatedOut = channels.filter((c) => rs.escalatedAdapters?.includes(c.adapter)).map(channelKey);
|
|
3232
|
+
const next = nextChannel(assignment, t, cfg, channels, [...rs.tried, ...Object.keys(rs.escalated ?? {}), ...escalatedOut], profile, demotedChannels);
|
|
2896
3233
|
if (next)
|
|
2897
3234
|
assignment = next;
|
|
2898
3235
|
// ponytail: nextChannel null (every channel already tried / none available) — keep the static
|
|
2899
3236
|
// assignment and proceed. Dispatching on a previously-tried channel beats deadlocking a resumed
|
|
2900
3237
|
// run; a park-instead policy can come later if it ever bites.
|
|
3238
|
+
// OBS-1169: …except onto an escalated adapter — that relaunches the broken bootstrap, so it parks infra below.
|
|
3239
|
+
else if (rs.escalatedAdapters?.includes(assignment.adapter))
|
|
3240
|
+
bootstrapExhausted = channelKey(assignment);
|
|
2901
3241
|
}
|
|
2902
|
-
const taskHistory = journal.read().filter((e) => e.taskId === t.id);
|
|
3242
|
+
const taskHistory = effectiveEvents(journal.read()).filter((e) => e.taskId === t.id); // OBS-1178: only an effective decision opens an engagement
|
|
2903
3243
|
// Lifetime identity counts every dispatch, including legacy and unchargeable rows. Releases
|
|
2904
3244
|
// reset the budget below, never this counter; observational annotations do not consume it.
|
|
2905
3245
|
let nextWorkerDispatchOrdinal = taskHistory.filter((e) => e.event === "task-dispatch").length;
|
|
2906
3246
|
const lastApproval = taskHistory.map((e) => e.event).lastIndexOf("task-approved");
|
|
2907
3247
|
const priorClimb = taskHistory.slice(lastApproval + 1).reverse().find((e) => e.event === "tier-escalated");
|
|
2908
3248
|
if (!hintsChanged && priorClimb && taskHistory.indexOf(priorClimb) > taskHistory.map((e) => e.event).lastIndexOf("task-dispatch")) {
|
|
2909
|
-
const target = channels.find((c) => channelKey(c) === priorClimb.data.to && !demotedChannels.has(channelKey(c))
|
|
3249
|
+
const target = channels.find((c) => channelKey(c) === priorClimb.data.to && !demotedChannels.has(channelKey(c))
|
|
3250
|
+
&& inTaskPool(t, cfg, channelKey(c)));
|
|
2910
3251
|
if (target)
|
|
2911
|
-
assignment =
|
|
3252
|
+
assignment = seatAssignment(target);
|
|
2912
3253
|
}
|
|
2913
3254
|
// pre-kill invariant: tried always contains the current assignment. Spread, never alias the
|
|
2914
3255
|
// journal-derived array (no hidden mutation of replayed state).
|
|
2915
3256
|
const tried = rs?.tried.length ? [...rs.tried] : [channelKey(assignment)];
|
|
2916
3257
|
if (!tried.includes(channelKey(assignment)))
|
|
2917
3258
|
tried.push(channelKey(assignment));
|
|
3259
|
+
// OBS-1169 add.1: channels excluded because their whole adapter was escalated away from ride
|
|
3260
|
+
// `tried` (nextChannel's one exclusion input) but were never launched — keep the true reason so
|
|
3261
|
+
// the dispatch row never labels an unlaunched sibling "already tried".
|
|
3262
|
+
const vendorEscalated = new Map();
|
|
3263
|
+
// …and a resume restores them with their reasons, apart from the channels actually dispatched.
|
|
3264
|
+
for (const [k, why] of Object.entries(rs?.escalated ?? {})) {
|
|
3265
|
+
if (tried.includes(k))
|
|
3266
|
+
continue;
|
|
3267
|
+
tried.push(k);
|
|
3268
|
+
vendorEscalated.set(k, why);
|
|
3269
|
+
}
|
|
3270
|
+
// OBS-1169 add.2: the ADAPTER exclusion itself, apart from any sibling channel — a single-channel
|
|
3271
|
+
// adapter has no untried sibling to carry it, yet its exhausted channel must stay out of the
|
|
3272
|
+
// recycle pool (and a resume's) until the release that clears `tried`.
|
|
3273
|
+
const escalatedAdapters = new Set(rs?.escalatedAdapters ?? []);
|
|
3274
|
+
const escalateAdapter = (adapter, reason) => {
|
|
3275
|
+
escalatedAdapters.add(adapter);
|
|
3276
|
+
for (const c of channels) {
|
|
3277
|
+
const k = channelKey(c);
|
|
3278
|
+
if (c.adapter !== adapter || tried.includes(k))
|
|
3279
|
+
continue;
|
|
3280
|
+
tried.push(k);
|
|
3281
|
+
vendorEscalated.set(k, reason);
|
|
3282
|
+
}
|
|
3283
|
+
};
|
|
2918
3284
|
// VIS-02 convention: absence = no seeding happened. The observable surface for criterion 2's
|
|
2919
3285
|
// exclusion-list-equality oracle. Daemon-side append only — no journal.ts write-path change (Phase 48
|
|
2920
3286
|
// stays unblocked); inert to replayStatuses (unknown events ignored, pinned at journal.test.ts:70-80).
|
|
@@ -2923,6 +3289,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2923
3289
|
attempts: rs.attempts, tried: [...tried], assignment,
|
|
2924
3290
|
workerDispatchOrdinal: previousDispatch?.data.workerDispatchOrdinal ?? null,
|
|
2925
3291
|
});
|
|
3292
|
+
if (bootstrapExhausted === channelKey(assignment)) { // a later tier-climb restore may have moved it off
|
|
3293
|
+
await park(t, `${bootstrapExhausted} failed at bootstrap and no eligible channel remains after resume${poolExhaustion(t, cfg)}`, "infra", assignment, rs?.attempts ?? 0, startMs, 0, 0, undefined, 0, "fresh", { cause: "bootstrap", channel: bootstrapExhausted });
|
|
3294
|
+
return;
|
|
3295
|
+
}
|
|
2926
3296
|
// Keep one live list: recovery retries must see exclusions added by onGate during the round.
|
|
2927
3297
|
const badReviewers = [...replayedReviewerExclusions];
|
|
2928
3298
|
const noteReviewEvent = (e) => {
|
|
@@ -2931,18 +3301,33 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2931
3301
|
if (!badReviewers.includes(e.payload.reviewer))
|
|
2932
3302
|
badReviewers.push(e.payload.reviewer);
|
|
2933
3303
|
}
|
|
3304
|
+
// OBS-1196: a seat whose same-subject re-emission delivered a valid verdict was a delivery flake,
|
|
3305
|
+
// not a bad reviewer — it stays seatable for this task's later rounds.
|
|
3306
|
+
if (e.name === "review-reemission" && e.payload.delivered === true && typeof e.payload.reviewer === "string") {
|
|
3307
|
+
const at = badReviewers.indexOf(e.payload.reviewer);
|
|
3308
|
+
if (at >= 0)
|
|
3309
|
+
badReviewers.splice(at, 1);
|
|
3310
|
+
}
|
|
2934
3311
|
};
|
|
2935
3312
|
// Keep the measured siblings, including a red judge, while replacing only a missing review.
|
|
2936
3313
|
// Every dispatched seat is excluded for this task, so the loop is bounded by the eligible pool.
|
|
2937
3314
|
const runReviewRecovery = async (task, ctx, allowSelection = true) => {
|
|
2938
3315
|
ctx.authorizeInfraRetry = cfg.executionPolicy?.boundedInfrastructure
|
|
2939
3316
|
? (subject, cause) => reserveInfrastructureRetry(journal.read(), task.id, subject, journal.append.bind(journal), cause) : undefined;
|
|
2940
|
-
if (
|
|
3317
|
+
if (repairSelection && allowSelection) {
|
|
2941
3318
|
const decision = repairSelectionDecision(journal.read(), task.id, true);
|
|
2942
3319
|
ctx.selectTests = decision.selectTests;
|
|
2943
3320
|
ctx.requiredRepairTests = decision.requiredFiles;
|
|
2944
3321
|
ctx.selectionReason = decision.reason;
|
|
2945
3322
|
}
|
|
3323
|
+
else if (repairSelection) {
|
|
3324
|
+
// A recheck runs the full suite; its recorded scope keeps a red there attributable.
|
|
3325
|
+
ctx.selectTests = false;
|
|
3326
|
+
ctx.selectionReason = "recheck-full-suite";
|
|
3327
|
+
}
|
|
3328
|
+
// OBS-1151: every round judges fresh; the journal's earlier judgments (this run and before a
|
|
3329
|
+
// resume) are only the subjects a reversed ruling is compared against.
|
|
3330
|
+
ctx.priorJudgments = priorJudgments(journal.read(), task.id);
|
|
2946
3331
|
// Resume and ordinary verification share this boundary. Count persisted rounds so a
|
|
2947
3332
|
// resumed daemon cannot reuse a previous round's identity within the same attempt.
|
|
2948
3333
|
ctx.buildReceiptIdentity = {
|
|
@@ -2977,7 +3362,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2977
3362
|
// reviewer-exclusion list (badReviewers), never a second parallel counter. OBS-189: scoped to the
|
|
2978
3363
|
// current engagement — an operator approval (uphold or accept) resets the round budget, so an upheld
|
|
2979
3364
|
// task can dispatch its funded attempt instead of re-parking against the whole journal's history.
|
|
2980
|
-
const reviewRoundsDrawn = () => reviewRoundsSinceApproval(
|
|
3365
|
+
const reviewRoundsDrawn = () => reviewRoundsSinceApproval(journal.read(), t.id, decisiveReviewRounds);
|
|
2981
3366
|
// OBS-193: journal the in-gate review retry (mirrors judge-retry) and exclude the flaked seat from
|
|
2982
3367
|
// later attempts' reviewer picks. One helper, called from both onGate sites (satisfied-gate + main).
|
|
2983
3368
|
const noteReviewRetry = (g) => {
|
|
@@ -3045,8 +3430,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3045
3430
|
...(typeof g.meta.producer === "string" ? { producer: g.meta.producer } : {}),
|
|
3046
3431
|
...(typeof g.meta.producerAttempt === "number" ? { producerAttempt: g.meta.producerAttempt } : {}),
|
|
3047
3432
|
} : {}),
|
|
3048
|
-
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
3049
|
-
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision", "failureEvidence"
|
|
3433
|
+
...((cfg.executionPolicy || repairSelection) && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
3434
|
+
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision", "failureEvidence",
|
|
3435
|
+
"forgivenFingerprints", "freshFingerprints", "baselineProvenance", "judgment", "adjudication", "judgeDisagreement", "interruptions"]
|
|
3050
3436
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
3051
3437
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
3052
3438
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
@@ -3072,7 +3458,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3072
3458
|
} : {}),
|
|
3073
3459
|
...(g.gate === "review" && typeof g.meta?.reviewer === "string" ? {
|
|
3074
3460
|
reviewer: g.meta.reviewer,
|
|
3075
|
-
...Object.fromEntries(["cause", "seatAuthoredBytes", "bytes", "rawPath", "briefPath", "timeoutMs", "unparseable", "noVerdict", "resolved", "reraised", "reviewerFloor", "reviewerFloorCause", "reviewerTier"]
|
|
3461
|
+
...Object.fromEntries(["cause", "seatAuthoredBytes", "bytes", "rawPath", "briefPath", "timeoutMs", "unparseable", "noVerdict", "resolved", "reraised", "settledAnchors", "reviewerFloor", "reviewerFloorCause", "reviewerTier"]
|
|
3076
3462
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
3077
3463
|
...(typeof g.meta.vendor === "string" ? { vendor: g.meta.vendor } : {}),
|
|
3078
3464
|
...(typeof g.meta.provider === "string" ? { provider: g.meta.provider } : {}),
|
|
@@ -3169,7 +3555,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3169
3555
|
// OBS-254: RE-DERIVED from the journal here, at prompt-build time, rather than trusted to survive
|
|
3170
3556
|
// in resume state. The journal already holds the upheld review's bytes; no reset of attempt or
|
|
3171
3557
|
// channel state can take them away, on any path, including `resume --retry-failed`.
|
|
3172
|
-
const upheldFeedback = upheldFeedbackByTask(journal.read()).get(t.id) ?? rs?.upheldFeedback;
|
|
3558
|
+
const upheldFeedback = upheldFeedbackByTask(effectiveEvents(journal.read())).get(t.id) ?? rs?.upheldFeedback; // OBS-1178: a refused uphold funds no brief
|
|
3173
3559
|
const carriedEvidence = priorRunEvidence.findings
|
|
3174
3560
|
.filter((finding) => finding.taskId === t.id)
|
|
3175
3561
|
.map(formatPriorFindingEvidence)
|
|
@@ -3184,7 +3570,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3184
3570
|
: "");
|
|
3185
3571
|
let ladderIdx = 0;
|
|
3186
3572
|
const engagementRows = () => {
|
|
3187
|
-
const rows = journal.read().filter((e) => e.taskId === t.id);
|
|
3573
|
+
const rows = effectiveEvents(journal.read()).filter((e) => e.taskId === t.id);
|
|
3188
3574
|
const approval = rows.map((e) => e.event).lastIndexOf("task-approved");
|
|
3189
3575
|
return rows.slice(approval + 1);
|
|
3190
3576
|
};
|
|
@@ -3206,7 +3592,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3206
3592
|
return;
|
|
3207
3593
|
const from = channelKey(assignment);
|
|
3208
3594
|
const to = channelKey(next);
|
|
3209
|
-
const poolBefore = channels.filter((c) => !tried.includes(channelKey(c)) && !demotedChannels.has(channelKey(c))
|
|
3595
|
+
const poolBefore = channels.filter((c) => !tried.includes(channelKey(c)) && !demotedChannels.has(channelKey(c))
|
|
3596
|
+
&& inTaskPool(t, cfg, channelKey(c))).map(channelKey);
|
|
3210
3597
|
journal.append("tier-escalated", t.id, {
|
|
3211
3598
|
attempt, cause, gate: gate.gate, fingerprint: normalizeGateFailure(gate.details).slice(0, 500),
|
|
3212
3599
|
from, to, fromTier: assignment.tier, toTier: next.tier, poolBefore,
|
|
@@ -3216,7 +3603,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3216
3603
|
if (c.tier === assignment.tier && channelKey(c) !== to)
|
|
3217
3604
|
climbSkips.add(channelKey(c));
|
|
3218
3605
|
climbProvenance = `tier-escalated ${from} → ${to} (${cause})`;
|
|
3219
|
-
assignment =
|
|
3606
|
+
assignment = seatAssignment(next);
|
|
3220
3607
|
tried.push(to);
|
|
3221
3608
|
return "climbed";
|
|
3222
3609
|
};
|
|
@@ -3250,12 +3637,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3250
3637
|
}
|
|
3251
3638
|
return next;
|
|
3252
3639
|
};
|
|
3253
|
-
// OBS-1161: capacity requeues spent on a seat for this task since
|
|
3254
|
-
// journal-derived so the budget survives a resume instead of restarting
|
|
3255
|
-
const
|
|
3256
|
-
const rows = journal.read().filter((e) => e.taskId === t.id);
|
|
3640
|
+
// OBS-1161: capacity requeues (OBS-1169: and bootstrap retries) spent on a seat for this task since
|
|
3641
|
+
// its last operator release — journal-derived so the budget survives a resume instead of restarting.
|
|
3642
|
+
const requeuesOn = (event, channel) => {
|
|
3643
|
+
const rows = effectiveEvents(journal.read()).filter((e) => e.taskId === t.id);
|
|
3257
3644
|
const since = rows.map((e) => e.event).lastIndexOf("task-approved");
|
|
3258
|
-
return rows.slice(since + 1).filter((e) => e.event ===
|
|
3645
|
+
return rows.slice(since + 1).filter((e) => e.event === event && e.data.channel === channel).length;
|
|
3259
3646
|
};
|
|
3260
3647
|
// OBS-202 (operator law: "you can spawn as many as you want"): channels are session FACTORIES,
|
|
3261
3648
|
// not consumed seats — a tried channel can always host a fresh worker session, and a fresh
|
|
@@ -3264,19 +3651,25 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3264
3651
|
// on artificial scarcity; MAX_ATTEMPTS and the review round cap are the real bounds. Only a
|
|
3265
3652
|
// fleet whose every channel is DEMOTED (verified dead: auth/setup/provider outage) has nothing
|
|
3266
3653
|
// left to spawn — that case alone still parks.
|
|
3654
|
+
// OBS-1187: shared with the demotion seam's pooled recycle so neither relaunches an escalated adapter.
|
|
3655
|
+
const recycleExcluded = () => [channelKey(assignment), ...channels.filter((c) => escalatedAdapters.has(c.adapter)).map(channelKey)];
|
|
3267
3656
|
const failoverOrRecycle = (site) => {
|
|
3268
3657
|
const next = failover(site);
|
|
3269
3658
|
if (next)
|
|
3270
3659
|
return next;
|
|
3271
|
-
|
|
3660
|
+
// OBS-1169: a vendor escalated away from stays out of the recycle pool until its release —
|
|
3661
|
+
// recycling any of its channels, launched or not, would launch the broken bootstrap again.
|
|
3662
|
+
const recycled = nextChannel(assignment, t, cfg, channels, recycleExcluded(), profile, demotedChannels)
|
|
3272
3663
|
?? (demotedChannels.has(channelKey(assignment)) ? null : assignment);
|
|
3273
3664
|
if (recycled)
|
|
3274
3665
|
journal.append("channel-recycle", t.id, { site, channel: channelKey(recycled) });
|
|
3275
3666
|
return recycled;
|
|
3276
3667
|
};
|
|
3277
|
-
const runConsult = (trigger, transcript, diffOrFeedback, gates) => {
|
|
3668
|
+
const runConsult = async (trigger, transcript, diffOrFeedback, gates) => {
|
|
3278
3669
|
consults++;
|
|
3279
|
-
|
|
3670
|
+
// OBS-1182: every launched consult seat, failed ones included, rides the verdict into the journal.
|
|
3671
|
+
const invocations = [];
|
|
3672
|
+
const v = await consult({
|
|
3280
3673
|
taskId: t.id, trigger,
|
|
3281
3674
|
journalTail: JSON.stringify(journal.read().slice(-20)),
|
|
3282
3675
|
transcript: transcript.slice(-8000),
|
|
@@ -3285,7 +3678,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3285
3678
|
// D-07: consult panes self-clean when the verdict is read (keepLlm) — only "forever" keeps them.
|
|
3286
3679
|
// v1.54 T1: channels = this run's doctor-filtered live list — consult.prefer seat liveness
|
|
3287
3680
|
// is judged against it, never rebuilt from config (installed-but-unauthed seats would stall).
|
|
3288
|
-
{ keep: keepLlm, onSlot: keepLlm ? (s) => keptSlots.push(s) : undefined, runId, channels: pools.consult });
|
|
3681
|
+
{ keep: keepLlm, onSlot: keepLlm ? (s) => keptSlots.push(s) : undefined, onInvocation: (inv) => invocations.push(inv), runId, channels: pools.consult, health });
|
|
3682
|
+
// A lone answering seat is already the row's adapter/model/vendor/effort; the list is journaled
|
|
3683
|
+
// only when a seat failed, so the row never repeats itself and a failed seat never vanishes.
|
|
3684
|
+
return invocations.some((inv) => inv.outcome === "failed") ? { ...v, invocations } : v;
|
|
3289
3685
|
};
|
|
3290
3686
|
// returns true → continue attempting, false → task is terminal (parked)
|
|
3291
3687
|
// trigger (why the consult ran) is threaded in so the decompose/human park keeps its cause —
|
|
@@ -3294,9 +3690,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3294
3690
|
journal.append("consult-verdict", t.id, {
|
|
3295
3691
|
action: v.action, notes: v.notes,
|
|
3296
3692
|
adapter: v.adapter ?? "unknown", model: v.model ?? "unknown", vendor: v.vendor ?? "unknown",
|
|
3693
|
+
...(v.effort ? { effort: v.effort } : {}),
|
|
3297
3694
|
...(v.reason ? { reason: v.reason } : {}),
|
|
3298
3695
|
...(v.guidance ? { guidance: v.guidance } : {}),
|
|
3299
3696
|
...(v.excludeAdapter ? { excludeAdapter: v.excludeAdapter } : {}),
|
|
3697
|
+
...(v.invocations?.length ? { invocations: v.invocations } : {}),
|
|
3300
3698
|
});
|
|
3301
3699
|
await driver.notify(`tickmarkr ${runId}: ${t.id} consult verdict: ${v.action}`, { tier: "attention" });
|
|
3302
3700
|
if (v.action === "retry") {
|
|
@@ -3316,15 +3714,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3316
3714
|
// nextChannel's existing tried parameter — zero router changes (D-03). Unknown adapter
|
|
3317
3715
|
// (zero matches) is a no-op expansion ⇒ ordinary channel-level reroute. Task-scoped:
|
|
3318
3716
|
// `tried` lives inside execTask, so a sibling task is unaffected.
|
|
3319
|
-
if (v.excludeAdapter)
|
|
3320
|
-
|
|
3321
|
-
if (c.adapter === v.excludeAdapter) {
|
|
3322
|
-
const k = channelKey(c);
|
|
3323
|
-
if (!tried.includes(k))
|
|
3324
|
-
tried.push(k);
|
|
3325
|
-
}
|
|
3326
|
-
}
|
|
3327
|
-
}
|
|
3717
|
+
if (v.excludeAdapter)
|
|
3718
|
+
escalateAdapter(v.excludeAdapter, `vendor escalated: consult excluded ${v.excludeAdapter}`);
|
|
3328
3719
|
const next = failoverOrRecycle("consult-reroute");
|
|
3329
3720
|
if (next) {
|
|
3330
3721
|
assignment = next;
|
|
@@ -3333,7 +3724,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3333
3724
|
tried.push(k);
|
|
3334
3725
|
return true;
|
|
3335
3726
|
}
|
|
3336
|
-
await park(t,
|
|
3727
|
+
await park(t, `consult said reroute but every channel is demoted (verified dead) — nothing left to spawn${poolExhaustion(t, cfg)}`, "reroute-exhausted", assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3337
3728
|
return false;
|
|
3338
3729
|
}
|
|
3339
3730
|
await park(t, `consult verdict: ${v.action} — ${v.notes}`, trigger, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode); // decompose|human
|
|
@@ -3345,7 +3736,157 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3345
3736
|
let satisfiedGate = satisfiedGates.get(t.id);
|
|
3346
3737
|
const replayedGates = fundedRerun ? undefined : replayedGateResults.get(t.id);
|
|
3347
3738
|
const recheck = approvalAction?.authority === "battery";
|
|
3348
|
-
|
|
3739
|
+
// OBS-1109: finished work is harvested, never redone — on resume too. A daemon that died while its
|
|
3740
|
+
// worker ran leaves an owned attempt between worker-launch and worker-result: its pane may hold the
|
|
3741
|
+
// attempt's own nonce-bound trailer, its branch the committed work. Only after the attempt's owned
|
|
3742
|
+
// processes are reaped is either read as its result — a matching trailer as the worker's claim,
|
|
3743
|
+
// commits without one as the same no-trailer evidence the live harvest synthesizes — and the gate
|
|
3744
|
+
// replay below then judges it under the PRODUCING attempt's author, with no new dispatch. An
|
|
3745
|
+
// unfinished attempt, a pane holding another attempt's trailer, or cleanup that cannot be proven
|
|
3746
|
+
// is declined onto the ordinary recovery path. A harvest an earlier resume recorded is gated again,
|
|
3747
|
+
// never recorded twice.
|
|
3748
|
+
const harvestInterruptedAttempt = async () => {
|
|
3749
|
+
const found = interruptedAttempt(journal.read(), t.id);
|
|
3750
|
+
if (!found || (!found.launch && !found.result))
|
|
3751
|
+
return found?.assignment;
|
|
3752
|
+
const { attempt } = found;
|
|
3753
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
3754
|
+
const recordHarvest = (commits, finished, summary) => journal.append("worker-result-harvested", t.id, {
|
|
3755
|
+
attempt, commits, summary: finished ? summary : HARVESTED_RESULT_SUMMARY, source: RESUME_HARVEST_SOURCE, trailer: finished,
|
|
3756
|
+
});
|
|
3757
|
+
// A declined attempt leaves the fold's spare (interruptedAttempt closes on this row), so its pane
|
|
3758
|
+
// is swept here, before the ordinary recovery path dispatches beside it — the baseline sweep, late.
|
|
3759
|
+
const decline = async (reason) => {
|
|
3760
|
+
journal.append("resume-harvest-declined", t.id, { attempt, reason });
|
|
3761
|
+
await reconcile({ spareLiveLlm: true });
|
|
3762
|
+
return undefined;
|
|
3763
|
+
};
|
|
3764
|
+
// Owned-process cleanup before any result is read as final: the attempt's own process group and
|
|
3765
|
+
// dispatch marker, reaped through the same reapWorker a live harvest uses. Uncertain is declined.
|
|
3766
|
+
const reapOwned = async (key, launch) => {
|
|
3767
|
+
workerOwners.set(key, { taskId: t.id, attempt, groupFile: `${launch.dispatchScript}.${launch.nonce}.pgid`,
|
|
3768
|
+
marker: launch.dispatchScript, identities: new Map(), descendants: new Map() });
|
|
3769
|
+
try {
|
|
3770
|
+
await reapWorker(key);
|
|
3771
|
+
return undefined;
|
|
3772
|
+
}
|
|
3773
|
+
catch (error) {
|
|
3774
|
+
return `owned-process cleanup uncertain: ${error instanceof Error ? error.message : String(error)}`;
|
|
3775
|
+
}
|
|
3776
|
+
};
|
|
3777
|
+
if (found.result) {
|
|
3778
|
+
// A daemon recorded this attempt's result, then died before its harvest row (a live daemon
|
|
3779
|
+
// before its no-trailer synthesis, or a resume between its two rows): finish that record from
|
|
3780
|
+
// the result it wrote — the pane is never re-read, the result never rewritten. A resume wrote
|
|
3781
|
+
// its result only after reaping; a live daemon writes it BEFORE handling a failed reap, so its
|
|
3782
|
+
// owned processes are reaped again here. A result that neither finished nor left commits has
|
|
3783
|
+
// nothing to gate: the ordinary recovery path owns it.
|
|
3784
|
+
if (!found.result.reaped) {
|
|
3785
|
+
if (!found.launch)
|
|
3786
|
+
return decline("owned-process cleanup unproven: the launch recorded no ownership evidence");
|
|
3787
|
+
const uncertain = await reapOwned(found.launch.slot, found.launch);
|
|
3788
|
+
if (uncertain)
|
|
3789
|
+
return decline(uncertain);
|
|
3790
|
+
}
|
|
3791
|
+
const commits = existsSync(wt) ? await commitsAheadOf(await integrationHead(intWt), wt) : [];
|
|
3792
|
+
if (!found.result.finished && commits.length === 0)
|
|
3793
|
+
return decline("unfinished");
|
|
3794
|
+
recordHarvest(commits, found.result.finished, found.result.summary);
|
|
3795
|
+
return found.assignment;
|
|
3796
|
+
}
|
|
3797
|
+
const launch = found.launch;
|
|
3798
|
+
const { nonce, slot } = launch;
|
|
3799
|
+
if (!existsSync(wt))
|
|
3800
|
+
return decline("task worktree missing");
|
|
3801
|
+
const adapter = adapters.find((a) => a.id === found.assignment.adapter);
|
|
3802
|
+
if (!adapter)
|
|
3803
|
+
return decline(`adapter ${found.assignment.adapter} unavailable to read the trailer`);
|
|
3804
|
+
// The journaled slot is read through THIS daemon's driver, a fresh instance since the last one
|
|
3805
|
+
// died. A pane the driver cannot read is DECLINED, never classified: an unreadable pane proves
|
|
3806
|
+
// neither a matching trailer nor the absence of a foreign one, so it is neither harvested nor
|
|
3807
|
+
// gated as trailerless evidence. A driver whose terminals outlive it (Orca) adopts the owned
|
|
3808
|
+
// terminal (Orca) or pane (Herdr, OBS-1203): bound only on ownership evidence — the journaled owned
|
|
3809
|
+
// name AND the task checkout — never a name alone. The journaled slot.id is NEVER trusted across
|
|
3810
|
+
// daemon instances: a fresh driver numbers its slots from one again, so an interrupted task's old
|
|
3811
|
+
// id can name whatever terminal THIS instance bound under that id (another task's adoption or allocation).
|
|
3812
|
+
// A driver with no read-only adoption is declined BEFORE anything is bound: slot() allocates —
|
|
3813
|
+
// herdr's reclaims the same-named pane (closing the evidence unread) and holds a dispatch lease
|
|
3814
|
+
// only run() releases — so it is never an adoption fallback; that driver keeps ordinary recovery.
|
|
3815
|
+
if (!driver.adopt)
|
|
3816
|
+
return decline("no read-only adoption on this driver");
|
|
3817
|
+
let ownedSlot;
|
|
3818
|
+
let pane;
|
|
3819
|
+
try {
|
|
3820
|
+
ownedSlot = await driver.adopt(slot);
|
|
3821
|
+
pane = await driver.read(ownedSlot, PANE_READ_ROWS);
|
|
3822
|
+
}
|
|
3823
|
+
catch (error) {
|
|
3824
|
+
return decline(`pane unreadable: ${error instanceof Error ? error.message : String(error)}`);
|
|
3825
|
+
}
|
|
3826
|
+
const parsed = adapter.parse(pane, nonce);
|
|
3827
|
+
const finished = parsed.cause === undefined;
|
|
3828
|
+
const foreign = !finished && [...pane.matchAll(/TICKMARKR_RESULT_([0-9a-z]+)/g)]
|
|
3829
|
+
.some(([, other]) => other !== nonce && adapter.parse(pane, other).cause === undefined);
|
|
3830
|
+
if (foreign)
|
|
3831
|
+
return decline("foreign-nonce");
|
|
3832
|
+
const taskBase = await integrationHead(intWt);
|
|
3833
|
+
if (!finished && (await commitsAheadOf(taskBase, wt)).length === 0)
|
|
3834
|
+
return decline("unfinished");
|
|
3835
|
+
const uncertain = await reapOwned(ownedSlot, launch);
|
|
3836
|
+
if (uncertain)
|
|
3837
|
+
return decline(uncertain);
|
|
3838
|
+
const commits = await commitsAheadOf(taskBase, wt); // re-measured: the reaped writer is gone now
|
|
3839
|
+
// The source tag makes a death between these two rows recoverable: the next resume finishes the record.
|
|
3840
|
+
journal.append("worker-result", t.id, {
|
|
3841
|
+
ok: parsed.ok, summary: parsed.summary, deviations: parsed.deviations, finished, exitCode: null, attempt,
|
|
3842
|
+
source: RESUME_HARVEST_SOURCE, ...(parsed.cause ? { cause: parsed.cause } : {}),
|
|
3843
|
+
});
|
|
3844
|
+
recordHarvest(commits, finished, parsed.summary);
|
|
3845
|
+
return found.assignment;
|
|
3846
|
+
};
|
|
3847
|
+
const harvestAuthor = rs && !satisfiedGate && !replayedGates && !recheck ? await harvestInterruptedAttempt() : undefined;
|
|
3848
|
+
// OBS-1202: a recheck bound to a stall park that recorded a reapFailure re-verifies the parked
|
|
3849
|
+
// attempt's owned census through the same reapWorker a live harvest uses, BEFORE any gate reads
|
|
3850
|
+
// its tree. Only a complete, empty census releases the harvested commits to the battery; an
|
|
3851
|
+
// unreadable (null) or surviving census keeps the task parked on the same kind — a stall carrying
|
|
3852
|
+
// the fresh reapFailure, bound by its own new token — and no worker is funded either way. Plain
|
|
3853
|
+
// approve on that park keeps its dispatch semantics.
|
|
3854
|
+
if (recheck) {
|
|
3855
|
+
const approvalRow = effectiveEvents(journal.read()).reverse()
|
|
3856
|
+
.find((e) => e.taskId === t.id && e.event === "task-approved" && e.data.release === RECHECK_RELEASE);
|
|
3857
|
+
if (typeof approvalRow?.data.reapFailure === "string") {
|
|
3858
|
+
const launch = [...journal.read()].reverse().find((e) => e.taskId === t.id && e.event === "worker-launch");
|
|
3859
|
+
const owned = launch?.data.slot;
|
|
3860
|
+
let uncertain;
|
|
3861
|
+
if (typeof launch?.data.nonce !== "string" || typeof launch.data.dispatchScript !== "string"
|
|
3862
|
+
|| typeof owned?.name !== "string" || typeof owned.cwd !== "string") {
|
|
3863
|
+
uncertain = "the parked attempt recorded no launch ownership evidence";
|
|
3864
|
+
}
|
|
3865
|
+
else {
|
|
3866
|
+
const key = { ...owned };
|
|
3867
|
+
workerOwners.set(key, { taskId: t.id, attempt: Number(launch.data.attempt ?? 0),
|
|
3868
|
+
groupFile: `${launch.data.dispatchScript}.${launch.data.nonce}.pgid`, marker: launch.data.dispatchScript,
|
|
3869
|
+
identities: new Map(), descendants: new Map() });
|
|
3870
|
+
try {
|
|
3871
|
+
await reapWorker(key);
|
|
3872
|
+
// reapWorker returns normally for a null census when no ownership file was readable (a
|
|
3873
|
+
// missing .pgid/.session with no descendants); recovery needs a RECORDED empty census.
|
|
3874
|
+
const census = reapReports.get(key)?.survivors;
|
|
3875
|
+
if (!Array.isArray(census) || census.length > 0) {
|
|
3876
|
+
uncertain = `worker group ${reapReports.get(key)?.processGroup ?? "unknown"} census unrecorded`;
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
catch (error) {
|
|
3880
|
+
uncertain = error instanceof Error ? error.message : String(error);
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3883
|
+
if (uncertain !== undefined) {
|
|
3884
|
+
await park(t, `worker could not be reaped before recheck: ${uncertain}`, "stall", null, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode, { reapFailure: uncertain });
|
|
3885
|
+
return;
|
|
3886
|
+
}
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
resumeGateReplay: if (satisfiedGate || replayedGates || recheck || harvestAuthor) {
|
|
3349
3890
|
const taskBase = await integrationHead(intWt);
|
|
3350
3891
|
taskBases.set(t.id, taskBase);
|
|
3351
3892
|
const taskBranch = `${branch}--${t.id}`;
|
|
@@ -3354,7 +3895,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3354
3895
|
// battery gates — read from the pending approval row, never re-selected here, so the tree
|
|
3355
3896
|
// approve counted and the tree the daemon gates are one and the same, checkout present or not.
|
|
3356
3897
|
const preservedRef = recheck
|
|
3357
|
-
?
|
|
3898
|
+
? effectiveEvents(journal.read()).reverse().find((e) => e.taskId === t.id && e.event === "task-approved" && e.data.release === RECHECK_RELEASE)?.data.recheckedRef
|
|
3358
3899
|
: undefined;
|
|
3359
3900
|
if (!existsSync(priorWt) && !preservedRef) {
|
|
3360
3901
|
if (satisfiedGate || recheck)
|
|
@@ -3366,14 +3907,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3366
3907
|
}
|
|
3367
3908
|
const resumeReason = recheck
|
|
3368
3909
|
? "operator recheck"
|
|
3369
|
-
:
|
|
3370
|
-
?
|
|
3371
|
-
:
|
|
3910
|
+
: harvestAuthor
|
|
3911
|
+
? "harvested interrupted attempt"
|
|
3912
|
+
: satisfiedGate
|
|
3913
|
+
? `approved gate ${satisfiedGate}`
|
|
3914
|
+
: `recorded gates on ${replayedGates.commit.slice(0, 10)}`;
|
|
3372
3915
|
const priorTaskTip = preservedRef ? (await shGit(`git rev-parse ${shq(preservedRef)}`, repoRoot)).stdout.trim() : await gitHead(priorWt);
|
|
3373
3916
|
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, preservedRef ? repoRoot : priorWt);
|
|
3374
3917
|
const commitsToCarry = preservedRef ? await commitsAheadOfRef(taskBase, priorTaskTip, repoRoot) : await commitsAheadOf(taskBase, priorWt);
|
|
3375
3918
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
3376
|
-
const carriedCommits = await cherryPickCommits(wt, commitsToCarry);
|
|
3919
|
+
const { carried: carriedCommits, accounted: accountedCommits } = await cherryPickCommits(wt, commitsToCarry);
|
|
3377
3920
|
// Reuse is about the tree the gates will actually inspect. The integration tip may have moved
|
|
3378
3921
|
// while the daemon was down, so compare after recreating the task on today's taskBase rather
|
|
3379
3922
|
// than against the stale worktree whose task-only history cannot see newly merged dependencies.
|
|
@@ -3383,13 +3926,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3383
3926
|
if (recheck) {
|
|
3384
3927
|
satisfiedGate = journal.replaySatisfiedGates(new Map([[t.id, currentTaskSubject]])).get(t.id);
|
|
3385
3928
|
}
|
|
3386
|
-
journal.append("worktree-recreation", t.id, {
|
|
3929
|
+
journal.append("worktree-recreation", t.id, {
|
|
3930
|
+
attempted: commitsToCarry, carried: carriedCommits, ...(accountedCommits.length > 0 ? { accounted: accountedCommits } : {}),
|
|
3931
|
+
});
|
|
3387
3932
|
// OBS-212: same fail-closed rule as the dispatch path — but this path is worse, because it runs
|
|
3388
3933
|
// ONLY the gates after the approved one and then MERGES. T3 took it on run-20260728-110135:
|
|
3389
3934
|
// approved past review at 11:22, recreated at 12:50, and phase-start{gates} / phase-start{merge}
|
|
3390
3935
|
// landed in the same second with zero gate-result events. Work missing here is merged unverified.
|
|
3391
3936
|
{
|
|
3392
|
-
const present = new Set(carriedCommits);
|
|
3937
|
+
const present = new Set([...carriedCommits, ...accountedCommits]);
|
|
3393
3938
|
for (const h of commitsToCarry) {
|
|
3394
3939
|
if (!present.has(h) && (await shGit(`git merge-base --is-ancestor ${shq(h)} HEAD`, wt)).code === 0) {
|
|
3395
3940
|
present.add(h);
|
|
@@ -3429,7 +3974,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3429
3974
|
const parkedAuthor = recheck
|
|
3430
3975
|
? [...journal.read()].reverse().find((e) => e.event === "task-dispatch" && e.taskId === t.id)?.data.assignment
|
|
3431
3976
|
: undefined;
|
|
3432
|
-
|
|
3977
|
+
// OBS-1109: a harvested attempt's author is read from the journal, not from this resume's path —
|
|
3978
|
+
// a later resume that replays the harvest's recorded gates still gates it under its producer.
|
|
3979
|
+
const gateAuthor = parkedAuthor ?? resumeHarvestAuthor(journal.read(), t.id) ?? rs?.lastAssignment ?? assignment;
|
|
3433
3980
|
const satisfiedIndex = satisfiedGate ? GATE_NAMES.indexOf(satisfiedGate) : -1;
|
|
3434
3981
|
// The serial pipeline could have at most one blocking result, so "everything after the
|
|
3435
3982
|
// approved gate" was enough. v1.85 can record both verdict siblings red in one round, and a
|
|
@@ -3458,6 +4005,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3458
4005
|
if (recheck) {
|
|
3459
4006
|
remainingGates = declaredGates.filter((gate) => gate !== satisfiedGate);
|
|
3460
4007
|
}
|
|
4008
|
+
else if (harvestAuthor) {
|
|
4009
|
+
remainingGates = declaredGates; // the harvested attempt was never gated: every declared gate runs
|
|
4010
|
+
}
|
|
3461
4011
|
else if (satisfiedGate) {
|
|
3462
4012
|
remainingGates = t.gates.filter((gate) => {
|
|
3463
4013
|
if (gate === satisfiedGate)
|
|
@@ -3526,7 +4076,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3526
4076
|
const startedAt = Date.now();
|
|
3527
4077
|
const provisioned = await withCommandContext(t.id, () => sh(commands.build, wt));
|
|
3528
4078
|
provisionedRow = {
|
|
3529
|
-
gate: "build", commit: !satisfiedGate && !recheck ? replayedGates.commit : currentTaskSubject,
|
|
4079
|
+
gate: "build", commit: !satisfiedGate && !recheck && replayedGates ? replayedGates.commit : currentTaskSubject,
|
|
3530
4080
|
exitCode: provisioned.code, durationMs: Date.now() - startedAt,
|
|
3531
4081
|
};
|
|
3532
4082
|
if (provisioned.code !== 0 && satisfiedGate !== "build") {
|
|
@@ -3540,7 +4090,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3540
4090
|
if (provisionedRow !== undefined)
|
|
3541
4091
|
journal.append("gate-provisioned", t.id, provisionedRow);
|
|
3542
4092
|
const resumedTask = { ...t, gates: remainingGates };
|
|
3543
|
-
const operatorContext = approvalReviewContext(journal.read(), t.id
|
|
4093
|
+
const operatorContext = approvalReviewContext(journal.read(), t.id);
|
|
3544
4094
|
gateLoop: while (true) {
|
|
3545
4095
|
fatalStop.signal.throwIfAborted();
|
|
3546
4096
|
executionSignal()?.throwIfAborted();
|
|
@@ -3551,7 +4101,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3551
4101
|
// This suffix is re-measured to decide whether resume may advance, but the interrupted
|
|
3552
4102
|
// attempt already paid for its red result. The next worker-backed round remains the next
|
|
3553
4103
|
// deterministic-fingerprint occurrence/review round for budget accounting.
|
|
3554
|
-
...(!satisfiedGate && !recheck ? { replayMeasurement: true } : {}),
|
|
4104
|
+
...(!satisfiedGate && !recheck && !harvestAuthor ? { replayMeasurement: true } : {}),
|
|
3555
4105
|
};
|
|
3556
4106
|
if (recheck && satisfiedGate === "review" && gateSubject.commit !== currentTaskSubject) {
|
|
3557
4107
|
satisfiedGate = undefined;
|
|
@@ -3562,14 +4112,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3562
4112
|
}
|
|
3563
4113
|
await trackedDriver.project?.(t.id, "in-review");
|
|
3564
4114
|
await waitForBaseline(t.id);
|
|
4115
|
+
// OBS-1106 residual: the live loop's timeout refresh, decided again from the journal, so a crash
|
|
4116
|
+
// cannot launder the copy it forbids. A harvested retry that landed nothing owes it exactly as
|
|
4117
|
+
// the live loop would have; a crash inside a forced battery before its test row still owes it.
|
|
4118
|
+
const lastRound = (() => {
|
|
4119
|
+
const own = journal.read().filter((e) => e.taskId === t.id);
|
|
4120
|
+
return own.slice(own.map((e) => e.event === "phase-start" && e.data.phase === "gates").lastIndexOf(true) + 1);
|
|
4121
|
+
})();
|
|
4122
|
+
const freshTimeout = !recheck && !satisfiedGate && ((harvestAuthor !== undefined && owesTimeoutRefresh(t, lastRound, gateSubject.commit))
|
|
4123
|
+
|| (lastRound.some((e) => e.event === "gate-fresh-forced" && e.data.commit === gateSubject.commit)
|
|
4124
|
+
&& !lastRound.some((e) => e.event === "gate-result" && e.data.gate === "test")));
|
|
3565
4125
|
journal.phaseStart(t.id, "gates");
|
|
4126
|
+
if (freshTimeout) {
|
|
4127
|
+
journal.append("gate-fresh-forced", t.id, { gate: "test", attempt: gateSubject.attempt, commit: gateSubject.commit, reason: "no-commit-timeout-red", resumed: true });
|
|
4128
|
+
}
|
|
3566
4129
|
const { results } = await withCommandContext(t.id, async () => runReviewRecovery(resumedTask, {
|
|
3567
4130
|
carriedAuthors: await subjectAuthors(journal.read(), t.id, wt, taskBase),
|
|
3568
4131
|
producer: producerNow(),
|
|
3569
4132
|
carriedFindings: outstandingReviewFindings(journal.read(), t.id),
|
|
3570
4133
|
operatorContext,
|
|
3571
4134
|
worktree: wt, baseRef: taskBase, result: priorResult, author: gateAuthor,
|
|
3572
|
-
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4135
|
+
commands, baseline, channels: pools.review, judgeChannels: pools.judge, health, adapters, cfg, artifactDir: journal.dir,
|
|
3573
4136
|
collateral: collateral.get(t.id) ?? [],
|
|
3574
4137
|
// a recheck re-verifies a human's release: it never selects tests down, it runs the suite.
|
|
3575
4138
|
via: cfg.visibility.llm === "pane"
|
|
@@ -3587,7 +4150,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3587
4150
|
// flaking seat is re-asked on every task.
|
|
3588
4151
|
reviewNoVerdicts,
|
|
3589
4152
|
recheck, // OBS-1055: a recheck discards cached reds — the battery re-measures what the operator questioned
|
|
4153
|
+
cachedRedBypass: freshTimeout ? "timeout-fresh" : undefined,
|
|
3590
4154
|
onGate: async (e) => {
|
|
4155
|
+
if (!liveEngagement())
|
|
4156
|
+
return;
|
|
3591
4157
|
if (e.phase === "start") {
|
|
3592
4158
|
notePhaseStart(e);
|
|
3593
4159
|
journal.phaseStart(t.id, phaseForGate(e.gate), { gate: e.gate, index: e.index, total: e.total, ...(e.parentAt === undefined ? {} : { parallel: true }) });
|
|
@@ -3603,10 +4169,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3603
4169
|
inParallelOrder(g.gate, () => {
|
|
3604
4170
|
journalGateResult(g);
|
|
3605
4171
|
noteReviewRetry(g);
|
|
3606
|
-
if (
|
|
3607
|
-
&& typeof g.meta?.reviewer === "string") {
|
|
4172
|
+
if (isGarbageReview(g))
|
|
3608
4173
|
badReviewers.push(g.meta.reviewer);
|
|
3609
|
-
}
|
|
3610
4174
|
});
|
|
3611
4175
|
},
|
|
3612
4176
|
}, false));
|
|
@@ -3649,6 +4213,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3649
4213
|
// erase an EARLIER chargeable attempt and attribute the park to the assignment the rewind
|
|
3650
4214
|
// restored. Ask the journal which state this is, and take the classified dispatch's own
|
|
3651
4215
|
// assignment back with it.
|
|
4216
|
+
if (await adjudicateInfraShapedRed(t, results, wt, gateSubject.commit, rs?.attempts ?? 0, false, (reason) => park(t, reason, "infra", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode)))
|
|
4217
|
+
return;
|
|
3652
4218
|
const classified = journal.classifiedDispatch(t.id);
|
|
3653
4219
|
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
3654
4220
|
return;
|
|
@@ -3667,7 +4233,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3667
4233
|
await park(t, `recheck red: pinned ${pin.via}:${pin.model} is unavailable to host the repair — refusing the ladder`, "gate-fail", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3668
4234
|
return;
|
|
3669
4235
|
}
|
|
3670
|
-
assignment =
|
|
4236
|
+
assignment = seatAssignment(seat);
|
|
3671
4237
|
}
|
|
3672
4238
|
// Carry completeness was verified before this replay battery. Apply the ordinary
|
|
3673
4239
|
// repair bounds here too; a restored red must fund the same findings-bearing dispatch.
|
|
@@ -3778,13 +4344,21 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3778
4344
|
}
|
|
3779
4345
|
// OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
|
|
3780
4346
|
if (demotedChannels.has(channelKey(assignment))) {
|
|
3781
|
-
|
|
4347
|
+
// OBS-1187: a pooled task recycles a live pool member (OBS-202) before it may park naming the pool;
|
|
4348
|
+
// it never keeps a demoted seat and never leaves the pool.
|
|
4349
|
+
const pooled = taskPool(t, cfg) !== undefined;
|
|
4350
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels)
|
|
4351
|
+
?? (pooled ? nextChannel(assignment, t, cfg, channels, recycleExcluded(), profile, demotedChannels) : null);
|
|
3782
4352
|
if (next) {
|
|
3783
4353
|
assignment = next;
|
|
3784
4354
|
const k = channelKey(next);
|
|
3785
4355
|
if (!tried.includes(k))
|
|
3786
4356
|
tried.push(k);
|
|
3787
4357
|
}
|
|
4358
|
+
else if (pooled) {
|
|
4359
|
+
await park(t, `${channelKey(assignment)} is demoted (verified dead) and no live pool member remains${poolExhaustion(t, cfg)}`, "reroute-exhausted", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4360
|
+
return;
|
|
4361
|
+
}
|
|
3788
4362
|
}
|
|
3789
4363
|
// v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
|
|
3790
4364
|
// + adapter capability resumes; every other path is today's fresh dispatch.
|
|
@@ -3809,7 +4383,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3809
4383
|
gate: banned, from: channelKey(assignment), to: next ? channelKey(next) : null,
|
|
3810
4384
|
});
|
|
3811
4385
|
if (!next) {
|
|
3812
|
-
await park(t, gateFailApprovalReason(t.id, `identical ${banned} failure twice this engagement — an identical retry is banned and no untried channel is left`), "gate-fail", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4386
|
+
await park(t, gateFailApprovalReason(t.id, `identical ${banned} failure twice this engagement — an identical retry is banned and no untried channel is left${poolExhaustion(t, cfg)}`), "gate-fail", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3813
4387
|
return;
|
|
3814
4388
|
}
|
|
3815
4389
|
assignment = next;
|
|
@@ -3886,7 +4460,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3886
4460
|
feedback = feedback ? `${feedback}\n\n${consultBrief}` : consultBrief;
|
|
3887
4461
|
}
|
|
3888
4462
|
}
|
|
3889
|
-
const scopeApproval =
|
|
4463
|
+
const scopeApproval = effectiveEvents(journaledSoFar).reverse().find((e) => e.taskId === t.id && (e.event === "task-approved" || e.event === "task-dispatch"));
|
|
3890
4464
|
retryMode = scopeApproval?.data.release === "scope-request" ? "fresh" : repairFindings
|
|
3891
4465
|
? "repair"
|
|
3892
4466
|
: priorSession
|
|
@@ -3915,6 +4489,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3915
4489
|
// spends two frontier attempts re-deriving a known defect has to be able to answer afterwards.
|
|
3916
4490
|
fatalStop.signal.throwIfAborted();
|
|
3917
4491
|
const workerDispatchOrdinal = nextWorkerDispatchOrdinal++;
|
|
4492
|
+
vendorEscalated.delete(channelKey(assignment)); // dispatched now: from here on it IS tried
|
|
3918
4493
|
journal.append("task-dispatch", t.id, {
|
|
3919
4494
|
...(scopeApproval?.data.release === "scope-request" ? { files: t.files, graphDefinitionHash: graphDefinitionHash(graph) } : {}),
|
|
3920
4495
|
assignment, attempt, workerDispatchOrdinal, provenance: dispatchProvenance([
|
|
@@ -3927,7 +4502,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3927
4502
|
.filter((key) => key !== channelKey(assignment)).sort(),
|
|
3928
4503
|
exclusionReasons: Object.fromEntries([...new Set([...demotedChannels, ...tried, ...climbSkips])]
|
|
3929
4504
|
.filter((key) => key !== channelKey(assignment))
|
|
3930
|
-
.map((key) => [key, demotedChannels.has(key) ? "demoted"
|
|
4505
|
+
.map((key) => [key, demotedChannels.has(key) ? "demoted"
|
|
4506
|
+
: vendorEscalated.get(key) ?? (tried.includes(key) ? "already tried" : "tier climb skipped")])),
|
|
3931
4507
|
...(outstandingFindings.length > 0 ? { carriedFindings: outstandingFindings } : {}),
|
|
3932
4508
|
...(carriedConsultGuidance ? { carriedConsultGuidance } : {}),
|
|
3933
4509
|
});
|
|
@@ -3947,12 +4523,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3947
4523
|
// OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
|
|
3948
4524
|
// cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
|
|
3949
4525
|
// orphan work a consult already verified as landed.
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
4526
|
+
const { carried: carriedCommits, accounted: accountedCommits } = commitsToCarry.length > 0
|
|
4527
|
+
? await cherryPickCommits(wt, commitsToCarry) : { carried: [], accounted: [] };
|
|
4528
|
+
if (recreating) {
|
|
4529
|
+
journal.append("worktree-recreation", t.id, {
|
|
4530
|
+
attempted: commitsToCarry, carried: carriedCommits, ...(accountedCommits.length > 0 ? { accounted: accountedCommits } : {}),
|
|
4531
|
+
});
|
|
3953
4532
|
}
|
|
3954
|
-
if (recreating)
|
|
3955
|
-
journal.append("worktree-recreation", t.id, { attempted: commitsToCarry, carried: carriedCommits });
|
|
3956
4533
|
// T2 review (material): harvest eligibility is "does this WORKTREE carry unverified work",
|
|
3957
4534
|
// measured against taskBase — the same base the fast-kill's delta probe and the gates
|
|
3958
4535
|
// themselves use. It was measured against this attempt's post-carry HEAD, which excluded
|
|
@@ -3963,7 +4540,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3963
4540
|
// quota, dead-channel and provider-death all classify the PRE-HARVEST outcome below and fire
|
|
3964
4541
|
// BEFORE the synthesis, so carried-only work reaches gates without bypassing any failover.
|
|
3965
4542
|
const priorNamed = [...new Set([...commitsToCarry, ...carriedCommits])];
|
|
3966
|
-
const presentCommits = new Set(carriedCommits);
|
|
4543
|
+
const presentCommits = new Set([...carriedCommits, ...accountedCommits]);
|
|
3967
4544
|
for (const h of commitsToCarry) {
|
|
3968
4545
|
if (!presentCommits.has(h) && (await shGit(`git merge-base --is-ancestor ${shq(h)} HEAD`, wt)).code === 0) {
|
|
3969
4546
|
presentCommits.add(h);
|
|
@@ -4020,10 +4597,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4020
4597
|
? `${new TextDecoder().decode(rawBytes.subarray(0, cap))}\n… diff truncated at gates.diffCap (${cap} bytes)`
|
|
4021
4598
|
+ `\nPARTIAL DIFF: the diff above is incomplete; files cut by the cap: ${droppedFiles.join(", ") || "(unknown)"}`
|
|
4022
4599
|
: raw.stdout;
|
|
4023
|
-
const
|
|
4600
|
+
const repeated = reraisedReviewChains(journaledSoFar, t.id).filter((chain) => chain.reraises > 1);
|
|
4601
|
+
const brief = repairBrief(repairFindings, diff, taskBase, repeated);
|
|
4024
4602
|
// anything the live brief holds beyond the journaled findings (a consult's guidance) is kept:
|
|
4025
4603
|
// a repair adds the diff and the fix-only contract, it never subtracts what was already known.
|
|
4026
|
-
|
|
4604
|
+
// OBS-1195: a resumed upheld repair re-derives the same review bytes on both carries; when the
|
|
4605
|
+
// repair already quotes them verbatim they are said once, and the operator's uphold line stays.
|
|
4606
|
+
// Both carries had their deferrals cut above, so the quote is compared after the same cut.
|
|
4607
|
+
const upheldQuote = upheldFeedback ? withoutDeferrals(`review: ${upheldFeedback}`) : undefined;
|
|
4608
|
+
const rest = upheldQuote && brief.includes(upheldQuote)
|
|
4609
|
+
? feedback.replace(upheldQuote, "").replace(/\n{3,}/g, "\n\n").trim()
|
|
4610
|
+
: feedback;
|
|
4611
|
+
feedback = rest && !brief.includes(rest) ? `${brief}\n\n${rest}` : brief;
|
|
4027
4612
|
journal.append("repair-dispatch", t.id, { workerDispatchOrdinal, diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
4028
4613
|
}
|
|
4029
4614
|
if (feedback || priorNamed.length > 0) {
|
|
@@ -4079,9 +4664,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4079
4664
|
sessionId, attempt, workerDispatchOrdinal, baselineBytes: resumeBaseline?.bytes ?? null,
|
|
4080
4665
|
});
|
|
4081
4666
|
const icmd = retryMode === "resume"
|
|
4082
|
-
? adapter.resumeCommand(sessionId, promptFile, assignment.model)
|
|
4667
|
+
? adapter.resumeCommand(sessionId, promptFile, assignment.model, assignment.effort)
|
|
4083
4668
|
: cfg.visibility.worker === "interactive" && driver.interactive
|
|
4084
|
-
? adapter.interactiveCommand(promptFile, assignment.model)
|
|
4669
|
+
? adapter.interactiveCommand(promptFile, assignment.model, assignment.effort)
|
|
4085
4670
|
: null;
|
|
4086
4671
|
// v1.69 T6: adapters that declare interactiveSeed launch the real TUI and inject the prompt as a
|
|
4087
4672
|
// user turn; they do NOT need the argv-seeding surface that interactiveCommand represents.
|
|
@@ -4142,6 +4727,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4142
4727
|
journal.append("worker-launch", t.id, {
|
|
4143
4728
|
attempt,
|
|
4144
4729
|
retryMode,
|
|
4730
|
+
// OBS-1109: the attempt's own ownership evidence, so a resumed daemon can harvest it.
|
|
4731
|
+
nonce, dispatchScript,
|
|
4145
4732
|
...(retryMode === "resume" ? { sessionId, workerDispatchOrdinal } : {}),
|
|
4146
4733
|
driver: trackedDriver.id,
|
|
4147
4734
|
slot: { ...slot },
|
|
@@ -4281,13 +4868,46 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4281
4868
|
let capacityBannerKilled = false; // OBS-1161: same banner filter and gates, transient outcome
|
|
4282
4869
|
let driverProbeFailed = false;
|
|
4283
4870
|
let heldLegs = [];
|
|
4871
|
+
// OBS-1108: contact uncertainty is LATCHED per owned slot and dispatch (this attempt's slot): one
|
|
4872
|
+
// row when contact is lost and one when it returns, never one per retry. Retries back off while
|
|
4873
|
+
// latched, and a latch held past its deadline throws into the transport-uncertain park, which
|
|
4874
|
+
// preserves the worktree for a recheck. A read that recovers inside the deadline charges nothing.
|
|
4875
|
+
let contactLost;
|
|
4876
|
+
const contactDeadlineMs = contactUnreadableDeadlineMs ?? taskTimeoutMinutes * 60_000;
|
|
4284
4877
|
const noteDriverUnreadable = (error) => {
|
|
4285
4878
|
if (error instanceof HeldProbeExhausted)
|
|
4286
4879
|
throw error;
|
|
4287
4880
|
driverProbeFailed = true;
|
|
4288
4881
|
heldLegs = ["quota:driver-unreadable", "stall:driver-unreadable"];
|
|
4289
|
-
|
|
4290
|
-
|
|
4882
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4883
|
+
if (contactLost) {
|
|
4884
|
+
contactLost.retries++;
|
|
4885
|
+
const unreadableMs = Date.now() - contactLost.since;
|
|
4886
|
+
if (unreadableMs >= contactDeadlineMs) {
|
|
4887
|
+
throw new ContactUnreadableExhausted(`contact with ${slot.name} unreadable for ${unreadableMs}ms, past its ${contactDeadlineMs}ms deadline, after ${contactLost.retries} retries: ${message}`);
|
|
4888
|
+
}
|
|
4889
|
+
return;
|
|
4890
|
+
}
|
|
4891
|
+
contactLost = { since: Date.now(), retries: 0 };
|
|
4892
|
+
journal.append("contact-unreadable", t.id, { slot: slot.name, attempt, source: "driver", state: "unreadable",
|
|
4893
|
+
concludes: false, deadlineMs: contactDeadlineMs, error: message });
|
|
4894
|
+
};
|
|
4895
|
+
const noteContactReadable = () => {
|
|
4896
|
+
if (!contactLost)
|
|
4897
|
+
return;
|
|
4898
|
+
journal.append("contact-recovered", t.id, { slot: slot.name, attempt, source: "driver", state: "readable",
|
|
4899
|
+
unreadableMs: Date.now() - contactLost.since, retries: contactLost.retries });
|
|
4900
|
+
contactLost = undefined;
|
|
4901
|
+
};
|
|
4902
|
+
// The pause after a slice whose contact probe failed. Unlatched: the unspent slice, at most 1 s.
|
|
4903
|
+
// Latched: a next-contact-at schedule of its own — doubling per retry, bounded by the contact and
|
|
4904
|
+
// hard deadlines and NEVER by the poll slice, which clamps to 100 ms once the stall clock expires
|
|
4905
|
+
// and would otherwise collapse the backoff into ~10 orca calls a second for the rest of the window.
|
|
4906
|
+
const contactPauseMs = (sliceStart, slice, hardDeadline) => {
|
|
4907
|
+
const now = Date.now();
|
|
4908
|
+
if (!contactLost)
|
|
4909
|
+
return Math.max(0, Math.min(slice - (now - sliceStart), CONTACT_BACKOFF_BASE_MS));
|
|
4910
|
+
return Math.max(1, Math.min(CONTACT_BACKOFF_BASE_MS * 2 ** Math.min(contactLost.retries, CONTACT_BACKOFF_DOUBLINGS), contactLost.since + contactDeadlineMs - now, hardDeadline - now));
|
|
4291
4911
|
};
|
|
4292
4912
|
// T2 review: print mode's "the exit marker appeared". Kept apart from `finished` (the
|
|
4293
4913
|
// trailer) but still needed by the keepPanes decision below, whose contract is about a
|
|
@@ -4297,7 +4917,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4297
4917
|
let startupFailure = false;
|
|
4298
4918
|
let startupEvidence;
|
|
4299
4919
|
const startupDetector = new StartupFailureDetector(adapter);
|
|
4920
|
+
// OBS-1169: bootstrap text is evidence only inside the same startup prefix — every sample below
|
|
4921
|
+
// feeds it, so a tool frame, an input box or a saturated read seen at ANY poll closes it for good.
|
|
4922
|
+
const bootstrapDetector = new StartupFailureDetector(adapter, BOOTSTRAP_FAILURE_RE, true);
|
|
4300
4923
|
const startupFailureInWindow = (text, launchedAt) => {
|
|
4924
|
+
bootstrapDetector.sample(text, launchedAt);
|
|
4301
4925
|
startupEvidence ??= startupDetector.sample(text, launchedAt);
|
|
4302
4926
|
return startupEvidence !== undefined;
|
|
4303
4927
|
};
|
|
@@ -4307,8 +4931,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4307
4931
|
? new SettledTrailerTracker(adapter.busyFrameMarkers) : undefined;
|
|
4308
4932
|
const sampleTrailer = (frame) => {
|
|
4309
4933
|
const parsed = adapter.parse(frame, nonce);
|
|
4310
|
-
|
|
4311
|
-
|
|
4934
|
+
// OBS-1175: the parser's cause, never the summary — a parsed trailer may carry a sentinel string.
|
|
4935
|
+
const trailer = new RegExp(trailerPattern(nonce)).test(frame) && parsed.cause === undefined;
|
|
4312
4936
|
return trailerFrames ? trailerFrames.sample(frame, trailer) : trailer;
|
|
4313
4937
|
};
|
|
4314
4938
|
let seedResult;
|
|
@@ -4363,6 +4987,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4363
4987
|
await park(t, "escalation ladder exhausted", "ladder-exhausted", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4364
4988
|
return false;
|
|
4365
4989
|
};
|
|
4990
|
+
// OBS-1169: the tree this worker receives — commits past it are this attempt's own work; the
|
|
4991
|
+
// carried ones beneath it are not, and must not make a bootstrap death look like work.
|
|
4992
|
+
const launchHead = await gitHead(wt);
|
|
4366
4993
|
try {
|
|
4367
4994
|
if (interactive) {
|
|
4368
4995
|
// v1.2 interactive: the TUI doesn't exit on completion — the trailer is the finish line.
|
|
@@ -4410,7 +5037,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4410
5037
|
return;
|
|
4411
5038
|
}
|
|
4412
5039
|
await noteLaunched();
|
|
4413
|
-
|
|
5040
|
+
// OBS-1108: the first owned-slot read joins the contact latch — an unreadable launch read is
|
|
5041
|
+
// retried by the wait loop below under the same deadline, never a task failure on its own.
|
|
5042
|
+
output = await workerTransport.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return ""; });
|
|
4414
5043
|
}
|
|
4415
5044
|
// The returning paths report the same fact on the result; both callbacks land on the one
|
|
4416
5045
|
// latch, and the second is a no-op. A seed that answered is never re-answered by the loop.
|
|
@@ -4497,7 +5126,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4497
5126
|
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
4498
5127
|
break;
|
|
4499
5128
|
}
|
|
4500
|
-
if (adapter.parse(output, nonce).
|
|
5129
|
+
if (adapter.parse(output, nonce).cause === "malformed-verdict")
|
|
4501
5130
|
break;
|
|
4502
5131
|
if (trailerFrames && new RegExp(trailerPattern(nonce)).test(output)) {
|
|
4503
5132
|
// A matching busy frame is still a live turn. Preserve the rolling progress budget.
|
|
@@ -4524,9 +5153,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4524
5153
|
});
|
|
4525
5154
|
}
|
|
4526
5155
|
await armCpuLeg(true);
|
|
4527
|
-
|
|
4528
|
-
if (spent < slice)
|
|
4529
|
-
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
5156
|
+
await new Promise((r) => setTimeout(r, contactPauseMs(sliceStart, slice, hardDeadline)));
|
|
4530
5157
|
continue;
|
|
4531
5158
|
}
|
|
4532
5159
|
// waitOutput is only a wake hint; a driver may miss a trailer already painted.
|
|
@@ -4539,14 +5166,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4539
5166
|
await sampleContext();
|
|
4540
5167
|
break;
|
|
4541
5168
|
}
|
|
4542
|
-
if (adapter.parse(paneText, nonce).
|
|
5169
|
+
if (adapter.parse(paneText, nonce).cause === "malformed-verdict") {
|
|
4543
5170
|
output = paneText;
|
|
4544
5171
|
break;
|
|
4545
5172
|
}
|
|
4546
5173
|
if (driverProbeFailed) {
|
|
4547
|
-
|
|
4548
|
-
if (spent < slice)
|
|
4549
|
-
await new Promise((resolve) => setTimeout(resolve, Math.min(slice - spent, 1_000)));
|
|
5174
|
+
await new Promise((resolve) => setTimeout(resolve, contactPauseMs(sliceStart, slice, hardDeadline)));
|
|
4550
5175
|
continue;
|
|
4551
5176
|
}
|
|
4552
5177
|
if (paneText.length > 0)
|
|
@@ -4654,11 +5279,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4654
5279
|
});
|
|
4655
5280
|
}
|
|
4656
5281
|
await armCpuLeg(true);
|
|
4657
|
-
|
|
4658
|
-
if (spent < slice)
|
|
4659
|
-
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
5282
|
+
await new Promise((r) => setTimeout(r, contactPauseMs(sliceStart, slice, hardDeadline)));
|
|
4660
5283
|
continue;
|
|
4661
5284
|
}
|
|
5285
|
+
// OBS-1108: every contact probe of this slice answered (a failed wait or read continues
|
|
5286
|
+
// above), so only here — never after one probe while another stays latched — has contact returned.
|
|
5287
|
+
noteContactReadable();
|
|
4662
5288
|
if (st !== lastStatus) {
|
|
4663
5289
|
lastStatus = st;
|
|
4664
5290
|
journal.append("worker-status", t.id, { slot: slot.name, status: st, attempt });
|
|
@@ -4977,7 +5603,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4977
5603
|
if (spent < slice)
|
|
4978
5604
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
4979
5605
|
}
|
|
4980
|
-
|
|
5606
|
+
// A wait that concludes on a clean slice's read (trailer, exit, startup failure) has contact back too.
|
|
5607
|
+
if (!driverProbeFailed)
|
|
5608
|
+
noteContactReadable();
|
|
5609
|
+
if (!finished && exitCode === null && adapter.parse(output, nonce).cause !== "malformed-verdict") {
|
|
5610
|
+
// OBS-1108: the hard deadline ended an attempt whose contact was still latched (its last
|
|
5611
|
+
// probe failed). That is contact loss, not a stall: keep the one infra park and the preserved
|
|
5612
|
+
// worktree instead of the hard-timeout classification and its repair charge.
|
|
5613
|
+
if (contactLost && driverProbeFailed && Date.now() >= hardDeadline) {
|
|
5614
|
+
throw new ContactUnreadableExhausted(`contact with ${slot.name} unreadable for ${Date.now() - contactLost.since}ms, still latched at the attempt's hard deadline after ${contactLost.retries} retries`);
|
|
5615
|
+
}
|
|
4981
5616
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
4982
5617
|
hardTimedOut = Date.now() >= hardDeadline;
|
|
4983
5618
|
timedOut ||= hardTimedOut;
|
|
@@ -5013,7 +5648,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5013
5648
|
const maxSettleRetries = 2;
|
|
5014
5649
|
let settleTries = 0;
|
|
5015
5650
|
settleParsed = adapter.parse(output, nonce);
|
|
5016
|
-
while (settleParsed.
|
|
5651
|
+
while (settleParsed.cause === "malformed-verdict" && settleTries < maxSettleRetries) {
|
|
5017
5652
|
const remaining = settleDeadline - Date.now();
|
|
5018
5653
|
if (remaining <= 0)
|
|
5019
5654
|
break;
|
|
@@ -5023,8 +5658,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5023
5658
|
settleParsed = adapter.parse(output, nonce);
|
|
5024
5659
|
settleTries++;
|
|
5025
5660
|
}
|
|
5026
|
-
if (settleParsed.
|
|
5027
|
-
finished = settleParsed.
|
|
5661
|
+
if (settleParsed.cause !== "malformed-verdict") {
|
|
5662
|
+
finished = settleParsed.cause === undefined && (!trailerFrames || trailerFrames.settled);
|
|
5028
5663
|
}
|
|
5029
5664
|
}
|
|
5030
5665
|
}
|
|
@@ -5117,6 +5752,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5117
5752
|
finally {
|
|
5118
5753
|
await cpuAccountant?.stop();
|
|
5119
5754
|
}
|
|
5755
|
+
// OBS-1169: the final pane read decides, at exit, whether the prefix is still the CLI's own startup.
|
|
5756
|
+
const bootstrapEvidence = processExited ? bootstrapDetector.sample(output, workerLaunchedAt) : undefined;
|
|
5120
5757
|
// SPEND-01 interactive metering race: the harvest loop breaks on the trailer, but the worker
|
|
5121
5758
|
// shell may still be running post-trailer bookkeeping (session-store flush, fake usage stamp,
|
|
5122
5759
|
// exit wrapper). Print mode already waits for TICKMARKR_EXIT, which follows that tail; drain
|
|
@@ -5246,6 +5883,58 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5246
5883
|
stallReaps = 0;
|
|
5247
5884
|
stallSeat = undefined;
|
|
5248
5885
|
}
|
|
5886
|
+
// OBS-1169: a CLI that died in its own bootstrap never read the brief. Startup-owned evidence
|
|
5887
|
+
// (known bootstrap text in the startup prefix StartupFailureDetector owns: inside the window, before
|
|
5888
|
+
// any tool frame or input box, in an unsaturated read), and only when it exited nonzero with no
|
|
5889
|
+
// parsed trailer and no commit of its own: the carried commits
|
|
5890
|
+
// beneath launchHead are an earlier attempt's, and harvesting them would replay that attempt's
|
|
5891
|
+
// red as a second fingerprint occurrence. One same-channel retry after a bounded backoff charges
|
|
5892
|
+
// no attempt, repair or fingerprint; after that the whole adapter shares the broken bootstrap,
|
|
5893
|
+
// so its channels are escalated away from and the existing failover moves on or parks infra.
|
|
5894
|
+
const bootstrap = !workerFinished && exitCode !== null && exitCode !== 0 && bootstrapEvidence
|
|
5895
|
+
&& (await commitsAheadOf(launchHead, wt)).length === 0 ? bootstrapEvidence.matchedBytes : undefined;
|
|
5896
|
+
if (bootstrap) {
|
|
5897
|
+
const from = channelKey(assignment);
|
|
5898
|
+
const retries = requeuesOn("bootstrap-retry", from);
|
|
5899
|
+
if (retries < BOOTSTRAP_RETRY_CAP) {
|
|
5900
|
+
journal.append("bootstrap-retry", t.id, {
|
|
5901
|
+
attempt, retry: retries + 1, of: BOOTSTRAP_RETRY_CAP, channel: from, assignment,
|
|
5902
|
+
matched: bootstrap, exitCode, backoffMs: bootstrapBackoffMs,
|
|
5903
|
+
});
|
|
5904
|
+
await new Promise((r) => setTimeout(r, bootstrapBackoffMs));
|
|
5905
|
+
attempt--;
|
|
5906
|
+
continue;
|
|
5907
|
+
}
|
|
5908
|
+
const reason = `vendor escalated: ${assignment.adapter} bootstrap failed on ${from}`;
|
|
5909
|
+
escalateAdapter(assignment.adapter, reason);
|
|
5910
|
+
const next = failover("bootstrap-failover");
|
|
5911
|
+
// `toAssignment` and `reason` let a resume replay this routing disposition, not only its refund.
|
|
5912
|
+
journal.append("bootstrap-failover", t.id, {
|
|
5913
|
+
from, to: next ? channelKey(next) : null, ...(next ? { toAssignment: next } : {}), matched: bootstrap, retries,
|
|
5914
|
+
escalated: [...vendorEscalated].filter(([, why]) => why === reason).map(([k]) => k), reason,
|
|
5915
|
+
});
|
|
5916
|
+
if (next) {
|
|
5917
|
+
await driver.notify(`tickmarkr ${runId}: ${t.id} bootstrap failover`, { tier: "attention" });
|
|
5918
|
+
if (!keepForever) {
|
|
5919
|
+
const idx = keptSlots.indexOf(slot);
|
|
5920
|
+
if (idx >= 0) {
|
|
5921
|
+
keptSlots.splice(idx, 1);
|
|
5922
|
+
try {
|
|
5923
|
+
await closeSlot(slot);
|
|
5924
|
+
}
|
|
5925
|
+
catch { /* cosmetic — reconcile is the backstop */ }
|
|
5926
|
+
}
|
|
5927
|
+
if (supersededWorkerSlot === slot)
|
|
5928
|
+
supersededWorkerSlot = undefined;
|
|
5929
|
+
}
|
|
5930
|
+
assignment = next;
|
|
5931
|
+
tried.push(channelKey(next));
|
|
5932
|
+
attempt--; // the dead bootstrap bought no worker turn
|
|
5933
|
+
continue;
|
|
5934
|
+
}
|
|
5935
|
+
await park(t, `${from} failed at bootstrap after ${retries} same-channel retry and no eligible channel remains${poolExhaustion(t, cfg)}`, "infra", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "bootstrap", channel: from, retries });
|
|
5936
|
+
return;
|
|
5937
|
+
}
|
|
5249
5938
|
const preHarvestResult = result;
|
|
5250
5939
|
// T2 (OBS-264): recognize committed no-trailer work BEFORE any no-trailer streak, provider,
|
|
5251
5940
|
// quota or dead-channel routing. Gates never trusted the trailer, so this successful synthesis
|
|
@@ -5322,7 +6011,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5322
6011
|
// the journal, never a loop-local counter, so a resume continues the count it left off at.
|
|
5323
6012
|
if (capacityMatch) {
|
|
5324
6013
|
const from = channelKey(assignment);
|
|
5325
|
-
const requeues =
|
|
6014
|
+
const requeues = requeuesOn("capacity-requeue", from);
|
|
5326
6015
|
const source = capacityBannerKilled ? "banner" : "exit";
|
|
5327
6016
|
if (requeues < CAPACITY_REQUEUE_CAP) {
|
|
5328
6017
|
journal.append("capacity-requeue", t.id, {
|
|
@@ -5355,7 +6044,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5355
6044
|
tried.push(channelKey(next));
|
|
5356
6045
|
continue;
|
|
5357
6046
|
}
|
|
5358
|
-
await park(t, `capacity exhausted on ${from} after ${requeues} requeues and no eligible channel at floor`, "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "capacity", channel: from, requeues });
|
|
6047
|
+
await park(t, `capacity exhausted on ${from} after ${requeues} requeues and no eligible channel at floor${poolExhaustion(t, cfg)}`, "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "capacity", channel: from, requeues });
|
|
5359
6048
|
return;
|
|
5360
6049
|
}
|
|
5361
6050
|
// quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
|
|
@@ -5418,7 +6107,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5418
6107
|
tried.push(channelKey(next));
|
|
5419
6108
|
continue;
|
|
5420
6109
|
}
|
|
5421
|
-
await park(t,
|
|
6110
|
+
await park(t, `quota exhausted on every eligible channel${poolExhaustion(t, cfg)}`, "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5422
6111
|
return;
|
|
5423
6112
|
}
|
|
5424
6113
|
// v1.65 T1: typed dead-channel failure — the parse boundary classified this no-trailer result
|
|
@@ -5466,7 +6155,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5466
6155
|
continue;
|
|
5467
6156
|
}
|
|
5468
6157
|
const chargedAttempts = adapterStartupFailure || startupFailure ? attempt : attempt + 1;
|
|
5469
|
-
await park(t, `dead channel (${dead}) and no eligible channel remains`, "reroute-exhausted", assignment, chargedAttempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
6158
|
+
await park(t, `dead channel (${dead}) and no eligible channel remains${poolExhaustion(t, cfg)}`, "reroute-exhausted", assignment, chargedAttempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5470
6159
|
return;
|
|
5471
6160
|
}
|
|
5472
6161
|
if (!finished) {
|
|
@@ -5511,6 +6200,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5511
6200
|
}
|
|
5512
6201
|
}
|
|
5513
6202
|
const onGate = async (e) => {
|
|
6203
|
+
if (!liveEngagement())
|
|
6204
|
+
return;
|
|
5514
6205
|
if (e.phase === "start") {
|
|
5515
6206
|
notePhaseStart(e);
|
|
5516
6207
|
journal.phaseStart(t.id, phaseForGate(e.gate), { gate: e.gate, index: e.index, total: e.total, ...(e.parentAt === undefined ? {} : { parallel: true }) });
|
|
@@ -5544,9 +6235,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5544
6235
|
journalGateResult(g);
|
|
5545
6236
|
noteReviewRetry(g);
|
|
5546
6237
|
// v1.1 failover: never re-ask a reviewer channel that produced garbage for this task
|
|
5547
|
-
if (
|
|
6238
|
+
if (isGarbageReview(g))
|
|
5548
6239
|
badReviewers.push(g.meta.reviewer);
|
|
5549
|
-
}
|
|
5550
6240
|
});
|
|
5551
6241
|
};
|
|
5552
6242
|
let results = [];
|
|
@@ -5562,12 +6252,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5562
6252
|
// sees the replay, without buying another command or reviewer invocation.
|
|
5563
6253
|
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
5564
6254
|
const previousRound = taskEvents.map((e) => e.event === "phase-start" && e.data.phase === "gates").lastIndexOf(true);
|
|
5565
|
-
const
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
|
|
6255
|
+
const lastRoundRows = taskEvents.slice(previousRound + 1).filter((e) => e.event === "gate-result"
|
|
6256
|
+
&& e.data.attempt === attempt - 1);
|
|
6257
|
+
const previousRows = retryMode === "repair" ? lastRoundRows : [];
|
|
6258
|
+
// OBS-1106 residual: see owesTimeoutRefresh — the replay below and the verdict cache are both refused.
|
|
6259
|
+
const freshTimeout = !explicitlyFundedAttempt && owesTimeoutRefresh(t, lastRoundRows, gateSubject.commit);
|
|
5569
6260
|
journal.phaseStart(t.id, "gates");
|
|
5570
|
-
|
|
6261
|
+
if (freshTimeout) {
|
|
6262
|
+
journal.append("gate-fresh-forced", t.id, { gate: "test", attempt, priorAttempt: attempt - 1, commit: gateSubject.commit, reason: "no-commit-timeout-red" });
|
|
6263
|
+
}
|
|
6264
|
+
const replay = !explicitlyFundedAttempt && !freshTimeout && previousRows.length > 0
|
|
5571
6265
|
&& previousRows.every((e) => e.data.commit === gateSubject.commit)
|
|
5572
6266
|
&& previousRows.some((e) => e.data.pass === false && e.data.skipped !== true);
|
|
5573
6267
|
if (replay) {
|
|
@@ -5597,10 +6291,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5597
6291
|
carriedFindings: outstandingFindings,
|
|
5598
6292
|
operatorContext,
|
|
5599
6293
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
5600
|
-
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
6294
|
+
commands, baseline, channels: pools.review, judgeChannels: pools.judge, health, adapters, cfg, artifactDir: journal.dir,
|
|
5601
6295
|
collateral: collateral.get(t.id) ?? [],
|
|
5602
6296
|
selectTests: !testGateFailed,
|
|
5603
|
-
cachedRedBypass: explicitlyFundedAttempt ? "operator-rerun" : undefined,
|
|
6297
|
+
cachedRedBypass: explicitlyFundedAttempt ? "operator-rerun" : freshTimeout ? "timeout-fresh" : undefined,
|
|
5604
6298
|
via: cfg.visibility.llm === "pane"
|
|
5605
6299
|
? {
|
|
5606
6300
|
driver: trackedDriver,
|
|
@@ -5707,6 +6401,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5707
6401
|
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5708
6402
|
return;
|
|
5709
6403
|
}
|
|
6404
|
+
if (await adjudicateInfraShapedRed(t, results, wt, gateSubject.commit, attempt, gateSubject.replayedFromAttempt !== undefined, (reason) => park(t, reason, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode)))
|
|
6405
|
+
return;
|
|
5710
6406
|
// OBS-547: who pays for this red is decided by the run's collateral prediction (see
|
|
5711
6407
|
// dispositionScopeRed) — an authoring defect parks unchargeable before any accounting below.
|
|
5712
6408
|
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
@@ -5799,6 +6495,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5799
6495
|
journal.append("consult-verdict", t.id, {
|
|
5800
6496
|
action: v.action, notes: v.notes,
|
|
5801
6497
|
adapter: v.adapter ?? "unknown", model: v.model ?? "unknown", vendor: v.vendor ?? "unknown",
|
|
6498
|
+
...(v.effort ? { effort: v.effort } : {}),
|
|
6499
|
+
...(v.invocations?.length ? { invocations: v.invocations } : {}),
|
|
5802
6500
|
capAdvisory: true,
|
|
5803
6501
|
});
|
|
5804
6502
|
}
|
|
@@ -5937,6 +6635,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5937
6635
|
for (const t of ready) {
|
|
5938
6636
|
const p = executeWithPolicy(t)
|
|
5939
6637
|
.catch(async (err) => {
|
|
6638
|
+
engagements.delete(t.id);
|
|
5940
6639
|
const cleanupErrors = [];
|
|
5941
6640
|
for (const [slot, owner] of workerOwners) {
|
|
5942
6641
|
if (owner.taskId !== t.id || !liveSlots.has(slot))
|
|
@@ -5968,7 +6667,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5968
6667
|
throw new Error(`could not preserve ${head}: ${saved.stderr}`);
|
|
5969
6668
|
}
|
|
5970
6669
|
journal.append("worktree-preserved", t.id, { ref, ...producerFields(producer) });
|
|
5971
|
-
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: "transport-uncertain", ref, ...cleanupEvidence });
|
|
6670
|
+
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: err instanceof ContactUnreadableExhausted ? "contact-unreadable" : "transport-uncertain", ref, ...cleanupEvidence });
|
|
5972
6671
|
return;
|
|
5973
6672
|
}
|
|
5974
6673
|
if (err instanceof ExecutionBudgetExceeded) {
|
|
@@ -5995,7 +6694,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5995
6694
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
5996
6695
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
5997
6696
|
})
|
|
5998
|
-
.finally(() => inflight.delete(t.id));
|
|
6697
|
+
.finally(() => { inflight.delete(t.id); engagements.delete(t.id); });
|
|
5999
6698
|
inflight.set(t.id, p);
|
|
6000
6699
|
}
|
|
6001
6700
|
if (inflight.size === 0) {
|
|
@@ -6031,8 +6730,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6031
6730
|
const waiters = [...inflight.values(), aborted];
|
|
6032
6731
|
// A free slot is itself a scheduling boundary: poll the append-only approval stream instead of
|
|
6033
6732
|
// sleeping until an unrelated long-running task settles.
|
|
6034
|
-
// An open board is polled on the same cadence, so a dead cockpit is noticed mid-task
|
|
6035
|
-
|
|
6733
|
+
// An open board is polled on the same cadence, so a dead cockpit is noticed mid-task; so is a
|
|
6734
|
+
// held reservation still awaiting its claim or its adoption.
|
|
6735
|
+
if (inflight.size < concurrency || boardOpened || heldRetry) {
|
|
6036
6736
|
waiters.push(new Promise((wake) => setTimeout(wake, APPROVAL_POLL_MS)));
|
|
6037
6737
|
}
|
|
6038
6738
|
await Promise.race(waiters); // aborted rejects on termination — unwinds the run
|
|
@@ -6094,7 +6794,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6094
6794
|
poll = setTimeout(pollApprovals, APPROVAL_POLL_MS);
|
|
6095
6795
|
let tipFailed;
|
|
6096
6796
|
try {
|
|
6097
|
-
const verification = withCommandContext(undefined, () => verifyIntegrationTipCached(intWt, commands, journal, { lastMergedTask, baseline, signal: controller.signal }), controller.signal);
|
|
6797
|
+
const verification = withCommandContext(undefined, () => verifyIntegrationTipCached(intWt, commands, journal, { lastMergedTask, baseline, signal: controller.signal, evidence: { quotaBytes: cfg.gates?.evidenceQuotaBytes } }), controller.signal);
|
|
6098
6798
|
activeTipVerify = { controller, settled: verification.then(() => { }, () => { }) };
|
|
6099
6799
|
tipFailed = await verification;
|
|
6100
6800
|
}
|
|
@@ -6139,6 +6839,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6139
6839
|
summary.approvalDisposition = outstanding.length === 0 ? "complete" : "outstanding";
|
|
6140
6840
|
if (outstanding.length > 0)
|
|
6141
6841
|
summary.outstandingApprovals = outstanding;
|
|
6842
|
+
const forgiven = forgivenFingerprints(journal.read());
|
|
6843
|
+
if (forgiven.length > 0)
|
|
6844
|
+
summary.forgiven = forgiven;
|
|
6142
6845
|
journal.append("run-end", undefined, { ...summary });
|
|
6143
6846
|
await reconcile(); // run-end boundary: nothing in flight — full sweep (empty desired set)
|
|
6144
6847
|
// OBS-28: lingering worktrees starve CLI probes; keepPanes:forever is the debug override.
|