tickmarkr 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/adapters/registry.js +6 -1
- package/dist/adapters/types.d.ts +3 -0
- package/dist/cli/commands/approve.d.ts +1 -0
- package/dist/cli/commands/approve.js +118 -9
- package/dist/cli/commands/doctor.d.ts +10 -0
- package/dist/cli/commands/doctor.js +44 -29
- package/dist/cli/commands/fleet.js +199 -33
- package/dist/cli/commands/init.js +196 -6
- package/dist/cli/commands/plan.js +20 -4
- package/dist/cli/commands/resume.js +4 -2
- package/dist/cli/commands/run.js +11 -2
- package/dist/cli/commands/verify.js +102 -81
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.js +3 -1
- package/dist/config/config.d.ts +25 -8
- package/dist/config/config.js +41 -29
- package/dist/config/fleet-overlay.d.ts +3 -9
- package/dist/config/fleet-overlay.js +114 -19
- package/dist/config/fleet-why.d.ts +7 -0
- package/dist/config/fleet-why.js +5 -0
- package/dist/drivers/index.d.ts +15 -1
- package/dist/drivers/index.js +39 -10
- package/dist/drivers/orca.d.ts +119 -10
- package/dist/drivers/orca.js +781 -110
- package/dist/gates/baseline.d.ts +17 -3
- package/dist/gates/baseline.js +63 -15
- package/dist/gates/cache.d.ts +101 -0
- package/dist/gates/cache.js +401 -0
- package/dist/gates/llm.d.ts +3 -0
- package/dist/gates/llm.js +11 -0
- package/dist/gates/review.d.ts +28 -3
- package/dist/gates/review.js +106 -14
- package/dist/gates/run-gates.d.ts +9 -1
- package/dist/gates/run-gates.js +407 -102
- package/dist/gates/test-manifest.d.ts +128 -0
- package/dist/gates/test-manifest.js +463 -0
- package/dist/gates/test-reporter.d.ts +4 -0
- package/dist/gates/test-reporter.js +57 -0
- package/dist/graph/graph.d.ts +2 -0
- package/dist/graph/graph.js +45 -1
- package/dist/route/preference.d.ts +22 -1
- package/dist/route/preference.js +123 -25
- package/dist/route/router.js +31 -6
- package/dist/run/consult.js +5 -4
- package/dist/run/daemon.d.ts +15 -0
- package/dist/run/daemon.js +638 -167
- package/dist/run/execution-budget.d.ts +25 -0
- package/dist/run/execution-budget.js +142 -0
- package/dist/run/git.d.ts +50 -1
- package/dist/run/git.js +131 -12
- package/dist/run/journal.d.ts +16 -2
- package/dist/run/journal.js +115 -25
- package/dist/run/lease.d.ts +58 -0
- package/dist/run/lease.js +310 -0
- package/dist/run/merge.d.ts +2 -0
- package/dist/run/merge.js +91 -3
- package/dist/run/operator-state.d.ts +11 -0
- package/dist/run/operator-state.js +17 -3
- package/dist/run/recovery.d.ts +8 -0
- package/dist/run/recovery.js +25 -0
- package/dist/run/repair-selection.d.ts +12 -0
- package/dist/run/repair-selection.js +56 -0
- package/dist/run/stall.d.ts +6 -1
- package/dist/run/stall.js +60 -3
- package/dist/tui/cockpit/board.d.ts +96 -0
- package/dist/tui/cockpit/board.js +346 -0
- package/dist/tui/cockpit/decision-actions.js +2 -0
- package/dist/tui/cockpit/layout.d.ts +5 -1
- package/dist/tui/cockpit/layout.js +8 -3
- package/dist/tui/cockpit/live-runtime.js +71 -21
- package/dist/tui/cockpit/run-view.d.ts +7 -5
- package/dist/tui/cockpit/run-view.js +12 -11
- package/dist/tui/ink/fleet-app.d.ts +49 -27
- package/dist/tui/ink/fleet-app.js +229 -38
- package/package.json +2 -2
- package/skills/tickmarkr-overseer/SKILL.md +173 -113
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +29 -2
package/dist/run/daemon.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { COMMAND_LEASE_TOKEN_ENV, commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
1
2
|
import { execFileSync, spawn } from "node:child_process";
|
|
2
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
4
|
import { shq } from "../adapters/types.js";
|
|
@@ -9,10 +10,10 @@ import { fileURLToPath } from "node:url";
|
|
|
9
10
|
import { stringify } from "yaml";
|
|
10
11
|
import { classifyDeadChannel, NO_TRAILER_SUMMARY, trailerPattern, UNPARSEABLE_TRAILER_SUMMARY, writePrompt } from "../adapters/prompt.js";
|
|
11
12
|
import { allAdapters, getAdapter, probeAll, readDoctor, rolePools } from "../adapters/registry.js";
|
|
12
|
-
import { SettledTrailerTracker, addUsage, channelKey, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
13
|
+
import { SettledTrailerTracker, addUsage, channelKey, matchesInputBox, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
13
14
|
import { bannerShell, paneDispatchCommand } from "../brand.js";
|
|
14
15
|
import { collateralHits } from "../compile/collateral.js";
|
|
15
|
-
import { DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
16
|
+
import { ExecutionPolicySchema, DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
16
17
|
import { DeliveryReadinessError } from "../drivers/herdr.js";
|
|
17
18
|
import { driverEvidence } from "../drivers/index.js";
|
|
18
19
|
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
@@ -24,17 +25,20 @@ import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHa
|
|
|
24
25
|
import { GATE_NAMES } from "../graph/schema.js";
|
|
25
26
|
import { distFingerprint } from "../cli/commands/version.js";
|
|
26
27
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
28
|
+
import { executionSignal, remainingExecutionMs, withExecutionBudget, withoutExecutionBudget, ExecutionBudgetExceeded } from "./execution-budget.js";
|
|
29
|
+
import { repairSelectionDecision } from "./repair-selection.js";
|
|
30
|
+
import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
27
31
|
import { runEnvironment } from "./environment.js";
|
|
28
|
-
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sh, shGit, SUITE_PARENT_ENV, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
32
|
+
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
29
33
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
30
|
-
import { activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
34
|
+
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
31
35
|
import { isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
32
36
|
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
33
37
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
34
38
|
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
35
39
|
import { desiredPanes } from "./reconcile.js";
|
|
36
40
|
import { readTierLiveness, readWatchBoard, supervisionBeatPath } from "./supervision.js";
|
|
37
|
-
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, } from "./stall.js";
|
|
41
|
+
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, reapOwnedProcessGroup, readOwnedProcessGroup, } from "./stall.js";
|
|
38
42
|
// The live set is also the ownership claim shared by task cleanup and termination.
|
|
39
43
|
export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
40
44
|
if (!liveSlots.delete(slot))
|
|
@@ -47,6 +51,10 @@ export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
|
47
51
|
throw error;
|
|
48
52
|
}
|
|
49
53
|
}
|
|
54
|
+
// A fixed attempt ceiling is independent of the rolling inactivity clock.
|
|
55
|
+
let attemptHardTimeoutMs;
|
|
56
|
+
export function setAttemptHardTimeoutMsForTests(ms) { attemptHardTimeoutMs = ms; }
|
|
57
|
+
export function resetAttemptHardTimeoutMsForTests() { attemptHardTimeoutMs = undefined; }
|
|
50
58
|
// Compatibility exports: gate dispatch can import stall.ts without importing the daemon.
|
|
51
59
|
export { harvestCpuFlatWindowMs, resetHarvestCpuFlatMsForTests, setHarvestCpuFlatMsForTests, workerTreeCpuMs, } from "./stall.js";
|
|
52
60
|
const MODE_RANK = { "staff-led": 0, "risk-based": 1, "partner-led": 2 };
|
|
@@ -260,6 +268,22 @@ export function decisiveReviewRounds(events) {
|
|
|
260
268
|
close(taskId); // Map iteration tolerates deleting the current key
|
|
261
269
|
return events.filter((e) => !spent.has(e));
|
|
262
270
|
}
|
|
271
|
+
/** OBS-1034: the files a byte cap drops from a repair diff — every file whose header starts at or past the cap, plus the one the cut lands inside.
|
|
272
|
+
* Offsets are UTF-8 BYTES (the cap's unit), never UTF-16 code units. `names` is the NUL-delimited `git diff --name-only` list in
|
|
273
|
+
* header order, so quoted headers (tabs, quotes, non-ASCII) still name their file. */
|
|
274
|
+
function repairDiffFilesCut(diff, cap, names) {
|
|
275
|
+
const files = [];
|
|
276
|
+
const header = Buffer.from("\ndiff --git ");
|
|
277
|
+
const starts = diff.subarray(0, header.length - 1).equals(header.subarray(1)) ? [0] : [];
|
|
278
|
+
for (let at = diff.indexOf(header); at !== -1; at = diff.indexOf(header, at + 1))
|
|
279
|
+
starts.push(at + 1);
|
|
280
|
+
starts.forEach((start, i) => {
|
|
281
|
+
const end = starts[i + 1] === undefined ? diff.length : starts[i + 1] - 1;
|
|
282
|
+
if (start >= cap || end > cap)
|
|
283
|
+
files.push(names[i] ?? "(unnamed)");
|
|
284
|
+
});
|
|
285
|
+
return files;
|
|
286
|
+
}
|
|
263
287
|
/** The fix-only contract: the findings verbatim, then the diff content of the work already landed. */
|
|
264
288
|
function repairBrief(findings, diff, baseRef) {
|
|
265
289
|
const repair = { findings, diff };
|
|
@@ -379,14 +403,67 @@ export function setWorkerStartupWindowMsForTests(ms) {
|
|
|
379
403
|
export function resetWorkerStartupWindowMsForTests() {
|
|
380
404
|
workerStartupWindowMs = WORKER_STARTUP_WINDOW_MS;
|
|
381
405
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
406
|
+
/** One seat, one startup prefix. Once execution or its input box is seen, later panes are ineligible. */
|
|
407
|
+
export class StartupFailureDetector {
|
|
408
|
+
adapter;
|
|
409
|
+
closed = false;
|
|
410
|
+
constructor(adapter) {
|
|
411
|
+
this.adapter = adapter;
|
|
412
|
+
}
|
|
413
|
+
sample(output, launchedAt) {
|
|
414
|
+
if (this.closed || launchedAt === undefined || Date.now() - launchedAt > workerStartupWindowMs)
|
|
415
|
+
return;
|
|
416
|
+
// read() returns a tail, not a guaranteed transcript origin. A saturated read cannot
|
|
417
|
+
// establish startup ownership: the first tool frame may already have scrolled out.
|
|
418
|
+
if (Buffer.byteLength(output) >= WORKER_STARTUP_WINDOW_BYTES || output.split("\n").length >= 500) {
|
|
419
|
+
this.closed = true;
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
const bounded = output;
|
|
423
|
+
const rows = bounded.split("\n");
|
|
424
|
+
let inputStart = rows.length;
|
|
425
|
+
const input = this.adapter.inputBox;
|
|
426
|
+
if (input && matchesInputBox(bounded, input)) {
|
|
427
|
+
// Locate the beginning, including multi-row composers whose footer proves the matcher.
|
|
428
|
+
let end = rows.length;
|
|
429
|
+
let low = 1;
|
|
430
|
+
while (low < end) {
|
|
431
|
+
const mid = Math.floor((low + end) / 2);
|
|
432
|
+
if (matchesInputBox(rows.slice(0, mid).join("\n"), input))
|
|
433
|
+
end = mid;
|
|
434
|
+
else
|
|
435
|
+
low = mid + 1;
|
|
436
|
+
}
|
|
437
|
+
inputStart = 0;
|
|
438
|
+
let high = end;
|
|
439
|
+
while (inputStart + 1 < high) {
|
|
440
|
+
const mid = Math.floor((inputStart + high) / 2);
|
|
441
|
+
if (matchesInputBox(rows.slice(mid, end).join("\n"), input))
|
|
442
|
+
inputStart = mid;
|
|
443
|
+
else
|
|
444
|
+
high = mid;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
let offset = 0;
|
|
448
|
+
for (const [index, row] of rows.entries()) {
|
|
449
|
+
const cleanRow = row.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "");
|
|
450
|
+
// Structured tool frames and the terminal's tool headings close the prefix BEFORE their body.
|
|
451
|
+
const toolFrame = /"(?:type|role)"\s*:\s*"(?:tool[^" ]*|function_call[^" ]*|command_execution)"|^\s*(?:[•⏺]\s*(?:Ran|Explored|Searched|Called|Bash|Read|Write|Edit|Shell|Exec|Tool)\b|⎋ |(?:tool[-_ ](?:call|output|result)|function_call)\b)/i.test(cleanRow);
|
|
452
|
+
if (toolFrame || index >= inputStart) {
|
|
453
|
+
this.closed = true;
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
if (!this.adapter.harnessBannerRows?.includes(cleanRow)) {
|
|
457
|
+
const match = WORKER_STARTUP_FAILURE_RE.exec(row);
|
|
458
|
+
if (match)
|
|
459
|
+
return {
|
|
460
|
+
matchedBytes: match[0], offset: offset + Buffer.byteLength(row.slice(0, match.index)),
|
|
461
|
+
row, rowNumber: index + 1,
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
offset += Buffer.byteLength(row) + 1;
|
|
465
|
+
}
|
|
466
|
+
}
|
|
390
467
|
}
|
|
391
468
|
// OBS-901/906: one daemon-owned writer for every execution surface. Drivers expose one retained
|
|
392
469
|
// stream through read(); the daemon persists that exact surface before any close/reconcile can make
|
|
@@ -570,17 +647,54 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
570
647
|
});
|
|
571
648
|
const { verifyIntegrationTip } = await import(${JSON.stringify(new URL(`./merge.${extension}`, import.meta.url).href)});
|
|
572
649
|
const { runWithVerificationBudget } = await import(${JSON.stringify(new URL(`./git.${extension}`, import.meta.url).href)});
|
|
650
|
+
const { runWithCommandLease } = await import(${JSON.stringify(new URL(`./lease.${extension}`, import.meta.url).href)});
|
|
651
|
+
let sequence = 0;
|
|
652
|
+
const grants = new Map();
|
|
653
|
+
const commandCapacities = new Map();
|
|
654
|
+
process.on('message', (message) => {
|
|
655
|
+
if (message.type === 'lease-grant') { grants.get(message.id)?.(message); grants.delete(message.id); }
|
|
656
|
+
});
|
|
657
|
+
const lease = async (command, execute) => {
|
|
658
|
+
const id = ++sequence;
|
|
659
|
+
const grant = new Promise(resolve => grants.set(id, resolve));
|
|
660
|
+
process.send({ type: 'lease-acquire', id, command });
|
|
661
|
+
const { capacity, token } = await grant;
|
|
662
|
+
commandCapacities.set(command, capacity);
|
|
663
|
+
try { return await runWithVerificationBudget(capacity, () => execute(token)); }
|
|
664
|
+
finally { process.send({ type: 'lease-release', id }); }
|
|
665
|
+
};
|
|
573
666
|
let input = ''; for await (const chunk of process.stdin) input += chunk;
|
|
574
667
|
const { intWt, commands, runDir, baseline, capacity } = JSON.parse(input);
|
|
575
668
|
try {
|
|
576
|
-
const result = await runWithVerificationBudget(capacity, () => verifyIntegrationTip(intWt, commands, runDir, baseline));
|
|
577
|
-
if (!cancelled) process.stdout.write(JSON.stringify(result));
|
|
669
|
+
const result = await runWithVerificationBudget(capacity, () => runWithCommandLease(lease, () => verifyIntegrationTip(intWt, commands, runDir, baseline)));
|
|
670
|
+
if (!cancelled) process.stdout.write(JSON.stringify(result.map(row => ({ ...row, capacity: [...commandCapacities].reverse().find(([command]) => command === row.cmd || command.startsWith(row.cmd + " "))?.[1] ?? capacity }))));
|
|
578
671
|
} catch (error) { if (!cancelled) throw error; }
|
|
672
|
+
finally { process.disconnect(); }
|
|
579
673
|
`;
|
|
580
674
|
const child = spawn(process.execPath, [
|
|
581
675
|
...(extension === "ts" ? ["--import", createRequire(import.meta.url).resolve("tsx")] : []),
|
|
582
676
|
"--input-type=module", "-e", script,
|
|
583
|
-
], { stdio: ["pipe", "pipe", "pipe"], detached: true });
|
|
677
|
+
], { env: commandLeaseEnvironment(process.env), stdio: ["pipe", "pipe", "pipe", "ipc"], detached: true });
|
|
678
|
+
const releases = new Map();
|
|
679
|
+
const leases = new Set();
|
|
680
|
+
let closed = false;
|
|
681
|
+
child.on("message", (message) => {
|
|
682
|
+
if (message.type === "lease-release") {
|
|
683
|
+
releases.get(message.id)?.();
|
|
684
|
+
releases.delete(message.id);
|
|
685
|
+
}
|
|
686
|
+
if (message.type !== "lease-acquire")
|
|
687
|
+
return;
|
|
688
|
+
const held = withCommandLease(message.command, async () => {
|
|
689
|
+
if (closed || signal.aborted)
|
|
690
|
+
return;
|
|
691
|
+
const released = new Promise((resolve) => releases.set(message.id, resolve));
|
|
692
|
+
child.send({ type: "lease-grant", id: message.id, capacity: resolvedCapacity(), token: currentCommandLeaseToken() });
|
|
693
|
+
await released;
|
|
694
|
+
});
|
|
695
|
+
leases.add(held);
|
|
696
|
+
void held.catch(() => { }).finally(() => leases.delete(held));
|
|
697
|
+
});
|
|
584
698
|
let output = "";
|
|
585
699
|
let errors = "";
|
|
586
700
|
child.stdout.on("data", (chunk) => { output += chunk; });
|
|
@@ -589,6 +703,10 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
589
703
|
const finished = new Promise((resolve, reject) => {
|
|
590
704
|
child.once("error", reject);
|
|
591
705
|
child.once("close", (code) => {
|
|
706
|
+
closed = true;
|
|
707
|
+
for (const release of releases.values())
|
|
708
|
+
release();
|
|
709
|
+
releases.clear();
|
|
592
710
|
if (signal.aborted && (code === 143 || code === null))
|
|
593
711
|
reject(signal.reason);
|
|
594
712
|
else if (code !== 0)
|
|
@@ -608,6 +726,7 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
608
726
|
}
|
|
609
727
|
finally {
|
|
610
728
|
signal.removeEventListener("abort", cancel);
|
|
729
|
+
await Promise.allSettled(leases);
|
|
611
730
|
}
|
|
612
731
|
}
|
|
613
732
|
/**
|
|
@@ -662,9 +781,10 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
662
781
|
for (const r of await (opts.signal
|
|
663
782
|
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal)
|
|
664
783
|
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline))) {
|
|
784
|
+
const measuredCapacity = r.capacity ?? capacity;
|
|
665
785
|
if (r.pass) {
|
|
666
786
|
// Q121s: a forgiven pass journals its fingerprints — honest about what was carried, never a silent green.
|
|
667
|
-
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity });
|
|
787
|
+
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
668
788
|
}
|
|
669
789
|
else {
|
|
670
790
|
journal.append("tip-verify-failed", undefined, {
|
|
@@ -677,13 +797,19 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
677
797
|
lastMergedTask: opts.lastMergedTask,
|
|
678
798
|
tip,
|
|
679
799
|
cmdHash,
|
|
680
|
-
capacity,
|
|
800
|
+
capacity: measuredCapacity,
|
|
681
801
|
});
|
|
682
802
|
tipFailed = true;
|
|
683
803
|
}
|
|
684
804
|
}
|
|
685
805
|
return tipFailed;
|
|
686
806
|
}
|
|
807
|
+
async function commitsAheadOfRef(base, head, cwd) {
|
|
808
|
+
if (head === base)
|
|
809
|
+
return [];
|
|
810
|
+
const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, cwd);
|
|
811
|
+
return r.code === 0 ? r.stdout.trim().split("\n").filter(Boolean) : [];
|
|
812
|
+
}
|
|
687
813
|
async function commitsAheadOf(base, wt) {
|
|
688
814
|
const head = await gitHead(wt);
|
|
689
815
|
if (head === base)
|
|
@@ -734,7 +860,6 @@ async function observeWorkerProcessTree(marker, cwd) {
|
|
|
734
860
|
}
|
|
735
861
|
return tree.size === 0 ? "empty" : "running";
|
|
736
862
|
}
|
|
737
|
-
const SUITE_COMMAND_RE = /(?:^|[\s/])(vitest(?:\.mjs)?|jest|mocha)(?:[\s/]|$)|\bnpm(?:\s+run)?\s+test\b/i;
|
|
738
863
|
function processCwd(pid) {
|
|
739
864
|
try {
|
|
740
865
|
return realpathSync(readlinkSync(`/proc/${pid}/cwd`));
|
|
@@ -802,7 +927,7 @@ export function countLiveSuites(snapshot, repoRoot, daemonPid = process.pid, cwd
|
|
|
802
927
|
// runner ("…as a vitest test whose…"), so a finished interactive worker counted as a live suite and
|
|
803
928
|
// held a task's gates for 9 min 46 s. A runner is named in a command's HEAD — `node <bin>`,
|
|
804
929
|
// `npm test`, `npx vitest`, `sh -c npm test` — never 140 KB into it: read the first four tokens only.
|
|
805
|
-
const candidates = rows.filter((row) => !ancestors.has(row.pid) &&
|
|
930
|
+
const candidates = rows.filter((row) => !ancestors.has(row.pid) && isRunnerCommand(row.command));
|
|
806
931
|
const attributable = new Set(candidates.filter((row) => {
|
|
807
932
|
if (descendants.has(row.pid))
|
|
808
933
|
return true;
|
|
@@ -1280,6 +1405,12 @@ function endedRunIdsForReconcile(repoRoot) {
|
|
|
1280
1405
|
return ended;
|
|
1281
1406
|
}
|
|
1282
1407
|
export async function runDaemon(repoRoot, opts = {}) {
|
|
1408
|
+
// OBS-1045: a daemon launched from inside a leased command (a nested runner, an operator's test
|
|
1409
|
+
// shell) inherits its ancestor's lease token through the environment, and every gate it ran would
|
|
1410
|
+
// ride that lease instead of taking its own. Clear it at run-start so the first gate leases itself.
|
|
1411
|
+
if (process.env[COMMAND_LEASE_TOKEN_ENV]) {
|
|
1412
|
+
delete process.env[COMMAND_LEASE_TOKEN_ENV];
|
|
1413
|
+
}
|
|
1283
1414
|
// v1.51 T2 / OBS-89 (v1.60): retired --quality env seam. Mode resolution owns premium routing;
|
|
1284
1415
|
// route() no longer reads the retired env at all, so the old entrypoint scrub is gone with it.
|
|
1285
1416
|
const adapters = opts.adapters ?? allAdapters();
|
|
@@ -1346,8 +1477,23 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1346
1477
|
// > repo > global > default. The resolved cfg carries mode-compiled floors; route() never sees the mode.
|
|
1347
1478
|
const rm = resolveRunMode(repoRoot, { flag: opts.mode, spec: graph.mode, globalDir: opts.globalDir });
|
|
1348
1479
|
const cfg = rm.cfg;
|
|
1480
|
+
if (opts.resume) {
|
|
1481
|
+
const start = Journal.open(repoRoot, runId).read().find((e) => e.event === "run-start");
|
|
1482
|
+
const snapshot = start?.data.effectivePolicy;
|
|
1483
|
+
if (snapshot !== undefined && (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)
|
|
1484
|
+
|| !("config" in snapshot) || !snapshot.config || typeof snapshot.config !== "object" || Array.isArray(snapshot.config))) {
|
|
1485
|
+
throw new Error("cannot restore malformed recorded execution policy");
|
|
1486
|
+
}
|
|
1487
|
+
const recorded = snapshot;
|
|
1488
|
+
// Historical runs did not opt in; current settings cannot grant them a new allowance.
|
|
1489
|
+
cfg.executionPolicy = recorded?.config?.executionPolicy === undefined
|
|
1490
|
+
? undefined : ExecutionPolicySchema.parse(recorded.config.executionPolicy);
|
|
1491
|
+
}
|
|
1492
|
+
if (cfg.executionPolicy && driver.id !== "subprocess") {
|
|
1493
|
+
throw new Error("experimental executionPolicy currently requires the subprocess driver");
|
|
1494
|
+
}
|
|
1349
1495
|
// Resolve both budgets once: worker fan-out divides by dispatch concurrency; verification
|
|
1350
|
-
// divides by one
|
|
1496
|
+
// divides by one runner command holding a lease. Explicit operator caps still win.
|
|
1351
1497
|
const concurrency = opts.concurrency ?? cfg.concurrency;
|
|
1352
1498
|
return await runWithForkBudget(concurrency, async () => {
|
|
1353
1499
|
const conservativeCapacity = resolvedCapacity();
|
|
@@ -1376,14 +1522,58 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1376
1522
|
// opens/closes keep the ledger exact. Termination then closes exactly what is still live — and a
|
|
1377
1523
|
// slot closed once (task-done, quota reroute, gate self-clean) can never be closed twice.
|
|
1378
1524
|
const liveSlots = new Set();
|
|
1379
|
-
|
|
1525
|
+
// A driver handle identifies the physical slot even if a retry returns a new Slot object.
|
|
1526
|
+
const trustAnsweredSlots = new Set();
|
|
1527
|
+
const workerOwners = new Map();
|
|
1528
|
+
const reapReports = new Map();
|
|
1529
|
+
const reapWorker = async (slot) => {
|
|
1530
|
+
const owner = workerOwners.get(slot);
|
|
1531
|
+
if (owner) {
|
|
1532
|
+
const processGroup = readOwnedProcessGroup(owner.groupFile);
|
|
1533
|
+
let survivors;
|
|
1534
|
+
try {
|
|
1535
|
+
const shared = processGroup !== undefined && [...workerOwners].some(([other, otherOwner]) => other !== slot && liveSlots.has(other) && readOwnedProcessGroup(otherOwner.groupFile) === processGroup);
|
|
1536
|
+
if (shared)
|
|
1537
|
+
throw new Error(`worker group ${processGroup} is shared with another live attempt`);
|
|
1538
|
+
survivors = await reapOwnedProcessGroup(processGroup, slot.cwd);
|
|
1539
|
+
}
|
|
1540
|
+
catch (error) {
|
|
1541
|
+
journal.append("worker-process-reaped", owner.taskId, { slot: slot.name, attempt: owner.attempt,
|
|
1542
|
+
processGroup: processGroup ?? null, survivors: null, error: String(error) });
|
|
1543
|
+
throw error;
|
|
1544
|
+
}
|
|
1545
|
+
reapReports.set(slot, { processGroup: processGroup ?? null, survivors });
|
|
1546
|
+
journal.append("worker-process-reaped", owner.taskId, {
|
|
1547
|
+
slot: slot.name, attempt: owner.attempt, processGroup: processGroup ?? null, survivors,
|
|
1548
|
+
});
|
|
1549
|
+
if (survivors && survivors.length > 0)
|
|
1550
|
+
throw new Error(`worker group ${processGroup} survivors: ${survivors.join(", ")}`);
|
|
1551
|
+
}
|
|
1552
|
+
};
|
|
1553
|
+
const closeSlot = (s) => closeLiveSlot(liveSlots, { close: async (slot) => {
|
|
1554
|
+
await reapWorker(slot);
|
|
1555
|
+
await driver.close(slot);
|
|
1556
|
+
} }, s);
|
|
1557
|
+
const budgetSlots = new Map();
|
|
1380
1558
|
const trackedDriver = {
|
|
1381
1559
|
id: driver.id,
|
|
1382
1560
|
interactive: driver.interactive,
|
|
1383
1561
|
...(driver.readSource ? { readSource: driver.readSource } : {}),
|
|
1384
1562
|
...(driver.describe ? { describe: driver.describe.bind(driver) } : {}),
|
|
1385
1563
|
...(driver.focus ? { focus: driver.focus.bind(driver) } : {}),
|
|
1386
|
-
slot: async (cwd, name, o) => {
|
|
1564
|
+
slot: async (cwd, name, o) => {
|
|
1565
|
+
const signal = executionSignal();
|
|
1566
|
+
signal?.throwIfAborted();
|
|
1567
|
+
const s = await driver.slot(cwd, name, o);
|
|
1568
|
+
liveSlots.add(s);
|
|
1569
|
+
if (signal)
|
|
1570
|
+
budgetSlots.get(signal)?.add(s);
|
|
1571
|
+
if (signal?.aborted) {
|
|
1572
|
+
await withoutExecutionBudget(() => closeSlot(s));
|
|
1573
|
+
signal.throwIfAborted();
|
|
1574
|
+
}
|
|
1575
|
+
return s;
|
|
1576
|
+
},
|
|
1387
1577
|
run: (s, cmd) => driver.run(s, cmd),
|
|
1388
1578
|
waitOutput: (s, p, ms, o) => driver.waitOutput(s, p, ms, o),
|
|
1389
1579
|
waitAgentStatus: (s, st, ms) => driver.waitAgentStatus(s, st, ms),
|
|
@@ -1444,6 +1634,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1444
1634
|
activeTipVerify.controller.abort(termination);
|
|
1445
1635
|
await activeTipVerify.settled;
|
|
1446
1636
|
}
|
|
1637
|
+
if (cfg.visibility.keepPanes === "forever") {
|
|
1638
|
+
for (const s of Array.from(liveSlots)) {
|
|
1639
|
+
try {
|
|
1640
|
+
await reapWorker(s);
|
|
1641
|
+
}
|
|
1642
|
+
catch { /* survivor evidence is retained by the reaper */ }
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1447
1645
|
if (cfg.visibility.keepPanes !== "forever") {
|
|
1448
1646
|
for (const s of Array.from(liveSlots)) { // snapshot: failed closes restore membership for the next sweep
|
|
1449
1647
|
try {
|
|
@@ -1467,6 +1665,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1467
1665
|
const demotedReviewers = new Set(journal.read()
|
|
1468
1666
|
.filter((event) => event.event === "review-pool-demotion" && typeof event.data.reviewer === "string")
|
|
1469
1667
|
.map((event) => event.data.reviewer));
|
|
1668
|
+
// OBS-1010: a demotion the JOURNAL carries into this session left the rotation for the run — a
|
|
1669
|
+
// resumed daemon excludes that seat from every review pick (reviewer role only; the worker channel
|
|
1670
|
+
// is untouched — channel-demotion is the worker fold, replayExcludedChannels). In-session demotions
|
|
1671
|
+
// keep their soft ordering above so a still-live seat can be re-tried at the end of the rotation.
|
|
1672
|
+
const replayedReviewerExclusions = opts.resume ? [...demotedReviewers] : [];
|
|
1673
|
+
// OBS-1025 add.2 / OBS-1052 (Leg-2): the run-scoped no-verdict tally run-gates retires a seat from at two
|
|
1674
|
+
// strikes. execTask's badReviewers already excludes a seat after ONE strike within a task, so this map
|
|
1675
|
+
// is what bounds a seat that flakes once per task (or once per session) across the run. Seeded from the
|
|
1676
|
+
// journal's review-no-verdict rows — the same notes noteReviewEvent appends — so a resumed daemon keeps
|
|
1677
|
+
// the count; run-gates appends to it in place, exactly as it does demotedReviewers.
|
|
1678
|
+
const reviewNoVerdicts = new Map();
|
|
1679
|
+
for (const event of journal.read()) {
|
|
1680
|
+
if (event.event !== "review-no-verdict" || event.data.noVerdict !== true || typeof event.data.reviewer !== "string")
|
|
1681
|
+
continue;
|
|
1682
|
+
reviewNoVerdicts.set(event.data.reviewer, [...(reviewNoVerdicts.get(event.data.reviewer) ?? []), String(event.data.cause)]);
|
|
1683
|
+
}
|
|
1470
1684
|
const reviewHistory = journal.read()
|
|
1471
1685
|
.filter((event) => event.event === "gate-result" && event.data.gate === "review" && typeof event.data.reviewer === "string")
|
|
1472
1686
|
.map((event) => event.data.reviewer);
|
|
@@ -1604,59 +1818,49 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1604
1818
|
journal.append("watch-board-reopen-failed", undefined, { pane: loss.pane, attempt, error: reopened.error, ...(boardless ? { boardless: true } : {}) });
|
|
1605
1819
|
console.error(`tickmarkr: board not reopened (attempt ${attempt}): ${reopened.error}`);
|
|
1606
1820
|
};
|
|
1607
|
-
//
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
for (;;) {
|
|
1628
|
-
signal?.throwIfAborted();
|
|
1629
|
-
const count = await liveSuiteCount(repoRoot);
|
|
1630
|
-
if (count === 0)
|
|
1631
|
-
break;
|
|
1632
|
-
// OBS-889: a census that never reaches zero held T5's gates with no row and no end. Proceed at
|
|
1633
|
-
// the ceiling, flagged: the verdict that follows was produced beside whatever is still counted.
|
|
1634
|
-
if (Date.now() - startedAt >= suiteWaitCeilingMs) {
|
|
1635
|
-
journal.append("suite-wait-ceiling", taskId, { count, waitedMs: Date.now() - startedAt });
|
|
1636
|
-
journal.append("suite-budget", taskId, {
|
|
1637
|
-
count, occupancyCap: occupancyCapacity.forkCap, conservativeCap: conservativeCapacity.forkCap,
|
|
1638
|
-
});
|
|
1639
|
-
return await runWithVerificationBudget(conservativeCapacity, run);
|
|
1640
|
-
}
|
|
1641
|
-
if (count !== lastCount)
|
|
1642
|
-
journal.append("suite-wait", taskId, { count });
|
|
1643
|
-
lastCount = count;
|
|
1644
|
-
await new Promise((wake) => setTimeout(wake, SUITE_POLL_MS));
|
|
1821
|
+
// Context carries attribution through gates and remote inference; only shell commands acquire.
|
|
1822
|
+
const commandLeases = new CommandLeases();
|
|
1823
|
+
const withCommandContext = (taskId, run, signal = executionSignal()) => runWithCommandLease((_command, execute) => commandLeases.run(async () => {
|
|
1824
|
+
let lastCount = -1;
|
|
1825
|
+
const startedAt = Date.now();
|
|
1826
|
+
for (;;) {
|
|
1827
|
+
signal?.throwIfAborted();
|
|
1828
|
+
executionSignal()?.throwIfAborted();
|
|
1829
|
+
const count = await liveSuiteCount(repoRoot);
|
|
1830
|
+
if (count === 0)
|
|
1831
|
+
break;
|
|
1832
|
+
if (count !== lastCount)
|
|
1833
|
+
journal.append("suite-wait", taskId, { count });
|
|
1834
|
+
lastCount = count;
|
|
1835
|
+
if (Date.now() - startedAt >= suiteWaitCeilingMs) {
|
|
1836
|
+
journal.append("suite-wait-ceiling", taskId, { count, waitedMs: Date.now() - startedAt });
|
|
1837
|
+
journal.append("suite-budget", taskId, {
|
|
1838
|
+
count, occupancyCap: occupancyCapacity.forkCap, conservativeCap: conservativeCapacity.forkCap,
|
|
1839
|
+
});
|
|
1840
|
+
return await runWithVerificationBudget(conservativeCapacity, execute);
|
|
1645
1841
|
}
|
|
1646
|
-
|
|
1647
|
-
}
|
|
1648
|
-
finally {
|
|
1649
|
-
suitePending--;
|
|
1650
|
-
release();
|
|
1842
|
+
await new Promise((wake) => setTimeout(wake, SUITE_POLL_MS));
|
|
1651
1843
|
}
|
|
1652
|
-
|
|
1844
|
+
return await execute();
|
|
1845
|
+
}, (count) => journal.append("suite-wait", taskId, { count }), SUITE_POLL_MS, signal), run);
|
|
1653
1846
|
let baseRef;
|
|
1654
1847
|
let baseline;
|
|
1655
1848
|
let baselinePending = false;
|
|
1656
1849
|
const waitForBaseline = async (taskId) => {
|
|
1657
1850
|
if (baselinePending)
|
|
1658
1851
|
journal.append("baseline-wait", taskId, { baseRef });
|
|
1659
|
-
|
|
1852
|
+
const signal = executionSignal();
|
|
1853
|
+
if (!signal) {
|
|
1854
|
+
await baselineCapture;
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
signal.throwIfAborted();
|
|
1858
|
+
// Capture is run-owned; stop this task waiting without cancelling another task's baseline.
|
|
1859
|
+
await new Promise((resolve, reject) => {
|
|
1860
|
+
const abort = () => reject(signal.reason);
|
|
1861
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
1862
|
+
baselineCapture.then(() => { signal.removeEventListener("abort", abort); resolve(); }, (error) => { signal.removeEventListener("abort", abort); reject(error); });
|
|
1863
|
+
});
|
|
1660
1864
|
};
|
|
1661
1865
|
// Phase 46 (RES-01/RES-02): the resume-state map is built ONCE here so execTask closes over it.
|
|
1662
1866
|
// Empty Map on fresh runs — every seed below conditions on resume.get(t.id), never on opts.resume (the
|
|
@@ -1671,7 +1875,23 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1671
1875
|
// every shell this run spawns will divide the machine by. Recorded evidence from another session
|
|
1672
1876
|
// is only reusable against this.
|
|
1673
1877
|
const sessionCapacity = resolvedCapacity();
|
|
1878
|
+
// R41: run-start/run-resume provenance — the protocol + effective lifecycle policy measured at
|
|
1879
|
+
// the repository root when this session began. Gate-result rows do NOT carry this value: each row
|
|
1880
|
+
// carries what run-gates measured for the task worktree the gate ran in (meta.verification), and
|
|
1881
|
+
// the replay guard below compares against a fresh measurement for the recreated worktree.
|
|
1882
|
+
const sessionVerification = verificationProtocol(process.env, repoRoot);
|
|
1674
1883
|
const replayedExclusions = opts.resume ? journal.replayExcludedChannels() : new Set();
|
|
1884
|
+
// Stamp each daemon session, including resumes with changed configuration or CLI overrides.
|
|
1885
|
+
// Keep the loaded-config hash's existing meaning; resolved execution inputs are explicit beside it.
|
|
1886
|
+
// Journal.append applies the normal credential redaction to the readable policy snapshot.
|
|
1887
|
+
const sessionMetadata = {
|
|
1888
|
+
environment: runEnvironment(cfg, channels, health),
|
|
1889
|
+
distFingerprint: distFingerprint(),
|
|
1890
|
+
verification: sessionVerification,
|
|
1891
|
+
effectivePolicy: { config: cfg, concurrency, driver: driver.id, commands },
|
|
1892
|
+
mode: rm.mode.mode,
|
|
1893
|
+
modeSource: rm.source,
|
|
1894
|
+
};
|
|
1675
1895
|
if (opts.resume) {
|
|
1676
1896
|
// v1.53 T5: a superseded run is dead — resuming it beside its successor is the exact
|
|
1677
1897
|
// two-concurrent-runs hazard supersession exists to prevent. Fail closed, naming the successor.
|
|
@@ -1687,6 +1907,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1687
1907
|
// The SAME comparator status uses (engagementComparable); one decision, two consumers. Fail closed:
|
|
1688
1908
|
// no resume path silently accepts a mismatched or unbound journal. --graph-changed is the operator's
|
|
1689
1909
|
// audited release for the stop-amend-resume workflow, journaling a graph-rehash naming both hashes.
|
|
1910
|
+
graph = applyScopeAmendments(graph, journal, true);
|
|
1690
1911
|
const loadedHash = graphDefinitionHash(graph);
|
|
1691
1912
|
const cmp = engagementComparable(journal.read(), loadedHash);
|
|
1692
1913
|
if (!cmp.comparable) {
|
|
@@ -1719,6 +1940,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1719
1940
|
graph = setStatus(graph, id, st);
|
|
1720
1941
|
}
|
|
1721
1942
|
journal.append("run-resume", undefined, {
|
|
1943
|
+
...sessionMetadata,
|
|
1722
1944
|
pid: process.pid, // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
1723
1945
|
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
1724
1946
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
@@ -1732,7 +1954,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1732
1954
|
baselinePending = true;
|
|
1733
1955
|
// Workers can run beside capture, but no gate may observe an absent or partial baseline.
|
|
1734
1956
|
// Keep publication and warnings inside the same barrier as the suite's final verdict.
|
|
1735
|
-
baselineCapture =
|
|
1957
|
+
baselineCapture = withCommandContext(undefined, async () => {
|
|
1736
1958
|
const captured = await captureBaseline(repoRoot, commands);
|
|
1737
1959
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(captured, null, 2));
|
|
1738
1960
|
baseline = captured;
|
|
@@ -1745,11 +1967,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1745
1967
|
// Attach a rejection handler immediately; gate and run-close awaits still propagate the error.
|
|
1746
1968
|
void baselineCapture.catch(() => { baselineFailed = true; });
|
|
1747
1969
|
writeFileSync(join(journal.dir, "graph.json"), readFileSync(join(tickmarkrDir(repoRoot), "graph.json")));
|
|
1748
|
-
// v1.70 T2: environment identity beside the graph/branch identity — running tickmarkr version,
|
|
1749
|
-
// loaded-config hash, and the probed CLI version of each adapter holding a channel in the run,
|
|
1750
|
-
// gathered through the existing probe/config-load paths (no second mechanism).
|
|
1751
|
-
const environment = runEnvironment(cfg, channels, health);
|
|
1752
1970
|
journal.append("run-start", undefined, {
|
|
1971
|
+
...sessionMetadata,
|
|
1753
1972
|
pid: process.pid, baseRef, commands, channels: channels.map(channelKey),
|
|
1754
1973
|
channelsByRole: {
|
|
1755
1974
|
worker: pools.worker.map(channelKey),
|
|
@@ -1759,9 +1978,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1759
1978
|
},
|
|
1760
1979
|
driver: driver.id,
|
|
1761
1980
|
driverEvidence: driverEvidence(cfg, driver, opts.driverOverride),
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
environment, ...(prior ? { supersedes: prior.runId } : {}),
|
|
1981
|
+
branch, graphDefinitionHash: graphDefinitionHash(graph),
|
|
1982
|
+
...(prior ? { supersedes: prior.runId } : {}),
|
|
1765
1983
|
}); // graphDefinitionHash: T3 engagement identity (status+resume share it); pid: v1.13 (VIS-11) liveness; mode/modeSource: v1.51 T2; supersedes: v1.53 T5
|
|
1766
1984
|
runStarted = true;
|
|
1767
1985
|
// v1.53 T5: mark the prior run AFTER this run's run-start exists, so the prior journal never
|
|
@@ -1848,6 +2066,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1848
2066
|
let mergeChain = Promise.resolve();
|
|
1849
2067
|
const mergeSerial = (taskBranch, t, gated) => {
|
|
1850
2068
|
const next = mergeChain.then(() => {
|
|
2069
|
+
executionSignal()?.throwIfAborted();
|
|
1851
2070
|
journal.phaseStart(t.id, "merge");
|
|
1852
2071
|
return mergeTask(intWt, taskBranch, `tickmarkr: merge ${t.id} ${t.title}`, gated);
|
|
1853
2072
|
});
|
|
@@ -1864,10 +2083,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1864
2083
|
];
|
|
1865
2084
|
return `${identity} — release with ${commands.map((command) => `\`${command}\``).join(" or ")}`;
|
|
1866
2085
|
};
|
|
1867
|
-
|
|
2086
|
+
// `present`: OBS-1007 add.4/5 — a gate DIAGNOSTIC may name only paths that exist in the tree or the
|
|
2087
|
+
// diff (`f.filepath`, `scripted.sh`, `gates.diffCap` lifted from review prose are not paths). A
|
|
2088
|
+
// worker's own scope refusal keeps the lexical rule: the file it needs may not exist yet.
|
|
2089
|
+
const namedPaths = (text, present) => [...text.matchAll(/(?:^|[\s`'"(])((?:[A-Za-z0-9_@.()[\]-]+\/)+[A-Za-z0-9_@.[\]-]+|[A-Za-z0-9_@-]+(?:\.[A-Za-z0-9_-]+)+)(?=$|[\s`'"),:;.!?])/g)]
|
|
1868
2090
|
.map((match) => match[1].replace(/^\.\//, "").replace(/\.$/, ""))
|
|
1869
2091
|
.filter((path) => !path.split("/").includes("..")
|
|
1870
|
-
&& (
|
|
2092
|
+
&& (present
|
|
2093
|
+
? present.has(path)
|
|
2094
|
+
: (/\.[A-Za-z][A-Za-z0-9_-]*$/.test(basename(path)) || existsSync(join(repoRoot, path)))));
|
|
2095
|
+
const diffPaths = async (base, wt) => {
|
|
2096
|
+
const out = await shGit(`git diff --name-only -z ${shq(base)}..HEAD`, wt);
|
|
2097
|
+
return new Set(out.code === 0 ? out.stdout.split("\0").filter(Boolean) : []);
|
|
2098
|
+
};
|
|
2099
|
+
/** The paths a gate diagnostic may name: the gated worktree's tracked tree plus its diff — never the main checkout. */
|
|
2100
|
+
const treeOrDiffPaths = async (base, wt) => {
|
|
2101
|
+
const tree = await shGit("git ls-files -z", wt);
|
|
2102
|
+
return new Set([...(tree.code === 0 ? tree.stdout.split("\0").filter(Boolean) : []), ...await diffPaths(base, wt)]);
|
|
2103
|
+
};
|
|
1871
2104
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
1872
2105
|
// OBS-979: a worker refusal can identify the missing authoring scope even when gates
|
|
1873
2106
|
// subsequently supply the park's disposition. Keep that actionable path on the park itself.
|
|
@@ -1904,7 +2137,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1904
2137
|
// count. The two callers arrive at it differently (the attempt loop's index already excludes the
|
|
1905
2138
|
// dispatch in flight; a gate replay's rs.attempts already includes it), and the ordinal journaled
|
|
1906
2139
|
// below is derived from it here so both paths number the same dispatch identically.
|
|
1907
|
-
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode) => {
|
|
2140
|
+
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode, changed = new Set()) => {
|
|
1908
2141
|
const scopeRed = results.find((g) => g.gate === "scope" && gateFailed(g));
|
|
1909
2142
|
const verdict = scopeRed?.meta?.collateral;
|
|
1910
2143
|
if (!verdict && t.files.length > 0) {
|
|
@@ -1917,17 +2150,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1917
2150
|
// A scope verdict already attributes actual out-of-scope EDITS using the collateral map.
|
|
1918
2151
|
// Path-only diagnostics from the other gates name code that needs fixing, not an edit the
|
|
1919
2152
|
// worker made. Do not replace the scope gate's stronger attribution with that inference.
|
|
1920
|
-
const paths = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details));
|
|
2153
|
+
const paths = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details, changed));
|
|
1921
2154
|
if (refusal.length || (paths.length > 0 && paths.every((path) => !allowed(path)))) {
|
|
1922
|
-
|
|
2155
|
+
const classification = {
|
|
1923
2156
|
gate: refusal.length ? "worker" : reds[0]?.gate,
|
|
1924
2157
|
paths: [...new Set(refusal.length ? refusal : paths)],
|
|
1925
2158
|
repair: `files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}`,
|
|
1926
2159
|
attempt: attempts + 1, chargeable: false, source: "diagnostic",
|
|
1927
|
-
}
|
|
2160
|
+
};
|
|
2161
|
+
if (refusal.length)
|
|
2162
|
+
journal.append("scope-request", t.id, classification);
|
|
2163
|
+
else
|
|
2164
|
+
journal.append("scope-authoring", t.id, classification);
|
|
1928
2165
|
// OBS-547 (Leg-2 T3 material): unchargeable ⇒ metered 0, exactly as the predicted path below —
|
|
1929
2166
|
// passing the physical count would write `meteredAttempts: 1` beside `attempts: 0`.
|
|
1930
|
-
|
|
2167
|
+
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${[...new Set(refusal)].map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2168
|
+
await park(t, `${refusal.length ? "scope request" : "authoring defect"} — files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}; current files[]: ${t.files.join(", ")}`, refusal.length ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, refusal.length ? { paths: [...new Set(refusal)], approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
1931
2169
|
return true;
|
|
1932
2170
|
}
|
|
1933
2171
|
}
|
|
@@ -1967,6 +2205,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1967
2205
|
approvalSweepCursor = events.length;
|
|
1968
2206
|
if (approvals.length === 0)
|
|
1969
2207
|
return;
|
|
2208
|
+
graph = applyScopeAmendments(graph, journal);
|
|
1970
2209
|
resume = journal.replayResumeState();
|
|
1971
2210
|
satisfiedGates = journal.replaySatisfiedGates();
|
|
1972
2211
|
replayedGateResults = resumeLifecycleOpen
|
|
@@ -1994,6 +2233,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1994
2233
|
};
|
|
1995
2234
|
const execTask = async (t) => {
|
|
1996
2235
|
const startMs = Date.now();
|
|
2236
|
+
let stallSeat;
|
|
2237
|
+
let stallReaps = 0;
|
|
2238
|
+
let reapedWorktreeRef;
|
|
1997
2239
|
const taskTimeoutMinutes = t.timeoutMinutes ?? cfg.taskTimeoutMinutes;
|
|
1998
2240
|
if (t.humanGate && !approved.has(t.id)) {
|
|
1999
2241
|
// GATE-08: the condition is the APPROVAL, never the code path. `!opts.resume` (or any run-phase
|
|
@@ -2070,10 +2312,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2070
2312
|
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
2071
2313
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
2072
2314
|
}
|
|
2073
|
-
else if (!hintsChanged && rs && rs.tried.length) {
|
|
2315
|
+
else if (!hintsChanged && rs && rs.tried.length && !t.routingHints?.pin) {
|
|
2074
2316
|
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
2075
2317
|
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
2076
2318
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
2319
|
+
// OBS-1034: a PIN is exempt — route() already returned it, and a review-upheld repair of the diff
|
|
2320
|
+
// the pin produced belongs on the pin, not on the next seat its own tried[] entry would pick.
|
|
2077
2321
|
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
|
|
2078
2322
|
if (next)
|
|
2079
2323
|
assignment = next;
|
|
@@ -2099,7 +2343,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2099
2343
|
// stays unblocked); inert to replayStatuses (unknown events ignored, pinned at journal.test.ts:70-80).
|
|
2100
2344
|
if (rs)
|
|
2101
2345
|
journal.append("resume-restore", t.id, { attempts: rs.attempts, tried: [...tried], assignment });
|
|
2102
|
-
|
|
2346
|
+
// Keep one live list: recovery retries must see exclusions added by onGate during the round.
|
|
2347
|
+
const badReviewers = [...replayedReviewerExclusions];
|
|
2103
2348
|
const noteReviewEvent = (e) => {
|
|
2104
2349
|
journal.append(e.name, t.id, e.payload);
|
|
2105
2350
|
if (e.name === "review-no-verdict" && typeof e.payload.reviewer === "string") {
|
|
@@ -2109,7 +2354,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2109
2354
|
};
|
|
2110
2355
|
// Keep the measured siblings, including a red judge, while replacing only a missing review.
|
|
2111
2356
|
// Every dispatched seat is excluded for this task, so the loop is bounded by the eligible pool.
|
|
2112
|
-
const runReviewRecovery = async (task, ctx) => {
|
|
2357
|
+
const runReviewRecovery = async (task, ctx, allowSelection = true) => {
|
|
2358
|
+
ctx.authorizeInfraRetry = cfg.executionPolicy?.boundedInfrastructure
|
|
2359
|
+
? (subject, cause) => reserveInfrastructureRetry(journal.read(), task.id, subject, journal.append.bind(journal), cause) : undefined;
|
|
2360
|
+
if (cfg.executionPolicy?.repairSelection && allowSelection) {
|
|
2361
|
+
const decision = repairSelectionDecision(journal.read(), task.id, true);
|
|
2362
|
+
ctx.selectTests = decision.selectTests;
|
|
2363
|
+
ctx.requiredRepairTests = decision.requiredFiles;
|
|
2364
|
+
ctx.selectionReason = decision.reason;
|
|
2365
|
+
}
|
|
2113
2366
|
const round = await runGates(task, ctx);
|
|
2114
2367
|
let review = round.results.find((g) => g.gate === "review");
|
|
2115
2368
|
while (review?.meta?.noVerdict === true) {
|
|
@@ -2182,6 +2435,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2182
2435
|
const unverdicted = (g.meta?.skipped === true && !g.pass) || noVerdictReview;
|
|
2183
2436
|
journal.append("gate-result", t.id, {
|
|
2184
2437
|
gate: g.gate, ...(unverdicted ? {} : { pass: g.pass }), details: g.details,
|
|
2438
|
+
// R41: the protocol + effective lifecycle policy the gate MEASURED under, stamped by run-gates
|
|
2439
|
+
// for the worktree it ran in; absent only for a row that never went through the battery.
|
|
2440
|
+
...(g.meta?.verification !== undefined ? { verification: g.meta.verification } : {}),
|
|
2185
2441
|
...(gateSubject ? { commit: gateSubject.commit, attempt: gateSubject.attempt } : {}),
|
|
2186
2442
|
...(gateSubject?.replayMeasurement ? { replayMeasurement: true } : {}),
|
|
2187
2443
|
...(gateSubject?.replayedFromAttempt !== undefined ? { replayedFromAttempt: gateSubject.replayedFromAttempt } : {}),
|
|
@@ -2190,6 +2446,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2190
2446
|
// be able to tell "the suite found a defect" from "the runner never ran", and the merge
|
|
2191
2447
|
// predicate's reason for refusing has to be legible in the ledger it refused from.
|
|
2192
2448
|
...(g.meta?.infra === true ? { infra: true } : {}),
|
|
2449
|
+
// OBS-1030: a dirty-tree refusal names its litter, its culprit command and the recovery ref ON
|
|
2450
|
+
// THE ROW — the ledger is where an operator looks for the ref before a recheck, not the prose.
|
|
2451
|
+
...(g.meta?.dirtyWorktree === true ? {
|
|
2452
|
+
dirtyWorktree: true, dirtyPaths: g.meta.paths,
|
|
2453
|
+
...(typeof g.meta.culprit === "string" ? { culprit: g.meta.culprit } : {}),
|
|
2454
|
+
...(typeof g.meta.preservedRef === "string" ? { preservedRef: g.meta.preservedRef } : {}),
|
|
2455
|
+
} : {}),
|
|
2456
|
+
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
2457
|
+
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision"]
|
|
2458
|
+
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
2193
2459
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
2194
2460
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
2195
2461
|
// oracle's recorded verdict when any journal-backed reader reconstructs it.
|
|
@@ -2359,7 +2625,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2359
2625
|
// FROM THE JOURNAL, not from process memory: a park, a `resume`, or a daemon restart must not
|
|
2360
2626
|
// hand the selector a clean slate it did not earn, and the journal is the only state that
|
|
2361
2627
|
// survives all three (same read as upheldFeedbackByTask above).
|
|
2362
|
-
let testGateFailed = journal.read().some((e) => e.event === "gate-result" && e.taskId === t.id && e.data.gate === "test" && e.data.pass === false
|
|
2628
|
+
let testGateFailed = journal.read().some((e) => e.event === "gate-result" && e.taskId === t.id && e.data.gate === "test" && e.data.pass === false
|
|
2629
|
+
&& !(cfg.executionPolicy?.boundedInfrastructure && e.data.disposition === "infrastructure"));
|
|
2363
2630
|
let retryMode = "fresh";
|
|
2364
2631
|
let lastContextTokens; // v1.23 reset signal, including stalled/quota attempts
|
|
2365
2632
|
// v1.29: only a gate-failed attempt can seed same-session retry. The next attempt consumes this
|
|
@@ -2469,7 +2736,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2469
2736
|
const taskBase = await integrationHead(intWt);
|
|
2470
2737
|
const taskBranch = `${branch}--${t.id}`;
|
|
2471
2738
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
2472
|
-
|
|
2739
|
+
// OBS-1022: the ref `approve --recheck` validated (commits ahead of the base) is the tree the
|
|
2740
|
+
// battery gates — read from the pending approval row, never re-selected here, so the tree
|
|
2741
|
+
// approve counted and the tree the daemon gates are one and the same, checkout present or not.
|
|
2742
|
+
const preservedRef = recheck
|
|
2743
|
+
? [...journal.read()].reverse().find((e) => e.taskId === t.id && e.event === "task-approved" && e.data.release === RECHECK_RELEASE)?.data.recheckedRef
|
|
2744
|
+
: undefined;
|
|
2745
|
+
if (!existsSync(priorWt) && !preservedRef) {
|
|
2473
2746
|
if (satisfiedGate || recheck)
|
|
2474
2747
|
throw new Error(`${recheck ? "recheck" : `approved gate ${satisfiedGate}`} cannot resume: task worktree is missing`);
|
|
2475
2748
|
// Observed passes are an optimization, never authority: without the task worktree there is no
|
|
@@ -2482,9 +2755,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2482
2755
|
: satisfiedGate
|
|
2483
2756
|
? `approved gate ${satisfiedGate}`
|
|
2484
2757
|
: `recorded gates on ${replayedGates.commit.slice(0, 10)}`;
|
|
2485
|
-
const priorTaskTip = await gitHead(priorWt);
|
|
2486
|
-
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, priorWt);
|
|
2487
|
-
const commitsToCarry = await commitsAheadOf(taskBase, priorWt);
|
|
2758
|
+
const priorTaskTip = preservedRef ? (await shGit(`git rev-parse ${shq(preservedRef)}`, repoRoot)).stdout.trim() : await gitHead(priorWt);
|
|
2759
|
+
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, preservedRef ? repoRoot : priorWt);
|
|
2760
|
+
const commitsToCarry = preservedRef ? await commitsAheadOfRef(taskBase, priorTaskTip, repoRoot) : await commitsAheadOf(taskBase, priorWt);
|
|
2488
2761
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
2489
2762
|
const carriedCommits = await cherryPickCommits(wt, commitsToCarry);
|
|
2490
2763
|
// Reuse is about the tree the gates will actually inspect. The integration tip may have moved
|
|
@@ -2523,16 +2796,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2523
2796
|
}
|
|
2524
2797
|
const workerEvent = [...journal.read()].reverse()
|
|
2525
2798
|
.find((e) => e.event === "worker-result" && e.taskId === t.id);
|
|
2526
|
-
|
|
2799
|
+
// OBS-1022: a task that FAILED before its harvest has no worker-result; a recheck judges the
|
|
2800
|
+
// committed tree on its own evidence, so the missing claim is recorded as exactly that.
|
|
2801
|
+
if (!workerEvent && !recheck)
|
|
2527
2802
|
throw new Error(`${resumeReason} cannot resume: worker result is missing`);
|
|
2528
|
-
const priorResult = {
|
|
2803
|
+
const priorResult = workerEvent ? {
|
|
2529
2804
|
ok: workerEvent.data.ok === true,
|
|
2530
2805
|
summary: typeof workerEvent.data.summary === "string" ? workerEvent.data.summary : "",
|
|
2531
2806
|
deviations: Array.isArray(workerEvent.data.deviations)
|
|
2532
2807
|
? workerEvent.data.deviations.filter((d) => typeof d === "string")
|
|
2533
2808
|
: [],
|
|
2534
2809
|
raw: "",
|
|
2535
|
-
};
|
|
2810
|
+
} : { ok: false, summary: "no worker result recorded (task failed before harvest); recheck gates the committed tree", deviations: [], raw: "" };
|
|
2536
2811
|
const parkedAuthor = recheck
|
|
2537
2812
|
? [...journal.read()].reverse().find((e) => e.event === "task-dispatch" && e.taskId === t.id)?.data.assignment
|
|
2538
2813
|
: undefined;
|
|
@@ -2598,7 +2873,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2598
2873
|
commit: replayedGates.commit, recorded: replayedCapacities, resolved: sessionCapacity,
|
|
2599
2874
|
});
|
|
2600
2875
|
}
|
|
2601
|
-
|
|
2876
|
+
// R41: the same rule for the verification protocol, measured for the RECREATED worktree
|
|
2877
|
+
// the re-run would gate in (its project npmrc included) — but a row with NO stamp is an OLDER
|
|
2878
|
+
// row from a different (pre-stamp) implementation, never "unchanged": it is re-run.
|
|
2879
|
+
const replayVerification = verificationProtocol(process.env, wt);
|
|
2880
|
+
const replayedVerifications = priorEvents
|
|
2881
|
+
.filter((e) => e.event === "gate-result" && e.taskId === t.id && e.data.commit === replayedGates.commit)
|
|
2882
|
+
.map((e) => e.data.verification);
|
|
2883
|
+
const sameProtocol = replayedVerifications.every((recorded) => sameVerification(recorded, replayVerification));
|
|
2884
|
+
if (!sameProtocol) {
|
|
2885
|
+
journal.append("gate-replay-verification-changed", t.id, {
|
|
2886
|
+
commit: replayedGates.commit, recorded: replayedVerifications, resolved: replayVerification,
|
|
2887
|
+
});
|
|
2888
|
+
}
|
|
2889
|
+
let reusable = sameWorld && sameProtocol && (exactCurrentCommit || canonicalCurrentCommit || recreatedLegacyCommit);
|
|
2602
2890
|
const reused = [];
|
|
2603
2891
|
const declaredGates = GATE_NAMES.filter((gate) => t.gates.includes(gate));
|
|
2604
2892
|
for (const gate of declaredGates) {
|
|
@@ -2614,6 +2902,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2614
2902
|
}
|
|
2615
2903
|
const resumedTask = { ...t, gates: remainingGates };
|
|
2616
2904
|
gateLoop: while (true) {
|
|
2905
|
+
executionSignal()?.throwIfAborted();
|
|
2617
2906
|
const gated = await gitHead(wt);
|
|
2618
2907
|
gateSubject = {
|
|
2619
2908
|
commit: await gateCommitSubject(taskBase, gated, wt),
|
|
@@ -2626,13 +2915,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2626
2915
|
await trackedDriver.project?.(t.id, "in-review");
|
|
2627
2916
|
await waitForBaseline(t.id);
|
|
2628
2917
|
journal.phaseStart(t.id, "gates");
|
|
2629
|
-
const { results } = await
|
|
2918
|
+
const { results } = await withCommandContext(t.id, () => runReviewRecovery(resumedTask, {
|
|
2630
2919
|
carriedFindings: outstandingReviewFindings(journal.read(), t.id),
|
|
2631
2920
|
worktree: wt, baseRef: taskBase, result: priorResult, author: gateAuthor,
|
|
2632
2921
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
2633
2922
|
collateral: collateral.get(t.id) ?? [],
|
|
2634
2923
|
// a recheck re-verifies a human's release: it never selects tests down, it runs the suite.
|
|
2635
|
-
pipeline: "v185",
|
|
2636
2924
|
via: cfg.visibility.llm === "pane"
|
|
2637
2925
|
? {
|
|
2638
2926
|
driver: trackedDriver,
|
|
@@ -2644,6 +2932,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2644
2932
|
: undefined,
|
|
2645
2933
|
excludeReviewers: badReviewers,
|
|
2646
2934
|
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
2935
|
+
// Leg-2 (OBS-1052): the run-scoped two-strike tally — without it retirement is inert and a
|
|
2936
|
+
// flaking seat is re-asked on every task. (carriedAuthors is held back: see SURGEON-LEG2-REPORT.)
|
|
2937
|
+
reviewNoVerdicts,
|
|
2647
2938
|
onGate: async (e) => {
|
|
2648
2939
|
if (e.phase === "start") {
|
|
2649
2940
|
notePhaseStart(e);
|
|
@@ -2665,7 +2956,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2665
2956
|
}
|
|
2666
2957
|
});
|
|
2667
2958
|
},
|
|
2668
|
-
}));
|
|
2959
|
+
}, false));
|
|
2669
2960
|
results.forEach(classifySignalOnlyTest);
|
|
2670
2961
|
if (pendingRechecks(journal.read()).has(t.id)) {
|
|
2671
2962
|
journal.append("recheck-battery", t.id, {
|
|
@@ -2683,6 +2974,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2683
2974
|
await park(t, `${infra.gate}: ${infra.details}`, "infra", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2684
2975
|
return;
|
|
2685
2976
|
}
|
|
2977
|
+
const capTrip = results.find(isDiffCapPark);
|
|
2978
|
+
if (capTrip) {
|
|
2979
|
+
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2980
|
+
return;
|
|
2981
|
+
}
|
|
2686
2982
|
// OBS-547: disposition FIRST, and through the same helper the ordinary attempt path uses. A
|
|
2687
2983
|
// crash between a journaled scope red and its classification must not turn a predicted red
|
|
2688
2984
|
// into a charged one just because a resume is what observed it.
|
|
@@ -2695,7 +2991,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2695
2991
|
// restored. Ask the journal which state this is, and take the classified dispatch's own
|
|
2696
2992
|
// assignment back with it.
|
|
2697
2993
|
const classified = journal.classifiedDispatch(t.id);
|
|
2698
|
-
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode))
|
|
2994
|
+
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
2699
2995
|
return;
|
|
2700
2996
|
gateFails++;
|
|
2701
2997
|
// Observed green gates are only measurements. If the resumed suffix is red, preserve that
|
|
@@ -2746,6 +3042,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2746
3042
|
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
2747
3043
|
let supersededWorkerSlot;
|
|
2748
3044
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
3045
|
+
executionSignal()?.throwIfAborted();
|
|
2749
3046
|
if (attempt !== providerDeathAttempt) {
|
|
2750
3047
|
providerDeathRequeues = 0;
|
|
2751
3048
|
providerDeathAttempt = attempt;
|
|
@@ -2892,7 +3189,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2892
3189
|
feedback = feedback ? `${feedback}\n\n${consultBrief}` : consultBrief;
|
|
2893
3190
|
}
|
|
2894
3191
|
}
|
|
2895
|
-
|
|
3192
|
+
const scopeApproval = [...journaledSoFar].reverse().find((e) => e.taskId === t.id && (e.event === "task-approved" || e.event === "task-dispatch"));
|
|
3193
|
+
retryMode = scopeApproval?.data.release === "scope-request" ? "fresh" : repairFindings
|
|
2896
3194
|
? "repair"
|
|
2897
3195
|
: priorSession
|
|
2898
3196
|
&& priorSession.channel === channelKey(assignment)
|
|
@@ -2919,6 +3217,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2919
3217
|
// ledger cannot tell a carried dispatch from an amnesiac one — the exact question a run that
|
|
2920
3218
|
// spends two frontier attempts re-deriving a known defect has to be able to answer afterwards.
|
|
2921
3219
|
journal.append("task-dispatch", t.id, {
|
|
3220
|
+
...(scopeApproval?.data.release === "scope-request" ? { files: t.files, graphDefinitionHash: graphDefinitionHash(graph) } : {}),
|
|
2922
3221
|
assignment, attempt, provenance: dispatchProvenance([
|
|
2923
3222
|
channelKey(assignment) === channelKey(r.assignment) ? r.provenance
|
|
2924
3223
|
: t.routingHints?.pin ? `pin ${t.routingHints.pin.via}:${t.routingHints.pin.model} not re-tried` : "ladder assignment",
|
|
@@ -2938,7 +3237,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2938
3237
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
2939
3238
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
2940
3239
|
const recreating = existsSync(priorWt);
|
|
3240
|
+
// Carry the interrupted worker's snapshot, never incidental files created by later gates.
|
|
3241
|
+
const preserved = recreating ? reapedWorktreeRef : undefined;
|
|
3242
|
+
reapedWorktreeRef = undefined;
|
|
2941
3243
|
const commitsToCarry = recreating ? await commitsAheadOf(taskBase, priorWt) : [];
|
|
3244
|
+
if (preserved)
|
|
3245
|
+
commitsToCarry.push(preserved);
|
|
2942
3246
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
2943
3247
|
// OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
|
|
2944
3248
|
// cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
|
|
@@ -3007,14 +3311,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3007
3311
|
if (repairFindings) {
|
|
3008
3312
|
const raw = await shGit(`git diff ${shq(taskBase)}..HEAD`, wt);
|
|
3009
3313
|
const cap = cfg.gates.diffCap ?? DEFAULT_DIFF_CAP; // same fallback the measuring gates use
|
|
3010
|
-
|
|
3011
|
-
|
|
3314
|
+
// OBS-1034: a capped brief says WHAT it cut — the files whose hunks start at or past the cap
|
|
3315
|
+
// (the file the cut lands inside is partial too) — on the row and in the worker's prompt.
|
|
3316
|
+
const rawBytes = Buffer.from(raw.stdout, "utf8"); // the cap is BYTES: measure and cut in UTF-8, not code units
|
|
3317
|
+
const capped = rawBytes.length > cap;
|
|
3318
|
+
const droppedFiles = capped ? repairDiffFilesCut(rawBytes, cap, [...await diffPaths(taskBase, wt)]) : [];
|
|
3319
|
+
const diff = capped
|
|
3320
|
+
? `${new TextDecoder().decode(rawBytes.subarray(0, cap))}\n… diff truncated at gates.diffCap (${cap} bytes)`
|
|
3321
|
+
+ `\nPARTIAL DIFF: the diff above is incomplete; files cut by the cap: ${droppedFiles.join(", ") || "(unknown)"}`
|
|
3012
3322
|
: raw.stdout;
|
|
3013
3323
|
const brief = repairBrief(repairFindings, diff, taskBase);
|
|
3014
3324
|
// anything the live brief holds beyond the journaled findings (a consult's guidance) is kept:
|
|
3015
3325
|
// a repair adds the diff and the fix-only contract, it never subtracts what was already known.
|
|
3016
3326
|
feedback = feedback && !brief.includes(feedback) ? `${brief}\n\n${feedback}` : brief;
|
|
3017
|
-
journal.append("repair-dispatch", t.id, { diffBytes: diff
|
|
3327
|
+
journal.append("repair-dispatch", t.id, { diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
3018
3328
|
}
|
|
3019
3329
|
if (feedback || priorNamed.length > 0) {
|
|
3020
3330
|
feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
|
|
@@ -3076,7 +3386,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3076
3386
|
? (hasSeed ? ":" : icmd)
|
|
3077
3387
|
: adapter.invoke(t, wt, assignment, { promptFile }).command;
|
|
3078
3388
|
const dispatchScript = promptFile.replace(/\.md$/, ".sh");
|
|
3389
|
+
// Approval can reset the attempt counter while the old pane remains retained.
|
|
3390
|
+
// Its ownership claim must survive a new engagement reusing the script path.
|
|
3391
|
+
const groupFile = `${dispatchScript}.${nonce}.pgid`;
|
|
3392
|
+
workerOwners.set(slot, { taskId: t.id, attempt, groupFile });
|
|
3079
3393
|
writeFileSync(dispatchScript, [
|
|
3394
|
+
// A driver may launch inside the daemon's group. That group is never worker-owned.
|
|
3395
|
+
`worker_pgid=$(ps -o pgid= -p $$ 2>/dev/null); daemon_pgid=$(ps -o pgid= -p ${process.pid} 2>/dev/null)`,
|
|
3396
|
+
`if [ -n "$worker_pgid" ] && [ -n "$daemon_pgid" ] && [ "$worker_pgid" != "$daemon_pgid" ]; then printf '%s\\n' "$worker_pgid" > ${shq(groupFile)}; fi`,
|
|
3080
3397
|
"export BASH_SILENCE_DEPRECATION_WARNING=1",
|
|
3081
3398
|
bannerShell(),
|
|
3082
3399
|
workerCmd,
|
|
@@ -3143,7 +3460,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3143
3460
|
}
|
|
3144
3461
|
if (hasSeed || cpuAccountant !== undefined)
|
|
3145
3462
|
return;
|
|
3146
|
-
cpuAccountant = new WorkerTreeCpuAccountant(dispatchScript, wt);
|
|
3463
|
+
cpuAccountant = new WorkerTreeCpuAccountant(dispatchScript, wt, () => readOwnedProcessGroup(groupFile));
|
|
3147
3464
|
await cpuAccountant.start();
|
|
3148
3465
|
};
|
|
3149
3466
|
const readCpuLeg = () => {
|
|
@@ -3237,18 +3554,36 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3237
3554
|
// window that could not have killed anything and briefed the next worker to fix a stall that
|
|
3238
3555
|
// never happened. Fourth conflation in the taxonomy OBS-53 opened.
|
|
3239
3556
|
let deadChannelKilled = false;
|
|
3557
|
+
let hardTimedOut = false;
|
|
3558
|
+
let quotaBannerKilled = false;
|
|
3559
|
+
let driverProbeFailed = false;
|
|
3560
|
+
let heldLegs = [];
|
|
3561
|
+
const noteDriverUnreadable = (error) => {
|
|
3562
|
+
driverProbeFailed = true;
|
|
3563
|
+
heldLegs = ["quota:driver-unreadable", "stall:driver-unreadable"];
|
|
3564
|
+
journal.append("contact-unreadable", t.id, { slot: slot.name, attempt, source: "driver", concludes: false,
|
|
3565
|
+
error: error instanceof Error ? error.message : String(error) });
|
|
3566
|
+
};
|
|
3240
3567
|
// T2 review: print mode's "the exit marker appeared". Kept apart from `finished` (the
|
|
3241
3568
|
// trailer) but still needed by the keepPanes decision below, whose contract is about a
|
|
3242
3569
|
// subprocess tree that REACHED its exit marker, not about what the worker claimed.
|
|
3243
3570
|
let processExited = false;
|
|
3244
3571
|
let earlyLaunchDead = false;
|
|
3245
3572
|
let startupFailure = false;
|
|
3573
|
+
let startupEvidence;
|
|
3574
|
+
const startupDetector = new StartupFailureDetector(adapter);
|
|
3575
|
+
const startupFailureInWindow = (text, launchedAt) => {
|
|
3576
|
+
startupEvidence ??= startupDetector.sample(text, launchedAt);
|
|
3577
|
+
return startupEvidence !== undefined;
|
|
3578
|
+
};
|
|
3246
3579
|
let deadWorkerPark;
|
|
3247
3580
|
let settleParsed;
|
|
3248
3581
|
const trailerFrames = interactive && adapter.busyFrameMarkers
|
|
3249
3582
|
? new SettledTrailerTracker(adapter.busyFrameMarkers) : undefined;
|
|
3250
3583
|
const sampleTrailer = (frame) => {
|
|
3251
|
-
const
|
|
3584
|
+
const parsed = adapter.parse(frame, nonce);
|
|
3585
|
+
const trailer = new RegExp(trailerPattern(nonce)).test(frame)
|
|
3586
|
+
&& parsed.summary !== NO_TRAILER_SUMMARY && parsed.summary !== UNPARSEABLE_TRAILER_SUMMARY;
|
|
3252
3587
|
return trailerFrames ? trailerFrames.sample(frame, trailer) : trailer;
|
|
3253
3588
|
};
|
|
3254
3589
|
let seedResult;
|
|
@@ -3262,11 +3597,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3262
3597
|
// DeliveryReadinessError, and that catch continues or returns with no result to read, so a
|
|
3263
3598
|
// latch built from one would start false with the key already sent, and the next modal on
|
|
3264
3599
|
// this slot would get the free Enter this contract exists to prevent.
|
|
3265
|
-
let trustAnswered =
|
|
3600
|
+
let trustAnswered = trustAnsweredSlots.has(slot.id);
|
|
3266
3601
|
const noteSeedTrustAnswered = () => {
|
|
3267
3602
|
if (trustAnswered)
|
|
3268
3603
|
return;
|
|
3269
3604
|
trustAnswered = true;
|
|
3605
|
+
trustAnsweredSlots.add(slot.id);
|
|
3270
3606
|
// Same audit line the loop below writes, so a live run shows the answer wherever it happened.
|
|
3271
3607
|
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id, phase: "seed" });
|
|
3272
3608
|
};
|
|
@@ -3320,7 +3656,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3320
3656
|
// failure rather than allowed to race the trailer wait.
|
|
3321
3657
|
try {
|
|
3322
3658
|
seedResult = await runInteractiveSeed({
|
|
3323
|
-
driver
|
|
3659
|
+
driver: trustAnswered ? {
|
|
3660
|
+
run: driver.run.bind(driver), waitOutput: driver.waitOutput.bind(driver), read: driver.read.bind(driver),
|
|
3661
|
+
} : driver,
|
|
3662
|
+
slot, adapter, assignment, promptFile, taskTimeoutMinutes,
|
|
3324
3663
|
onTrustAnswered: noteSeedTrustAnswered,
|
|
3325
3664
|
});
|
|
3326
3665
|
}
|
|
@@ -3367,6 +3706,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3367
3706
|
// pageRepeatMs elapses, so an operator who missed the first one is paged again.
|
|
3368
3707
|
let lastPagedStatus;
|
|
3369
3708
|
let lastPagedAt = 0;
|
|
3709
|
+
let pagesSuppressed = 0; // OBS-1006: pageable slices swallowed by the cadence since the last row
|
|
3370
3710
|
// T1 review: worker-status is journaled on CHANGE, not per slice — every journal append
|
|
3371
3711
|
// is narrated to the run's live surface (cli/commands/run.ts) and feeds activity.ts's
|
|
3372
3712
|
// `now:` cell, so a per-slice append wrote a status line per worker per ~30s slice and
|
|
@@ -3392,9 +3732,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3392
3732
|
let cpuHeld = false; // likewise for the CPU leg's stand-down (OBS-548)
|
|
3393
3733
|
let paneReadHeld = false;
|
|
3394
3734
|
let paneStatusHeld = false;
|
|
3395
|
-
|
|
3735
|
+
const hardDeadline = attemptStart + (attemptHardTimeoutMs ?? stallWindowMs * 4);
|
|
3736
|
+
await armCpuLeg(true);
|
|
3737
|
+
while (Date.now() < hardDeadline) {
|
|
3738
|
+
if (termSignal)
|
|
3739
|
+
throw new Error(`terminated by ${termSignal}`);
|
|
3740
|
+
executionSignal()?.throwIfAborted();
|
|
3741
|
+
driverProbeFailed = false;
|
|
3396
3742
|
const sliceStart = Date.now();
|
|
3397
|
-
const remaining = stallWindowMs - (sliceStart - lastProgressAt);
|
|
3743
|
+
const remaining = Math.max(100, Math.min(stallWindowMs - (sliceStart - lastProgressAt), hardDeadline - sliceStart));
|
|
3398
3744
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3399
3745
|
const startupLeft = workerLaunchedAt === undefined
|
|
3400
3746
|
? 0
|
|
@@ -3411,11 +3757,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3411
3757
|
// otherwise cost a minute per pair of CPU samples. At the shipped 5m gate this leaves
|
|
3412
3758
|
// every slice before the gate exactly as long as it already was.
|
|
3413
3759
|
slice = Math.min(slice, harvestSliceMs(sliceStart - lastProgressAt));
|
|
3414
|
-
|
|
3760
|
+
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
3761
|
+
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true }).catch((error) => { noteDriverUnreadable(error); return false; })) {
|
|
3415
3762
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
3416
3763
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
3417
3764
|
// parseable trailer or a digit-suffixed exit marker in the harvest is completion.
|
|
3418
|
-
output = await driver.read(slot, PANE_READ_ROWS); // TUI transcripts carry chrome — read deeper than print's 500
|
|
3765
|
+
output = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; }); // TUI transcripts carry chrome — read deeper than print's 500
|
|
3419
3766
|
finished = sampleTrailer(output);
|
|
3420
3767
|
const exit = exitRe.exec(output);
|
|
3421
3768
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3424,6 +3771,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3424
3771
|
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
3425
3772
|
break;
|
|
3426
3773
|
}
|
|
3774
|
+
if (adapter.parse(output, nonce).summary === UNPARSEABLE_TRAILER_SUMMARY)
|
|
3775
|
+
break;
|
|
3427
3776
|
if (trailerFrames && new RegExp(trailerPattern(nonce)).test(output)) {
|
|
3428
3777
|
// A matching busy frame is still a live turn. Preserve the rolling progress budget.
|
|
3429
3778
|
await sampleContext();
|
|
@@ -3440,19 +3789,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3440
3789
|
paneText = await driver.read(slot, PANE_READ_ROWS);
|
|
3441
3790
|
}
|
|
3442
3791
|
catch (error) {
|
|
3443
|
-
|
|
3444
|
-
// sees a pane. The outer attempt finally owns accountant cleanup on that path. Only
|
|
3445
|
-
// an undetectable status makes the read failure relevant to the death detector, and
|
|
3446
|
-
// that genuinely unmeasurable pair fails open to the rolling timeout.
|
|
3447
|
-
let paneUndetectable = true;
|
|
3448
|
-
try {
|
|
3449
|
-
paneUndetectable = await driver.status(slot) === "unknown";
|
|
3450
|
-
}
|
|
3451
|
-
catch {
|
|
3452
|
-
// Two unreadable pane probes are still unmeasurable, never proof of death.
|
|
3453
|
-
}
|
|
3454
|
-
if (!paneUndetectable)
|
|
3455
|
-
throw error;
|
|
3792
|
+
noteDriverUnreadable(error);
|
|
3456
3793
|
if (!paneReadHeld) {
|
|
3457
3794
|
paneReadHeld = true;
|
|
3458
3795
|
noteWorkerLiveness("worker-dead-held", {
|
|
@@ -3460,14 +3797,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3460
3797
|
error: error instanceof Error ? error.message : String(error),
|
|
3461
3798
|
});
|
|
3462
3799
|
}
|
|
3463
|
-
await armCpuLeg(
|
|
3800
|
+
await armCpuLeg(true);
|
|
3464
3801
|
const spent = Date.now() - sliceStart;
|
|
3465
3802
|
if (spent < slice)
|
|
3466
3803
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
3467
3804
|
continue;
|
|
3468
3805
|
}
|
|
3469
3806
|
// waitOutput is only a wake hint; a driver may miss a trailer already painted.
|
|
3470
|
-
if (
|
|
3807
|
+
if (sampleTrailer(paneText)) {
|
|
3471
3808
|
output = paneText;
|
|
3472
3809
|
finished = true;
|
|
3473
3810
|
const exit = exitRe.exec(output);
|
|
@@ -3476,6 +3813,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3476
3813
|
await sampleContext();
|
|
3477
3814
|
break;
|
|
3478
3815
|
}
|
|
3816
|
+
if (adapter.parse(paneText, nonce).summary === UNPARSEABLE_TRAILER_SUMMARY) {
|
|
3817
|
+
output = paneText;
|
|
3818
|
+
break;
|
|
3819
|
+
}
|
|
3820
|
+
if (driverProbeFailed) {
|
|
3821
|
+
const spent = Date.now() - sliceStart;
|
|
3822
|
+
if (spent < slice)
|
|
3823
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(slice - spent, 1_000)));
|
|
3824
|
+
continue;
|
|
3825
|
+
}
|
|
3479
3826
|
if (paneText.length > 0)
|
|
3480
3827
|
everHadOutput = true;
|
|
3481
3828
|
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
@@ -3534,14 +3881,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3534
3881
|
// filtered by identity, never by novelty — a banner already on screen at launch
|
|
3535
3882
|
// classifies exactly like one printed mid-attempt (T1 review: a novelty baseline
|
|
3536
3883
|
// exculpated the launch-throttle case forever).
|
|
3537
|
-
|
|
3884
|
+
const bannerMatch = QUOTA_RE.exec(stallSnapshotBannerRows(paneText));
|
|
3885
|
+
if (bannerMatch)
|
|
3538
3886
|
quotaStreak++;
|
|
3539
3887
|
else
|
|
3540
3888
|
quotaStreak = 0;
|
|
3541
|
-
if (
|
|
3889
|
+
if (bannerMatch && !stallProgress.rowSignalSaturated && !nudgeFailed
|
|
3890
|
+
&& !(driver.nudge && NUDGEABLE_ADAPTERS.has(adapter.id) && (!nudged || nudgeDeadline !== undefined))
|
|
3891
|
+
&& readCpuLeg().state === "flat"
|
|
3892
|
+
&& quotaStreak >= 2 && sliceNow - lastProgressAt >= quotaBannerSilentMs) {
|
|
3542
3893
|
// no `output =` here: the post-loop no-trailer tail re-reads the pane anyway, so an
|
|
3543
3894
|
// assignment would only split the classification read from the verdict read.
|
|
3544
|
-
|
|
3895
|
+
quotaBannerKilled = true;
|
|
3896
|
+
journal.append("quota-banner", t.id, { slot: slot.name, attempt, silentMs: sliceNow - lastProgressAt, matched: bannerMatch[0], excerpt: bannerMatch.input, regex: QUOTA_RE.source });
|
|
3545
3897
|
break;
|
|
3546
3898
|
}
|
|
3547
3899
|
// T1 (OBS-262): the `paged` latch is deleted — status is sampled EVERY slice (and
|
|
@@ -3554,6 +3906,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3554
3906
|
st = await driver.status(slot);
|
|
3555
3907
|
}
|
|
3556
3908
|
catch (error) {
|
|
3909
|
+
noteDriverUnreadable(error);
|
|
3557
3910
|
if (!paneStatusHeld) {
|
|
3558
3911
|
paneStatusHeld = true;
|
|
3559
3912
|
noteWorkerLiveness("worker-dead-held", {
|
|
@@ -3561,7 +3914,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3561
3914
|
error: error instanceof Error ? error.message : String(error),
|
|
3562
3915
|
});
|
|
3563
3916
|
}
|
|
3564
|
-
await armCpuLeg(
|
|
3917
|
+
await armCpuLeg(true);
|
|
3565
3918
|
const spent = Date.now() - sliceStart;
|
|
3566
3919
|
if (spent < slice)
|
|
3567
3920
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
@@ -3579,6 +3932,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3579
3932
|
const paneText = await driver.read(slot, 80);
|
|
3580
3933
|
if (matchesTrustDialog(paneText, adapter.trustDialog)) {
|
|
3581
3934
|
trustAnswered = true;
|
|
3935
|
+
trustAnsweredSlots.add(slot.id);
|
|
3582
3936
|
// v1.25 T1: audit trail for live runs — prove the dialog appeared and was answered.
|
|
3583
3937
|
// Latch + sendKey + no-page continue stay byte-identical; this append is additive only.
|
|
3584
3938
|
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id });
|
|
@@ -3704,20 +4058,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3704
4058
|
// lastProgressAt. Either one advancing is output growth.
|
|
3705
4059
|
const lastOutputGrowthAt = Math.max(stallProgress.lastRowGrowthAt ?? 0, lastProgressAt);
|
|
3706
4060
|
// Everything the fast-kill can decide from the CHANNEL, evaluated before the CPU leg is
|
|
3707
|
-
// asked for
|
|
3708
|
-
// slice must not pay for it. The CPU reading is consulted at the kill itself, below.
|
|
4061
|
+
// asked for a conclusion. The shared CPU reading is consulted at the kill below.
|
|
3709
4062
|
const fastKillEligible = !nudgePending && !nudgeFailed
|
|
3710
4063
|
&& sliceNow - lastOutputGrowthAt >= deadChannelFastKillMs
|
|
3711
4064
|
&& worktreeSinceLaunch === "unchanged";
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
// slice from BOTH windows. Arming per-reader let the harvest's own "not silent enough
|
|
3716
|
-
// yet" branch stop the sampler the fast-kill had just started, and a restarted
|
|
3717
|
-
// accountant re-bases its monotonic total — the leg could then never read flat. The
|
|
3718
|
-
// accountant also carries the readers' nudge holds: when neither reader may conclude,
|
|
3719
|
-
// there is no reason to fork its 10 Hz process snapshots through the remaining window.
|
|
3720
|
-
await armCpuLeg(harvestCpuEligible || fastKillEligible);
|
|
4065
|
+
// All silent conclusions share the attempt's continuous CPU ledger. A nudge
|
|
4066
|
+
// hold or a saturated pane must not reset its history before the stall boundary.
|
|
4067
|
+
await armCpuLeg(true);
|
|
3721
4068
|
// T2 (OBS-264): the liveness triad CONCLUDES the wait on finished work — commits ahead
|
|
3722
4069
|
// of the task base (this attempt's own AND any carried forward — see the eligibility
|
|
3723
4070
|
// comment at the dispatch site), a flat worker-tree CPU delta, and >= harvestSilentMs
|
|
@@ -3775,7 +4122,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3775
4122
|
// concluded here". A live CPU delta is a live worker, by construction; an UNMEASURABLE
|
|
3776
4123
|
// reading keeps the fail-open contract the probe already states (undefined is never
|
|
3777
4124
|
// evidence a worker stopped), so on a host whose `ps` cannot be read the fast-kill stands
|
|
3778
|
-
// down and the
|
|
4125
|
+
// down and the fixed attempt ceiling bounds the hold.
|
|
3779
4126
|
if (fastKillEligible) {
|
|
3780
4127
|
const leg = readCpuLeg();
|
|
3781
4128
|
if (leg.state === "flat") {
|
|
@@ -3794,6 +4141,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3794
4141
|
});
|
|
3795
4142
|
}
|
|
3796
4143
|
}
|
|
4144
|
+
const stallCpu = readCpuLeg();
|
|
4145
|
+
heldLegs = [
|
|
4146
|
+
...(stallProgress.rowSignalSaturated ? ["quota:row-signal-saturated", "stall:row-signal-saturated"] : []),
|
|
4147
|
+
...(stallCpu.state !== "flat" ? [`stall:cpu-${stallCpu.state}`] : []),
|
|
4148
|
+
...(nudgeFailed || nudgePending ? ["quota:nudge", "stall:nudge"] : []),
|
|
4149
|
+
...(worktreeSincePriorRead === "unreadable" ? ["stall:worktree-unreadable"] : []),
|
|
4150
|
+
];
|
|
4151
|
+
if (sliceNow - lastProgressAt >= stallWindowMs && heldLegs.length === 0) {
|
|
4152
|
+
timedOut = true;
|
|
4153
|
+
break;
|
|
4154
|
+
}
|
|
3797
4155
|
if (nudgeable && !nudged && sliceNow - lastProgressAt >= nudgeAfterSilentMs) {
|
|
3798
4156
|
nudged = true;
|
|
3799
4157
|
// T1 review: a false return is a driver-delivery outcome (missing pin, readiness
|
|
@@ -3801,16 +4159,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3801
4159
|
// one failure is a flake class, retried once in-slice after a short settle. Only a
|
|
3802
4160
|
// failed RETRY condemns the channel. Both failures happen inside this slice, so the
|
|
3803
4161
|
// latch stays immediate and exactly one failure is journaled per attempt.
|
|
3804
|
-
let delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE);
|
|
4162
|
+
let delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE).catch((error) => { noteDriverUnreadable(error); return false; });
|
|
3805
4163
|
if (!delivered) {
|
|
3806
4164
|
await new Promise((r) => setTimeout(r, NUDGE_REDELIVER_MS));
|
|
3807
|
-
delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE);
|
|
4165
|
+
delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE).catch((error) => { noteDriverUnreadable(error); return false; });
|
|
3808
4166
|
}
|
|
3809
4167
|
if (delivered) {
|
|
3810
4168
|
// absorb the nudge's own echo BEFORE arming the grace timer — post-nudge progress
|
|
3811
4169
|
// is measured against this baseline, not against the echo.
|
|
3812
|
-
const echo = await driver.read(slot, PANE_READ_ROWS);
|
|
3813
|
-
|
|
4170
|
+
const echo = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4171
|
+
if (echo !== undefined)
|
|
4172
|
+
stallProgress.observe({ paneText: echo, contextTokens });
|
|
3814
4173
|
nudgeDeadline = Date.now() + workerNudgeGraceMs;
|
|
3815
4174
|
journal.append("worker-nudge", t.id, { slot: slot.name, attempt });
|
|
3816
4175
|
}
|
|
@@ -3818,15 +4177,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3818
4177
|
// Delivery failure is journal evidence about the daemon's contact attempt only.
|
|
3819
4178
|
// The fast-kill's worktree clause remains wholly filesystem/git-derived above.
|
|
3820
4179
|
nudgeFailed = true;
|
|
3821
|
-
journal.append("worker-nudge-failed", t.id, { slot: slot.name, attempt });
|
|
4180
|
+
journal.append("worker-nudge-failed", t.id, { slot: slot.name, attempt, cause: "delivery-refused-or-unconfirmed" });
|
|
3822
4181
|
}
|
|
3823
4182
|
}
|
|
3824
|
-
if (nudged && nudgeDeadline !== undefined && sliceNow >= nudgeDeadline && sliceNow - lastProgressAt >= workerNudgeGraceMs) {
|
|
4183
|
+
if (!stallProgress.rowSignalSaturated && readCpuLeg().state === "flat" && nudged && nudgeDeadline !== undefined && sliceNow >= nudgeDeadline && sliceNow - lastProgressAt >= workerNudgeGraceMs) {
|
|
3825
4184
|
// grace spent, still no post-nudge progress: re-harvest once (the trailer may have
|
|
3826
4185
|
// landed between polls), then conclude the wait as a stall NOW — the consult sees the
|
|
3827
4186
|
// un-answered nudge instead of the remainder of the window.
|
|
4187
|
+
const finalPane = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4188
|
+
if (finalPane === undefined)
|
|
4189
|
+
continue;
|
|
3828
4190
|
nudgeDeadline = undefined;
|
|
3829
|
-
output =
|
|
4191
|
+
output = finalPane;
|
|
3830
4192
|
finished = sampleTrailer(output);
|
|
3831
4193
|
const exit = exitRe.exec(output);
|
|
3832
4194
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3836,8 +4198,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3836
4198
|
break;
|
|
3837
4199
|
}
|
|
3838
4200
|
journal.append("worker-nudge-expired", t.id, { slot: slot.name, attempt, graceMs: workerNudgeGraceMs });
|
|
3839
|
-
|
|
3840
|
-
|
|
4201
|
+
timedOut = true;
|
|
4202
|
+
break;
|
|
3841
4203
|
}
|
|
3842
4204
|
// Unlatched page (T1): the page DECISION fires and is journaled every slice the
|
|
3843
4205
|
// operator is the right actor — i.e. the nudge path doesn't own it (non-allowlisted
|
|
@@ -3850,23 +4212,32 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3850
4212
|
// once pageRepeatMs elapses, so a missed first page is not the last one.
|
|
3851
4213
|
const pageable = (st === "blocked" || st === "idle")
|
|
3852
4214
|
&& (!nudgePending || nudgeFailed);
|
|
4215
|
+
// OBS-1006: the row is appended INSIDE the delivery guard — one row per page delivered, never
|
|
4216
|
+
// one per slice — and it carries how many pageable slices were suppressed since the last one.
|
|
3853
4217
|
if (pageable) {
|
|
3854
|
-
journal.append("operator-page", t.id, { slot: slot.name, attempt, status: st });
|
|
3855
4218
|
if (st !== lastPagedStatus || sliceNow - lastPagedAt >= pageRepeatMs) {
|
|
4219
|
+
journal.append("operator-page", t.id, { slot: slot.name, attempt, status: st, suppressed: pagesSuppressed });
|
|
4220
|
+
pagesSuppressed = 0;
|
|
3856
4221
|
lastPagedStatus = st;
|
|
3857
4222
|
lastPagedAt = sliceNow;
|
|
3858
4223
|
const why = st === "blocked" ? "is blocked on a prompt — approve in its pane" : "looks idle without finishing — check its pane";
|
|
3859
4224
|
await driver.notify(`tickmarkr ${runId}: ${slot.name} ${why}`, { tier: "attention" });
|
|
3860
4225
|
}
|
|
4226
|
+
else {
|
|
4227
|
+
pagesSuppressed++;
|
|
4228
|
+
}
|
|
3861
4229
|
}
|
|
3862
4230
|
// a dead pane or a false-positive marker display returns fast — sleep the unspent slice, never hot-spin
|
|
3863
4231
|
const spent = Date.now() - sliceStart;
|
|
3864
4232
|
if (spent < slice)
|
|
3865
4233
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
3866
4234
|
}
|
|
3867
|
-
if (!finished && exitCode === null) {
|
|
4235
|
+
if (!finished && exitCode === null && adapter.parse(output, nonce).summary !== UNPARSEABLE_TRAILER_SUMMARY) {
|
|
3868
4236
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
3869
|
-
|
|
4237
|
+
hardTimedOut = Date.now() >= hardDeadline;
|
|
4238
|
+
timedOut ||= hardTimedOut;
|
|
4239
|
+
if (hardTimedOut)
|
|
4240
|
+
journal.append("worker-hard-timeout", t.id, { slot: slot.name, attempt, heldLegs });
|
|
3870
4241
|
try {
|
|
3871
4242
|
output = await driver.read(slot, PANE_READ_ROWS);
|
|
3872
4243
|
}
|
|
@@ -3874,14 +4245,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3874
4245
|
// The poll loop already recorded the unreadable pane. Retain the last readable bytes
|
|
3875
4246
|
// so this ambiguous path still reaches the ordinary timeout/consult backstop.
|
|
3876
4247
|
}
|
|
3877
|
-
finished =
|
|
4248
|
+
finished = sampleTrailer(output);
|
|
3878
4249
|
const exit = exitRe.exec(output);
|
|
3879
4250
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3880
4251
|
processExited = exit !== null;
|
|
3881
4252
|
}
|
|
3882
4253
|
if (finished && !trailerFrames) {
|
|
3883
|
-
await driver.waitAgentStatus(slot, "idle", 5_000);
|
|
3884
|
-
|
|
4254
|
+
await driver.waitAgentStatus(slot, "idle", 5_000).catch(noteDriverUnreadable);
|
|
4255
|
+
const settled = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; });
|
|
4256
|
+
if (sampleTrailer(settled))
|
|
4257
|
+
output = settled;
|
|
3885
4258
|
}
|
|
3886
4259
|
// T5 / OBS-111: an interactive harvest can race the TUI's final paint. When the pane
|
|
3887
4260
|
// contains the nonce token but the JSON hasn't balanced yet, settle and re-read through
|
|
@@ -3939,6 +4312,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3939
4312
|
// there is the trailer regex; the exit marker only sets exitCode), and the cause taxonomy
|
|
3940
4313
|
// already names this shape "clean-exit-no-trailer" — unreachable in print mode until now.
|
|
3941
4314
|
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
4315
|
+
executionSignal()?.throwIfAborted();
|
|
3942
4316
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
3943
4317
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3944
4318
|
const startupLeft = workerLaunchedAt === undefined
|
|
@@ -3953,6 +4327,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3953
4327
|
}
|
|
3954
4328
|
// T2 (OBS-264): same probe cadence the interactive loop uses — see harvestSliceMs.
|
|
3955
4329
|
slice = Math.min(slice, harvestSliceMs(Date.now() - lastProgressAt));
|
|
4330
|
+
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
3956
4331
|
if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
3957
4332
|
processExited = true;
|
|
3958
4333
|
break;
|
|
@@ -4000,7 +4375,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4000
4375
|
// interactive attempts to the same exit marker before close and the post-hoc usage disk read
|
|
4001
4376
|
// so a writer never races the reader (real CLIs can flush usage asynchronously after the trailer).
|
|
4002
4377
|
if (interactive && finished && !exitRe.test(output)) {
|
|
4003
|
-
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true });
|
|
4378
|
+
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true }).catch(noteDriverUnreadable);
|
|
4004
4379
|
}
|
|
4005
4380
|
// Free availability reroutes deliberately reuse the charged-attempt ordinal. Stream artifacts
|
|
4006
4381
|
// cannot: every dispatch owns bytes that must remain independently recoverable, so derive the
|
|
@@ -4037,8 +4412,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4037
4412
|
try {
|
|
4038
4413
|
await closeSlot(slot);
|
|
4039
4414
|
if (!workerFinished) {
|
|
4415
|
+
const ref = await preserveWorktree(wt);
|
|
4416
|
+
if (ref) {
|
|
4417
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
4418
|
+
reapedWorktreeRef = ref;
|
|
4419
|
+
}
|
|
4040
4420
|
journal.append("worker-reaped-before-harvest", t.id, {
|
|
4041
|
-
slot: slot.name, attempt, cause: workerCause ?? "stall-timeout",
|
|
4421
|
+
slot: slot.name, attempt, cause: hardTimedOut ? "hard-timeout" : workerCause ?? "stall-timeout",
|
|
4422
|
+
...reapReports.get(slot),
|
|
4423
|
+
...(hardTimedOut ? { heldLegs } : {}),
|
|
4424
|
+
...(startupEvidence ? { evidence: startupEvidence } : {}),
|
|
4042
4425
|
});
|
|
4043
4426
|
}
|
|
4044
4427
|
}
|
|
@@ -4085,6 +4468,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4085
4468
|
// commits survive via the existing commitsToCarry/cherryPickCommits carry-forward.
|
|
4086
4469
|
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4087
4470
|
return;
|
|
4471
|
+
if (!workerFinished && !processExited && (timedOut || deadChannelKilled || quotaBannerKilled)) {
|
|
4472
|
+
const seat = channelKey(assignment);
|
|
4473
|
+
stallReaps = stallSeat === seat ? stallReaps + 1 : 1;
|
|
4474
|
+
stallSeat = seat;
|
|
4475
|
+
if (stallReaps >= 2) {
|
|
4476
|
+
const ref = await preserveWorktree(wt);
|
|
4477
|
+
if (ref)
|
|
4478
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
4479
|
+
await park(t, `two consecutive stall reaps without a gate on seat ${seat}`, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { seat, stallReaps });
|
|
4480
|
+
return;
|
|
4481
|
+
}
|
|
4482
|
+
}
|
|
4483
|
+
else {
|
|
4484
|
+
stallReaps = 0;
|
|
4485
|
+
stallSeat = undefined;
|
|
4486
|
+
}
|
|
4088
4487
|
const preHarvestResult = result;
|
|
4089
4488
|
// T2 (OBS-264): recognize committed no-trailer work BEFORE any no-trailer streak, provider,
|
|
4090
4489
|
// quota or dead-channel routing. Gates never trusted the trailer, so this successful synthesis
|
|
@@ -4276,6 +4675,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4276
4675
|
}
|
|
4277
4676
|
// Provisioning can mutate the shared dependency tree the pristine capture is still using.
|
|
4278
4677
|
await waitForBaseline(t.id);
|
|
4678
|
+
stallReaps = 0;
|
|
4679
|
+
stallSeat = undefined;
|
|
4279
4680
|
graph = setStatus(graph, t.id, "gated");
|
|
4280
4681
|
saveGraph(repoRoot, graph);
|
|
4281
4682
|
// OBS-47: re-assert the node_modules link BEFORE gates run on any attempt. A worker may have
|
|
@@ -4340,6 +4741,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4340
4741
|
let results = [];
|
|
4341
4742
|
let commits = [];
|
|
4342
4743
|
gateLoop: while (true) {
|
|
4744
|
+
executionSignal()?.throwIfAborted();
|
|
4343
4745
|
const gated = await gitHead(wt);
|
|
4344
4746
|
gateSubject = { commit: await gateCommitSubject(taskBase, gated, wt), attempt };
|
|
4345
4747
|
await trackedDriver.project?.(t.id, "in-review");
|
|
@@ -4374,12 +4776,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4374
4776
|
}
|
|
4375
4777
|
}
|
|
4376
4778
|
else {
|
|
4377
|
-
({ results, commits } = await
|
|
4779
|
+
({ results, commits } = await withCommandContext(t.id, () => runReviewRecovery(t, {
|
|
4378
4780
|
carriedFindings: outstandingFindings,
|
|
4379
4781
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
4380
4782
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4381
4783
|
collateral: collateral.get(t.id) ?? [],
|
|
4382
|
-
|
|
4784
|
+
selectTests: !testGateFailed,
|
|
4383
4785
|
via: cfg.visibility.llm === "pane"
|
|
4384
4786
|
? {
|
|
4385
4787
|
driver: trackedDriver,
|
|
@@ -4398,13 +4800,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4398
4800
|
: undefined,
|
|
4399
4801
|
excludeReviewers: badReviewers,
|
|
4400
4802
|
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
4803
|
+
reviewNoVerdicts, // Leg-2 (OBS-1052): see the recheck site above
|
|
4401
4804
|
onGate,
|
|
4402
4805
|
})));
|
|
4403
4806
|
}
|
|
4404
4807
|
results.forEach(classifySignalOnlyTest);
|
|
4405
4808
|
graph = addEvidence(graph, t.id, { commits, gateResults: results, artifacts: [promptFile] });
|
|
4406
4809
|
saveGraph(repoRoot, graph);
|
|
4407
|
-
if (results.some((g) => g.gate === "test" && !g.pass
|
|
4810
|
+
if (results.some((g) => g.gate === "test" && !g.pass
|
|
4811
|
+
&& !(cfg.executionPolicy?.boundedInfrastructure && failureDisposition(g) === "infrastructure")))
|
|
4408
4812
|
testGateFailed = true;
|
|
4409
4813
|
if (results.every(gateSatisfied)) {
|
|
4410
4814
|
// T6: every gate — the review included — is satisfied on this commit, so the failure brief
|
|
@@ -4416,6 +4820,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4416
4820
|
// gate-fail brief is, so prior-RUN evidence (retired by its own rule, not by this reviewer)
|
|
4417
4821
|
// survives and only this run's settled findings go.
|
|
4418
4822
|
feedback = withCarriedEvidence("");
|
|
4823
|
+
executionSignal()?.throwIfAborted();
|
|
4419
4824
|
const m = await mergeSerial(taskBranch, t, gated);
|
|
4420
4825
|
if (m.tipMoved) {
|
|
4421
4826
|
journal.append("tip-moved", t.id, m.tipMoved);
|
|
@@ -4465,12 +4870,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4465
4870
|
// flowing through the chargeable quality path below.
|
|
4466
4871
|
const infraFailure = results.find((g) => gateFailed(g) && g.meta?.infra === true);
|
|
4467
4872
|
if (infraFailure) {
|
|
4468
|
-
await park(t, `${infraFailure.gate}: ${infraFailure.details}`, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4873
|
+
await park(t, `${infraFailure.gate}: ${infraFailure.details}${infraFailure.meta?.recoveryBlocked ? ` — ${infraFailure.meta.recoveryBlocked}` : ""}`, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4874
|
+
return;
|
|
4875
|
+
}
|
|
4876
|
+
// OBS-1007: a cap trip is a typed park of its own — the diff cannot shrink by retrying and the
|
|
4877
|
+
// only honest verb is a recheck under a raised cap. Decided BEFORE the authoring classifier, which
|
|
4878
|
+
// used to lift `gates.diffCap` out of the cap prose as a files[] hint.
|
|
4879
|
+
const capTrip = results.find(isDiffCapPark);
|
|
4880
|
+
if (capTrip) {
|
|
4881
|
+
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4469
4882
|
return;
|
|
4470
4883
|
}
|
|
4471
4884
|
// OBS-547: who pays for this red is decided by the run's collateral prediction (see
|
|
4472
4885
|
// dispositionScopeRed) — an authoring defect parks unchargeable before any accounting below.
|
|
4473
|
-
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4886
|
+
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
4474
4887
|
return;
|
|
4475
4888
|
gateFails++; // this attempt's gates failed — the one place quality degradation is verified (never inferred from attempts)
|
|
4476
4889
|
// v1.53 T3: prefer the CLI's own session id captured from this attempt's output (kimi's resume
|
|
@@ -4630,6 +5043,39 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4630
5043
|
return;
|
|
4631
5044
|
}
|
|
4632
5045
|
};
|
|
5046
|
+
const executeWithPolicy = async (t) => {
|
|
5047
|
+
const policy = cfg.executionPolicy;
|
|
5048
|
+
if (!policy || (t.humanGate && !approved.has(t.id)))
|
|
5049
|
+
return execTask(t);
|
|
5050
|
+
await withExecutionBudget({ limitMs: policy.taskExecutionLimitMs, taskId: t.id,
|
|
5051
|
+
readEvents: () => journal.read(), append: journal.append.bind(journal) }, async () => {
|
|
5052
|
+
const signal = executionSignal();
|
|
5053
|
+
const slots = new Set();
|
|
5054
|
+
budgetSlots.set(signal, slots);
|
|
5055
|
+
let closing;
|
|
5056
|
+
const closeOwned = () => withoutExecutionBudget(async () => {
|
|
5057
|
+
for (const slot of slots)
|
|
5058
|
+
await closeSlot(slot);
|
|
5059
|
+
});
|
|
5060
|
+
const stop = () => {
|
|
5061
|
+
closing = closeOwned();
|
|
5062
|
+
// Awaited again below; prevent an unhandled rejection while the worker unwinds.
|
|
5063
|
+
void closing.catch(() => { });
|
|
5064
|
+
};
|
|
5065
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
5066
|
+
try {
|
|
5067
|
+
await execTask(t);
|
|
5068
|
+
}
|
|
5069
|
+
finally {
|
|
5070
|
+
signal.removeEventListener("abort", stop);
|
|
5071
|
+
if (signal.aborted) {
|
|
5072
|
+
await closing;
|
|
5073
|
+
await closeOwned();
|
|
5074
|
+
}
|
|
5075
|
+
budgetSlots.delete(signal);
|
|
5076
|
+
}
|
|
5077
|
+
});
|
|
5078
|
+
};
|
|
4633
5079
|
taskLoopStarted = true;
|
|
4634
5080
|
const inflight = new Map();
|
|
4635
5081
|
// A settled worker can release a dependency or an approval in the same poll tick.
|
|
@@ -4662,8 +5108,33 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4662
5108
|
.filter((t) => !inflight.has(t.id))
|
|
4663
5109
|
.slice(0, Math.max(0, concurrency - inflight.size));
|
|
4664
5110
|
for (const t of ready) {
|
|
4665
|
-
const p =
|
|
5111
|
+
const p = executeWithPolicy(t)
|
|
4666
5112
|
.catch(async (err) => {
|
|
5113
|
+
const cleanupErrors = [];
|
|
5114
|
+
for (const [slot, owner] of workerOwners) {
|
|
5115
|
+
if (owner.taskId !== t.id || !liveSlots.has(slot))
|
|
5116
|
+
continue;
|
|
5117
|
+
try {
|
|
5118
|
+
await withoutExecutionBudget(() => closeSlot(slot));
|
|
5119
|
+
}
|
|
5120
|
+
catch (error) {
|
|
5121
|
+
// Cleanup evidence must not replace the task's original failure or abort siblings.
|
|
5122
|
+
cleanupErrors.push({ slot: slot.name, attempt: owner.attempt, error: String(error) });
|
|
5123
|
+
}
|
|
5124
|
+
}
|
|
5125
|
+
const cleanupEvidence = cleanupErrors.length ? { cleanupErrors } : {};
|
|
5126
|
+
if (err instanceof ExecutionBudgetExceeded) {
|
|
5127
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
5128
|
+
const ref = await preserveWorktree(wt);
|
|
5129
|
+
if (ref)
|
|
5130
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
5131
|
+
const dispatch = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "task-dispatch");
|
|
5132
|
+
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: "execution-budget-exhausted", limitMs: cfg.executionPolicy.taskExecutionLimitMs,
|
|
5133
|
+
...cleanupEvidence,
|
|
5134
|
+
...(Number.isInteger(dispatch?.data.attempt) ? { attempt: dispatch.data.attempt } : {}),
|
|
5135
|
+
...(dispatch?.data.assignment ? { assignment: dispatch.data.assignment } : {}) });
|
|
5136
|
+
return;
|
|
5137
|
+
}
|
|
4667
5138
|
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
4668
5139
|
const dispatch = [...taskEvents].reverse().find((e) => e.event === "task-dispatch");
|
|
4669
5140
|
// OBS-206: shared rule with `resume --retry-failed` — see classifyTaskFailure.
|
|
@@ -4671,7 +5142,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4671
5142
|
const attempts = dispatch && Number.isInteger(dispatch.data.attempt) ? dispatch.data.attempt : 0;
|
|
4672
5143
|
graph = setStatus(graph, t.id, "failed");
|
|
4673
5144
|
saveGraph(repoRoot, graph);
|
|
4674
|
-
journal.append("task-failed", t.id, { error: String(err), kind, attempts });
|
|
5145
|
+
journal.append("task-failed", t.id, { error: String(err), kind, attempts, ...cleanupEvidence });
|
|
4675
5146
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
4676
5147
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
4677
5148
|
})
|
|
@@ -4767,7 +5238,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4767
5238
|
poll = setTimeout(pollApprovals, APPROVAL_POLL_MS);
|
|
4768
5239
|
let tipFailed;
|
|
4769
5240
|
try {
|
|
4770
|
-
const verification =
|
|
5241
|
+
const verification = withCommandContext(undefined, () => verifyIntegrationTipCached(intWt, commands, journal, { lastMergedTask, baseline, signal: controller.signal }), controller.signal);
|
|
4771
5242
|
activeTipVerify = { controller, settled: verification.then(() => { }, () => { }) };
|
|
4772
5243
|
tipFailed = await verification;
|
|
4773
5244
|
}
|