tickmarkr 2.5.5 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/cli/commands/approve.js +64 -3
- package/dist/cli/commands/doctor.d.ts +10 -0
- package/dist/cli/commands/fleet.js +4 -0
- package/dist/cli/commands/plan.js +20 -4
- package/dist/cli/commands/verify.js +102 -81
- package/dist/config/config.d.ts +11 -0
- package/dist/config/config.js +21 -12
- package/dist/config/fleet-overlay.js +55 -17
- package/dist/drivers/index.js +2 -1
- package/dist/drivers/orca.d.ts +21 -1
- package/dist/drivers/orca.js +209 -27
- package/dist/gates/baseline.d.ts +14 -3
- package/dist/gates/baseline.js +61 -14
- package/dist/gates/cache.d.ts +14 -13
- package/dist/gates/cache.js +17 -5
- package/dist/gates/llm.d.ts +3 -0
- package/dist/gates/llm.js +11 -0
- package/dist/gates/review.d.ts +28 -3
- package/dist/gates/review.js +106 -14
- package/dist/gates/run-gates.d.ts +6 -1
- package/dist/gates/run-gates.js +299 -101
- package/dist/gates/test-manifest.d.ts +30 -1
- package/dist/gates/test-manifest.js +113 -39
- package/dist/gates/test-reporter.js +14 -6
- package/dist/graph/graph.d.ts +2 -0
- package/dist/graph/graph.js +45 -1
- package/dist/run/consult.js +5 -4
- package/dist/run/daemon.d.ts +2 -0
- package/dist/run/daemon.js +480 -99
- package/dist/run/execution-budget.d.ts +25 -0
- package/dist/run/execution-budget.js +142 -0
- package/dist/run/git.d.ts +40 -0
- package/dist/run/git.js +89 -4
- package/dist/run/journal.d.ts +3 -1
- package/dist/run/journal.js +38 -16
- package/dist/run/lease.d.ts +44 -0
- package/dist/run/lease.js +226 -3
- package/dist/run/recovery.d.ts +8 -0
- package/dist/run/recovery.js +25 -0
- package/dist/run/repair-selection.d.ts +12 -0
- package/dist/run/repair-selection.js +56 -0
- package/dist/run/stall.d.ts +6 -1
- package/dist/run/stall.js +60 -3
- package/dist/tui/ink/fleet-app.d.ts +10 -2
- package/dist/tui/ink/fleet-app.js +33 -15
- package/package.json +2 -2
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +29 -2
package/dist/run/daemon.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
1
|
+
import { COMMAND_LEASE_TOKEN_ENV, commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
2
2
|
import { execFileSync, spawn } from "node:child_process";
|
|
3
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
4
4
|
import { shq } from "../adapters/types.js";
|
|
@@ -13,7 +13,7 @@ import { allAdapters, getAdapter, probeAll, readDoctor, rolePools } from "../ada
|
|
|
13
13
|
import { SettledTrailerTracker, addUsage, channelKey, matchesInputBox, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
14
14
|
import { bannerShell, paneDispatchCommand } from "../brand.js";
|
|
15
15
|
import { collateralHits } from "../compile/collateral.js";
|
|
16
|
-
import { DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
16
|
+
import { ExecutionPolicySchema, DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
17
17
|
import { DeliveryReadinessError } from "../drivers/herdr.js";
|
|
18
18
|
import { driverEvidence } from "../drivers/index.js";
|
|
19
19
|
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
@@ -25,8 +25,11 @@ import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHa
|
|
|
25
25
|
import { GATE_NAMES } from "../graph/schema.js";
|
|
26
26
|
import { distFingerprint } from "../cli/commands/version.js";
|
|
27
27
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
28
|
+
import { executionSignal, remainingExecutionMs, withExecutionBudget, withoutExecutionBudget, ExecutionBudgetExceeded } from "./execution-budget.js";
|
|
29
|
+
import { repairSelectionDecision } from "./repair-selection.js";
|
|
30
|
+
import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
28
31
|
import { runEnvironment } from "./environment.js";
|
|
29
|
-
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sh, shGit, SUITE_PARENT_ENV, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
32
|
+
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
30
33
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
31
34
|
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
32
35
|
import { isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
@@ -35,7 +38,7 @@ import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verif
|
|
|
35
38
|
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
36
39
|
import { desiredPanes } from "./reconcile.js";
|
|
37
40
|
import { readTierLiveness, readWatchBoard, supervisionBeatPath } from "./supervision.js";
|
|
38
|
-
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, } from "./stall.js";
|
|
41
|
+
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, reapOwnedProcessGroup, readOwnedProcessGroup, } from "./stall.js";
|
|
39
42
|
// The live set is also the ownership claim shared by task cleanup and termination.
|
|
40
43
|
export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
41
44
|
if (!liveSlots.delete(slot))
|
|
@@ -48,6 +51,10 @@ export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
|
48
51
|
throw error;
|
|
49
52
|
}
|
|
50
53
|
}
|
|
54
|
+
// A fixed attempt ceiling is independent of the rolling inactivity clock.
|
|
55
|
+
let attemptHardTimeoutMs;
|
|
56
|
+
export function setAttemptHardTimeoutMsForTests(ms) { attemptHardTimeoutMs = ms; }
|
|
57
|
+
export function resetAttemptHardTimeoutMsForTests() { attemptHardTimeoutMs = undefined; }
|
|
51
58
|
// Compatibility exports: gate dispatch can import stall.ts without importing the daemon.
|
|
52
59
|
export { harvestCpuFlatWindowMs, resetHarvestCpuFlatMsForTests, setHarvestCpuFlatMsForTests, workerTreeCpuMs, } from "./stall.js";
|
|
53
60
|
const MODE_RANK = { "staff-led": 0, "risk-based": 1, "partner-led": 2 };
|
|
@@ -261,6 +268,22 @@ export function decisiveReviewRounds(events) {
|
|
|
261
268
|
close(taskId); // Map iteration tolerates deleting the current key
|
|
262
269
|
return events.filter((e) => !spent.has(e));
|
|
263
270
|
}
|
|
271
|
+
/** OBS-1034: the files a byte cap drops from a repair diff — every file whose header starts at or past the cap, plus the one the cut lands inside.
|
|
272
|
+
* Offsets are UTF-8 BYTES (the cap's unit), never UTF-16 code units. `names` is the NUL-delimited `git diff --name-only` list in
|
|
273
|
+
* header order, so quoted headers (tabs, quotes, non-ASCII) still name their file. */
|
|
274
|
+
function repairDiffFilesCut(diff, cap, names) {
|
|
275
|
+
const files = [];
|
|
276
|
+
const header = Buffer.from("\ndiff --git ");
|
|
277
|
+
const starts = diff.subarray(0, header.length - 1).equals(header.subarray(1)) ? [0] : [];
|
|
278
|
+
for (let at = diff.indexOf(header); at !== -1; at = diff.indexOf(header, at + 1))
|
|
279
|
+
starts.push(at + 1);
|
|
280
|
+
starts.forEach((start, i) => {
|
|
281
|
+
const end = starts[i + 1] === undefined ? diff.length : starts[i + 1] - 1;
|
|
282
|
+
if (start >= cap || end > cap)
|
|
283
|
+
files.push(names[i] ?? "(unnamed)");
|
|
284
|
+
});
|
|
285
|
+
return files;
|
|
286
|
+
}
|
|
264
287
|
/** The fix-only contract: the findings verbatim, then the diff content of the work already landed. */
|
|
265
288
|
function repairBrief(findings, diff, baseRef) {
|
|
266
289
|
const repair = { findings, diff };
|
|
@@ -781,6 +804,12 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
781
804
|
}
|
|
782
805
|
return tipFailed;
|
|
783
806
|
}
|
|
807
|
+
async function commitsAheadOfRef(base, head, cwd) {
|
|
808
|
+
if (head === base)
|
|
809
|
+
return [];
|
|
810
|
+
const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, cwd);
|
|
811
|
+
return r.code === 0 ? r.stdout.trim().split("\n").filter(Boolean) : [];
|
|
812
|
+
}
|
|
784
813
|
async function commitsAheadOf(base, wt) {
|
|
785
814
|
const head = await gitHead(wt);
|
|
786
815
|
if (head === base)
|
|
@@ -1376,6 +1405,12 @@ function endedRunIdsForReconcile(repoRoot) {
|
|
|
1376
1405
|
return ended;
|
|
1377
1406
|
}
|
|
1378
1407
|
export async function runDaemon(repoRoot, opts = {}) {
|
|
1408
|
+
// OBS-1045: a daemon launched from inside a leased command (a nested runner, an operator's test
|
|
1409
|
+
// shell) inherits its ancestor's lease token through the environment, and every gate it ran would
|
|
1410
|
+
// ride that lease instead of taking its own. Clear it at run-start so the first gate leases itself.
|
|
1411
|
+
if (process.env[COMMAND_LEASE_TOKEN_ENV]) {
|
|
1412
|
+
delete process.env[COMMAND_LEASE_TOKEN_ENV];
|
|
1413
|
+
}
|
|
1379
1414
|
// v1.51 T2 / OBS-89 (v1.60): retired --quality env seam. Mode resolution owns premium routing;
|
|
1380
1415
|
// route() no longer reads the retired env at all, so the old entrypoint scrub is gone with it.
|
|
1381
1416
|
const adapters = opts.adapters ?? allAdapters();
|
|
@@ -1442,6 +1477,21 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1442
1477
|
// > repo > global > default. The resolved cfg carries mode-compiled floors; route() never sees the mode.
|
|
1443
1478
|
const rm = resolveRunMode(repoRoot, { flag: opts.mode, spec: graph.mode, globalDir: opts.globalDir });
|
|
1444
1479
|
const cfg = rm.cfg;
|
|
1480
|
+
if (opts.resume) {
|
|
1481
|
+
const start = Journal.open(repoRoot, runId).read().find((e) => e.event === "run-start");
|
|
1482
|
+
const snapshot = start?.data.effectivePolicy;
|
|
1483
|
+
if (snapshot !== undefined && (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)
|
|
1484
|
+
|| !("config" in snapshot) || !snapshot.config || typeof snapshot.config !== "object" || Array.isArray(snapshot.config))) {
|
|
1485
|
+
throw new Error("cannot restore malformed recorded execution policy");
|
|
1486
|
+
}
|
|
1487
|
+
const recorded = snapshot;
|
|
1488
|
+
// Historical runs did not opt in; current settings cannot grant them a new allowance.
|
|
1489
|
+
cfg.executionPolicy = recorded?.config?.executionPolicy === undefined
|
|
1490
|
+
? undefined : ExecutionPolicySchema.parse(recorded.config.executionPolicy);
|
|
1491
|
+
}
|
|
1492
|
+
if (cfg.executionPolicy && driver.id !== "subprocess") {
|
|
1493
|
+
throw new Error("experimental executionPolicy currently requires the subprocess driver");
|
|
1494
|
+
}
|
|
1445
1495
|
// Resolve both budgets once: worker fan-out divides by dispatch concurrency; verification
|
|
1446
1496
|
// divides by one runner command holding a lease. Explicit operator caps still win.
|
|
1447
1497
|
const concurrency = opts.concurrency ?? cfg.concurrency;
|
|
@@ -1472,14 +1522,58 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1472
1522
|
// opens/closes keep the ledger exact. Termination then closes exactly what is still live — and a
|
|
1473
1523
|
// slot closed once (task-done, quota reroute, gate self-clean) can never be closed twice.
|
|
1474
1524
|
const liveSlots = new Set();
|
|
1475
|
-
|
|
1525
|
+
// A driver handle identifies the physical slot even if a retry returns a new Slot object.
|
|
1526
|
+
const trustAnsweredSlots = new Set();
|
|
1527
|
+
const workerOwners = new Map();
|
|
1528
|
+
const reapReports = new Map();
|
|
1529
|
+
const reapWorker = async (slot) => {
|
|
1530
|
+
const owner = workerOwners.get(slot);
|
|
1531
|
+
if (owner) {
|
|
1532
|
+
const processGroup = readOwnedProcessGroup(owner.groupFile);
|
|
1533
|
+
let survivors;
|
|
1534
|
+
try {
|
|
1535
|
+
const shared = processGroup !== undefined && [...workerOwners].some(([other, otherOwner]) => other !== slot && liveSlots.has(other) && readOwnedProcessGroup(otherOwner.groupFile) === processGroup);
|
|
1536
|
+
if (shared)
|
|
1537
|
+
throw new Error(`worker group ${processGroup} is shared with another live attempt`);
|
|
1538
|
+
survivors = await reapOwnedProcessGroup(processGroup, slot.cwd);
|
|
1539
|
+
}
|
|
1540
|
+
catch (error) {
|
|
1541
|
+
journal.append("worker-process-reaped", owner.taskId, { slot: slot.name, attempt: owner.attempt,
|
|
1542
|
+
processGroup: processGroup ?? null, survivors: null, error: String(error) });
|
|
1543
|
+
throw error;
|
|
1544
|
+
}
|
|
1545
|
+
reapReports.set(slot, { processGroup: processGroup ?? null, survivors });
|
|
1546
|
+
journal.append("worker-process-reaped", owner.taskId, {
|
|
1547
|
+
slot: slot.name, attempt: owner.attempt, processGroup: processGroup ?? null, survivors,
|
|
1548
|
+
});
|
|
1549
|
+
if (survivors && survivors.length > 0)
|
|
1550
|
+
throw new Error(`worker group ${processGroup} survivors: ${survivors.join(", ")}`);
|
|
1551
|
+
}
|
|
1552
|
+
};
|
|
1553
|
+
const closeSlot = (s) => closeLiveSlot(liveSlots, { close: async (slot) => {
|
|
1554
|
+
await reapWorker(slot);
|
|
1555
|
+
await driver.close(slot);
|
|
1556
|
+
} }, s);
|
|
1557
|
+
const budgetSlots = new Map();
|
|
1476
1558
|
const trackedDriver = {
|
|
1477
1559
|
id: driver.id,
|
|
1478
1560
|
interactive: driver.interactive,
|
|
1479
1561
|
...(driver.readSource ? { readSource: driver.readSource } : {}),
|
|
1480
1562
|
...(driver.describe ? { describe: driver.describe.bind(driver) } : {}),
|
|
1481
1563
|
...(driver.focus ? { focus: driver.focus.bind(driver) } : {}),
|
|
1482
|
-
slot: async (cwd, name, o) => {
|
|
1564
|
+
slot: async (cwd, name, o) => {
|
|
1565
|
+
const signal = executionSignal();
|
|
1566
|
+
signal?.throwIfAborted();
|
|
1567
|
+
const s = await driver.slot(cwd, name, o);
|
|
1568
|
+
liveSlots.add(s);
|
|
1569
|
+
if (signal)
|
|
1570
|
+
budgetSlots.get(signal)?.add(s);
|
|
1571
|
+
if (signal?.aborted) {
|
|
1572
|
+
await withoutExecutionBudget(() => closeSlot(s));
|
|
1573
|
+
signal.throwIfAborted();
|
|
1574
|
+
}
|
|
1575
|
+
return s;
|
|
1576
|
+
},
|
|
1483
1577
|
run: (s, cmd) => driver.run(s, cmd),
|
|
1484
1578
|
waitOutput: (s, p, ms, o) => driver.waitOutput(s, p, ms, o),
|
|
1485
1579
|
waitAgentStatus: (s, st, ms) => driver.waitAgentStatus(s, st, ms),
|
|
@@ -1540,6 +1634,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1540
1634
|
activeTipVerify.controller.abort(termination);
|
|
1541
1635
|
await activeTipVerify.settled;
|
|
1542
1636
|
}
|
|
1637
|
+
if (cfg.visibility.keepPanes === "forever") {
|
|
1638
|
+
for (const s of Array.from(liveSlots)) {
|
|
1639
|
+
try {
|
|
1640
|
+
await reapWorker(s);
|
|
1641
|
+
}
|
|
1642
|
+
catch { /* survivor evidence is retained by the reaper */ }
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1543
1645
|
if (cfg.visibility.keepPanes !== "forever") {
|
|
1544
1646
|
for (const s of Array.from(liveSlots)) { // snapshot: failed closes restore membership for the next sweep
|
|
1545
1647
|
try {
|
|
@@ -1563,6 +1665,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1563
1665
|
const demotedReviewers = new Set(journal.read()
|
|
1564
1666
|
.filter((event) => event.event === "review-pool-demotion" && typeof event.data.reviewer === "string")
|
|
1565
1667
|
.map((event) => event.data.reviewer));
|
|
1668
|
+
// OBS-1010: a demotion the JOURNAL carries into this session left the rotation for the run — a
|
|
1669
|
+
// resumed daemon excludes that seat from every review pick (reviewer role only; the worker channel
|
|
1670
|
+
// is untouched — channel-demotion is the worker fold, replayExcludedChannels). In-session demotions
|
|
1671
|
+
// keep their soft ordering above so a still-live seat can be re-tried at the end of the rotation.
|
|
1672
|
+
const replayedReviewerExclusions = opts.resume ? [...demotedReviewers] : [];
|
|
1673
|
+
// OBS-1025 add.2 / OBS-1052 (Leg-2): the run-scoped no-verdict tally run-gates retires a seat from at two
|
|
1674
|
+
// strikes. execTask's badReviewers already excludes a seat after ONE strike within a task, so this map
|
|
1675
|
+
// is what bounds a seat that flakes once per task (or once per session) across the run. Seeded from the
|
|
1676
|
+
// journal's review-no-verdict rows — the same notes noteReviewEvent appends — so a resumed daemon keeps
|
|
1677
|
+
// the count; run-gates appends to it in place, exactly as it does demotedReviewers.
|
|
1678
|
+
const reviewNoVerdicts = new Map();
|
|
1679
|
+
for (const event of journal.read()) {
|
|
1680
|
+
if (event.event !== "review-no-verdict" || event.data.noVerdict !== true || typeof event.data.reviewer !== "string")
|
|
1681
|
+
continue;
|
|
1682
|
+
reviewNoVerdicts.set(event.data.reviewer, [...(reviewNoVerdicts.get(event.data.reviewer) ?? []), String(event.data.cause)]);
|
|
1683
|
+
}
|
|
1566
1684
|
const reviewHistory = journal.read()
|
|
1567
1685
|
.filter((event) => event.event === "gate-result" && event.data.gate === "review" && typeof event.data.reviewer === "string")
|
|
1568
1686
|
.map((event) => event.data.reviewer);
|
|
@@ -1702,11 +1820,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1702
1820
|
};
|
|
1703
1821
|
// Context carries attribution through gates and remote inference; only shell commands acquire.
|
|
1704
1822
|
const commandLeases = new CommandLeases();
|
|
1705
|
-
const withCommandContext = (taskId, run, signal) => runWithCommandLease((_command, execute) => commandLeases.run(async () => {
|
|
1823
|
+
const withCommandContext = (taskId, run, signal = executionSignal()) => runWithCommandLease((_command, execute) => commandLeases.run(async () => {
|
|
1706
1824
|
let lastCount = -1;
|
|
1707
1825
|
const startedAt = Date.now();
|
|
1708
1826
|
for (;;) {
|
|
1709
1827
|
signal?.throwIfAborted();
|
|
1828
|
+
executionSignal()?.throwIfAborted();
|
|
1710
1829
|
const count = await liveSuiteCount(repoRoot);
|
|
1711
1830
|
if (count === 0)
|
|
1712
1831
|
break;
|
|
@@ -1730,7 +1849,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1730
1849
|
const waitForBaseline = async (taskId) => {
|
|
1731
1850
|
if (baselinePending)
|
|
1732
1851
|
journal.append("baseline-wait", taskId, { baseRef });
|
|
1733
|
-
|
|
1852
|
+
const signal = executionSignal();
|
|
1853
|
+
if (!signal) {
|
|
1854
|
+
await baselineCapture;
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
signal.throwIfAborted();
|
|
1858
|
+
// Capture is run-owned; stop this task waiting without cancelling another task's baseline.
|
|
1859
|
+
await new Promise((resolve, reject) => {
|
|
1860
|
+
const abort = () => reject(signal.reason);
|
|
1861
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
1862
|
+
baselineCapture.then(() => { signal.removeEventListener("abort", abort); resolve(); }, (error) => { signal.removeEventListener("abort", abort); reject(error); });
|
|
1863
|
+
});
|
|
1734
1864
|
};
|
|
1735
1865
|
// Phase 46 (RES-01/RES-02): the resume-state map is built ONCE here so execTask closes over it.
|
|
1736
1866
|
// Empty Map on fresh runs — every seed below conditions on resume.get(t.id), never on opts.resume (the
|
|
@@ -1745,7 +1875,23 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1745
1875
|
// every shell this run spawns will divide the machine by. Recorded evidence from another session
|
|
1746
1876
|
// is only reusable against this.
|
|
1747
1877
|
const sessionCapacity = resolvedCapacity();
|
|
1878
|
+
// R41: run-start/run-resume provenance — the protocol + effective lifecycle policy measured at
|
|
1879
|
+
// the repository root when this session began. Gate-result rows do NOT carry this value: each row
|
|
1880
|
+
// carries what run-gates measured for the task worktree the gate ran in (meta.verification), and
|
|
1881
|
+
// the replay guard below compares against a fresh measurement for the recreated worktree.
|
|
1882
|
+
const sessionVerification = verificationProtocol(process.env, repoRoot);
|
|
1748
1883
|
const replayedExclusions = opts.resume ? journal.replayExcludedChannels() : new Set();
|
|
1884
|
+
// Stamp each daemon session, including resumes with changed configuration or CLI overrides.
|
|
1885
|
+
// Keep the loaded-config hash's existing meaning; resolved execution inputs are explicit beside it.
|
|
1886
|
+
// Journal.append applies the normal credential redaction to the readable policy snapshot.
|
|
1887
|
+
const sessionMetadata = {
|
|
1888
|
+
environment: runEnvironment(cfg, channels, health),
|
|
1889
|
+
distFingerprint: distFingerprint(),
|
|
1890
|
+
verification: sessionVerification,
|
|
1891
|
+
effectivePolicy: { config: cfg, concurrency, driver: driver.id, commands },
|
|
1892
|
+
mode: rm.mode.mode,
|
|
1893
|
+
modeSource: rm.source,
|
|
1894
|
+
};
|
|
1749
1895
|
if (opts.resume) {
|
|
1750
1896
|
// v1.53 T5: a superseded run is dead — resuming it beside its successor is the exact
|
|
1751
1897
|
// two-concurrent-runs hazard supersession exists to prevent. Fail closed, naming the successor.
|
|
@@ -1794,6 +1940,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1794
1940
|
graph = setStatus(graph, id, st);
|
|
1795
1941
|
}
|
|
1796
1942
|
journal.append("run-resume", undefined, {
|
|
1943
|
+
...sessionMetadata,
|
|
1797
1944
|
pid: process.pid, // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
1798
1945
|
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
1799
1946
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
@@ -1820,11 +1967,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1820
1967
|
// Attach a rejection handler immediately; gate and run-close awaits still propagate the error.
|
|
1821
1968
|
void baselineCapture.catch(() => { baselineFailed = true; });
|
|
1822
1969
|
writeFileSync(join(journal.dir, "graph.json"), readFileSync(join(tickmarkrDir(repoRoot), "graph.json")));
|
|
1823
|
-
// v1.70 T2: environment identity beside the graph/branch identity — running tickmarkr version,
|
|
1824
|
-
// loaded-config hash, and the probed CLI version of each adapter holding a channel in the run,
|
|
1825
|
-
// gathered through the existing probe/config-load paths (no second mechanism).
|
|
1826
|
-
const environment = runEnvironment(cfg, channels, health);
|
|
1827
1970
|
journal.append("run-start", undefined, {
|
|
1971
|
+
...sessionMetadata,
|
|
1828
1972
|
pid: process.pid, baseRef, commands, channels: channels.map(channelKey),
|
|
1829
1973
|
channelsByRole: {
|
|
1830
1974
|
worker: pools.worker.map(channelKey),
|
|
@@ -1834,9 +1978,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1834
1978
|
},
|
|
1835
1979
|
driver: driver.id,
|
|
1836
1980
|
driverEvidence: driverEvidence(cfg, driver, opts.driverOverride),
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
environment, ...(prior ? { supersedes: prior.runId } : {}),
|
|
1981
|
+
branch, graphDefinitionHash: graphDefinitionHash(graph),
|
|
1982
|
+
...(prior ? { supersedes: prior.runId } : {}),
|
|
1840
1983
|
}); // graphDefinitionHash: T3 engagement identity (status+resume share it); pid: v1.13 (VIS-11) liveness; mode/modeSource: v1.51 T2; supersedes: v1.53 T5
|
|
1841
1984
|
runStarted = true;
|
|
1842
1985
|
// v1.53 T5: mark the prior run AFTER this run's run-start exists, so the prior journal never
|
|
@@ -1923,6 +2066,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1923
2066
|
let mergeChain = Promise.resolve();
|
|
1924
2067
|
const mergeSerial = (taskBranch, t, gated) => {
|
|
1925
2068
|
const next = mergeChain.then(() => {
|
|
2069
|
+
executionSignal()?.throwIfAborted();
|
|
1926
2070
|
journal.phaseStart(t.id, "merge");
|
|
1927
2071
|
return mergeTask(intWt, taskBranch, `tickmarkr: merge ${t.id} ${t.title}`, gated);
|
|
1928
2072
|
});
|
|
@@ -1939,10 +2083,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1939
2083
|
];
|
|
1940
2084
|
return `${identity} — release with ${commands.map((command) => `\`${command}\``).join(" or ")}`;
|
|
1941
2085
|
};
|
|
1942
|
-
|
|
2086
|
+
// `present`: OBS-1007 add.4/5 — a gate DIAGNOSTIC may name only paths that exist in the tree or the
|
|
2087
|
+
// diff (`f.filepath`, `scripted.sh`, `gates.diffCap` lifted from review prose are not paths). A
|
|
2088
|
+
// worker's own scope refusal keeps the lexical rule: the file it needs may not exist yet.
|
|
2089
|
+
const namedPaths = (text, present) => [...text.matchAll(/(?:^|[\s`'"(])((?:[A-Za-z0-9_@.()[\]-]+\/)+[A-Za-z0-9_@.[\]-]+|[A-Za-z0-9_@-]+(?:\.[A-Za-z0-9_-]+)+)(?=$|[\s`'"),:;.!?])/g)]
|
|
1943
2090
|
.map((match) => match[1].replace(/^\.\//, "").replace(/\.$/, ""))
|
|
1944
2091
|
.filter((path) => !path.split("/").includes("..")
|
|
1945
|
-
&& (
|
|
2092
|
+
&& (present
|
|
2093
|
+
? present.has(path)
|
|
2094
|
+
: (/\.[A-Za-z][A-Za-z0-9_-]*$/.test(basename(path)) || existsSync(join(repoRoot, path)))));
|
|
2095
|
+
const diffPaths = async (base, wt) => {
|
|
2096
|
+
const out = await shGit(`git diff --name-only -z ${shq(base)}..HEAD`, wt);
|
|
2097
|
+
return new Set(out.code === 0 ? out.stdout.split("\0").filter(Boolean) : []);
|
|
2098
|
+
};
|
|
2099
|
+
/** The paths a gate diagnostic may name: the gated worktree's tracked tree plus its diff — never the main checkout. */
|
|
2100
|
+
const treeOrDiffPaths = async (base, wt) => {
|
|
2101
|
+
const tree = await shGit("git ls-files -z", wt);
|
|
2102
|
+
return new Set([...(tree.code === 0 ? tree.stdout.split("\0").filter(Boolean) : []), ...await diffPaths(base, wt)]);
|
|
2103
|
+
};
|
|
1946
2104
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
1947
2105
|
// OBS-979: a worker refusal can identify the missing authoring scope even when gates
|
|
1948
2106
|
// subsequently supply the park's disposition. Keep that actionable path on the park itself.
|
|
@@ -1979,7 +2137,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1979
2137
|
// count. The two callers arrive at it differently (the attempt loop's index already excludes the
|
|
1980
2138
|
// dispatch in flight; a gate replay's rs.attempts already includes it), and the ordinal journaled
|
|
1981
2139
|
// below is derived from it here so both paths number the same dispatch identically.
|
|
1982
|
-
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode) => {
|
|
2140
|
+
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode, changed = new Set()) => {
|
|
1983
2141
|
const scopeRed = results.find((g) => g.gate === "scope" && gateFailed(g));
|
|
1984
2142
|
const verdict = scopeRed?.meta?.collateral;
|
|
1985
2143
|
if (!verdict && t.files.length > 0) {
|
|
@@ -1992,7 +2150,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1992
2150
|
// A scope verdict already attributes actual out-of-scope EDITS using the collateral map.
|
|
1993
2151
|
// Path-only diagnostics from the other gates name code that needs fixing, not an edit the
|
|
1994
2152
|
// worker made. Do not replace the scope gate's stronger attribution with that inference.
|
|
1995
|
-
const paths = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details));
|
|
2153
|
+
const paths = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details, changed));
|
|
1996
2154
|
if (refusal.length || (paths.length > 0 && paths.every((path) => !allowed(path)))) {
|
|
1997
2155
|
const classification = {
|
|
1998
2156
|
gate: refusal.length ? "worker" : reds[0]?.gate,
|
|
@@ -2075,6 +2233,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2075
2233
|
};
|
|
2076
2234
|
const execTask = async (t) => {
|
|
2077
2235
|
const startMs = Date.now();
|
|
2236
|
+
let stallSeat;
|
|
2237
|
+
let stallReaps = 0;
|
|
2238
|
+
let reapedWorktreeRef;
|
|
2078
2239
|
const taskTimeoutMinutes = t.timeoutMinutes ?? cfg.taskTimeoutMinutes;
|
|
2079
2240
|
if (t.humanGate && !approved.has(t.id)) {
|
|
2080
2241
|
// GATE-08: the condition is the APPROVAL, never the code path. `!opts.resume` (or any run-phase
|
|
@@ -2151,10 +2312,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2151
2312
|
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
2152
2313
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
2153
2314
|
}
|
|
2154
|
-
else if (!hintsChanged && rs && rs.tried.length) {
|
|
2315
|
+
else if (!hintsChanged && rs && rs.tried.length && !t.routingHints?.pin) {
|
|
2155
2316
|
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
2156
2317
|
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
2157
2318
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
2319
|
+
// OBS-1034: a PIN is exempt — route() already returned it, and a review-upheld repair of the diff
|
|
2320
|
+
// the pin produced belongs on the pin, not on the next seat its own tried[] entry would pick.
|
|
2158
2321
|
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
|
|
2159
2322
|
if (next)
|
|
2160
2323
|
assignment = next;
|
|
@@ -2180,7 +2343,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2180
2343
|
// stays unblocked); inert to replayStatuses (unknown events ignored, pinned at journal.test.ts:70-80).
|
|
2181
2344
|
if (rs)
|
|
2182
2345
|
journal.append("resume-restore", t.id, { attempts: rs.attempts, tried: [...tried], assignment });
|
|
2183
|
-
|
|
2346
|
+
// Keep one live list: recovery retries must see exclusions added by onGate during the round.
|
|
2347
|
+
const badReviewers = [...replayedReviewerExclusions];
|
|
2184
2348
|
const noteReviewEvent = (e) => {
|
|
2185
2349
|
journal.append(e.name, t.id, e.payload);
|
|
2186
2350
|
if (e.name === "review-no-verdict" && typeof e.payload.reviewer === "string") {
|
|
@@ -2190,7 +2354,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2190
2354
|
};
|
|
2191
2355
|
// Keep the measured siblings, including a red judge, while replacing only a missing review.
|
|
2192
2356
|
// Every dispatched seat is excluded for this task, so the loop is bounded by the eligible pool.
|
|
2193
|
-
const runReviewRecovery = async (task, ctx) => {
|
|
2357
|
+
const runReviewRecovery = async (task, ctx, allowSelection = true) => {
|
|
2358
|
+
ctx.authorizeInfraRetry = cfg.executionPolicy?.boundedInfrastructure
|
|
2359
|
+
? (subject, cause) => reserveInfrastructureRetry(journal.read(), task.id, subject, journal.append.bind(journal), cause) : undefined;
|
|
2360
|
+
if (cfg.executionPolicy?.repairSelection && allowSelection) {
|
|
2361
|
+
const decision = repairSelectionDecision(journal.read(), task.id, true);
|
|
2362
|
+
ctx.selectTests = decision.selectTests;
|
|
2363
|
+
ctx.requiredRepairTests = decision.requiredFiles;
|
|
2364
|
+
ctx.selectionReason = decision.reason;
|
|
2365
|
+
}
|
|
2194
2366
|
const round = await runGates(task, ctx);
|
|
2195
2367
|
let review = round.results.find((g) => g.gate === "review");
|
|
2196
2368
|
while (review?.meta?.noVerdict === true) {
|
|
@@ -2263,6 +2435,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2263
2435
|
const unverdicted = (g.meta?.skipped === true && !g.pass) || noVerdictReview;
|
|
2264
2436
|
journal.append("gate-result", t.id, {
|
|
2265
2437
|
gate: g.gate, ...(unverdicted ? {} : { pass: g.pass }), details: g.details,
|
|
2438
|
+
// R41: the protocol + effective lifecycle policy the gate MEASURED under, stamped by run-gates
|
|
2439
|
+
// for the worktree it ran in; absent only for a row that never went through the battery.
|
|
2440
|
+
...(g.meta?.verification !== undefined ? { verification: g.meta.verification } : {}),
|
|
2266
2441
|
...(gateSubject ? { commit: gateSubject.commit, attempt: gateSubject.attempt } : {}),
|
|
2267
2442
|
...(gateSubject?.replayMeasurement ? { replayMeasurement: true } : {}),
|
|
2268
2443
|
...(gateSubject?.replayedFromAttempt !== undefined ? { replayedFromAttempt: gateSubject.replayedFromAttempt } : {}),
|
|
@@ -2271,6 +2446,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2271
2446
|
// be able to tell "the suite found a defect" from "the runner never ran", and the merge
|
|
2272
2447
|
// predicate's reason for refusing has to be legible in the ledger it refused from.
|
|
2273
2448
|
...(g.meta?.infra === true ? { infra: true } : {}),
|
|
2449
|
+
// OBS-1030: a dirty-tree refusal names its litter, its culprit command and the recovery ref ON
|
|
2450
|
+
// THE ROW — the ledger is where an operator looks for the ref before a recheck, not the prose.
|
|
2451
|
+
...(g.meta?.dirtyWorktree === true ? {
|
|
2452
|
+
dirtyWorktree: true, dirtyPaths: g.meta.paths,
|
|
2453
|
+
...(typeof g.meta.culprit === "string" ? { culprit: g.meta.culprit } : {}),
|
|
2454
|
+
...(typeof g.meta.preservedRef === "string" ? { preservedRef: g.meta.preservedRef } : {}),
|
|
2455
|
+
} : {}),
|
|
2456
|
+
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
2457
|
+
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision"]
|
|
2458
|
+
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
2274
2459
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
2275
2460
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
2276
2461
|
// oracle's recorded verdict when any journal-backed reader reconstructs it.
|
|
@@ -2440,7 +2625,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2440
2625
|
// FROM THE JOURNAL, not from process memory: a park, a `resume`, or a daemon restart must not
|
|
2441
2626
|
// hand the selector a clean slate it did not earn, and the journal is the only state that
|
|
2442
2627
|
// survives all three (same read as upheldFeedbackByTask above).
|
|
2443
|
-
let testGateFailed = journal.read().some((e) => e.event === "gate-result" && e.taskId === t.id && e.data.gate === "test" && e.data.pass === false
|
|
2628
|
+
let testGateFailed = journal.read().some((e) => e.event === "gate-result" && e.taskId === t.id && e.data.gate === "test" && e.data.pass === false
|
|
2629
|
+
&& !(cfg.executionPolicy?.boundedInfrastructure && e.data.disposition === "infrastructure"));
|
|
2444
2630
|
let retryMode = "fresh";
|
|
2445
2631
|
let lastContextTokens; // v1.23 reset signal, including stalled/quota attempts
|
|
2446
2632
|
// v1.29: only a gate-failed attempt can seed same-session retry. The next attempt consumes this
|
|
@@ -2550,7 +2736,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2550
2736
|
const taskBase = await integrationHead(intWt);
|
|
2551
2737
|
const taskBranch = `${branch}--${t.id}`;
|
|
2552
2738
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
2553
|
-
|
|
2739
|
+
// OBS-1022: the ref `approve --recheck` validated (commits ahead of the base) is the tree the
|
|
2740
|
+
// battery gates — read from the pending approval row, never re-selected here, so the tree
|
|
2741
|
+
// approve counted and the tree the daemon gates are one and the same, checkout present or not.
|
|
2742
|
+
const preservedRef = recheck
|
|
2743
|
+
? [...journal.read()].reverse().find((e) => e.taskId === t.id && e.event === "task-approved" && e.data.release === RECHECK_RELEASE)?.data.recheckedRef
|
|
2744
|
+
: undefined;
|
|
2745
|
+
if (!existsSync(priorWt) && !preservedRef) {
|
|
2554
2746
|
if (satisfiedGate || recheck)
|
|
2555
2747
|
throw new Error(`${recheck ? "recheck" : `approved gate ${satisfiedGate}`} cannot resume: task worktree is missing`);
|
|
2556
2748
|
// Observed passes are an optimization, never authority: without the task worktree there is no
|
|
@@ -2563,9 +2755,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2563
2755
|
: satisfiedGate
|
|
2564
2756
|
? `approved gate ${satisfiedGate}`
|
|
2565
2757
|
: `recorded gates on ${replayedGates.commit.slice(0, 10)}`;
|
|
2566
|
-
const priorTaskTip = await gitHead(priorWt);
|
|
2567
|
-
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, priorWt);
|
|
2568
|
-
const commitsToCarry = await commitsAheadOf(taskBase, priorWt);
|
|
2758
|
+
const priorTaskTip = preservedRef ? (await shGit(`git rev-parse ${shq(preservedRef)}`, repoRoot)).stdout.trim() : await gitHead(priorWt);
|
|
2759
|
+
const priorTaskSubject = await gateCommitSubject(taskBase, priorTaskTip, preservedRef ? repoRoot : priorWt);
|
|
2760
|
+
const commitsToCarry = preservedRef ? await commitsAheadOfRef(taskBase, priorTaskTip, repoRoot) : await commitsAheadOf(taskBase, priorWt);
|
|
2569
2761
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
2570
2762
|
const carriedCommits = await cherryPickCommits(wt, commitsToCarry);
|
|
2571
2763
|
// Reuse is about the tree the gates will actually inspect. The integration tip may have moved
|
|
@@ -2604,16 +2796,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2604
2796
|
}
|
|
2605
2797
|
const workerEvent = [...journal.read()].reverse()
|
|
2606
2798
|
.find((e) => e.event === "worker-result" && e.taskId === t.id);
|
|
2607
|
-
|
|
2799
|
+
// OBS-1022: a task that FAILED before its harvest has no worker-result; a recheck judges the
|
|
2800
|
+
// committed tree on its own evidence, so the missing claim is recorded as exactly that.
|
|
2801
|
+
if (!workerEvent && !recheck)
|
|
2608
2802
|
throw new Error(`${resumeReason} cannot resume: worker result is missing`);
|
|
2609
|
-
const priorResult = {
|
|
2803
|
+
const priorResult = workerEvent ? {
|
|
2610
2804
|
ok: workerEvent.data.ok === true,
|
|
2611
2805
|
summary: typeof workerEvent.data.summary === "string" ? workerEvent.data.summary : "",
|
|
2612
2806
|
deviations: Array.isArray(workerEvent.data.deviations)
|
|
2613
2807
|
? workerEvent.data.deviations.filter((d) => typeof d === "string")
|
|
2614
2808
|
: [],
|
|
2615
2809
|
raw: "",
|
|
2616
|
-
};
|
|
2810
|
+
} : { ok: false, summary: "no worker result recorded (task failed before harvest); recheck gates the committed tree", deviations: [], raw: "" };
|
|
2617
2811
|
const parkedAuthor = recheck
|
|
2618
2812
|
? [...journal.read()].reverse().find((e) => e.event === "task-dispatch" && e.taskId === t.id)?.data.assignment
|
|
2619
2813
|
: undefined;
|
|
@@ -2679,7 +2873,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2679
2873
|
commit: replayedGates.commit, recorded: replayedCapacities, resolved: sessionCapacity,
|
|
2680
2874
|
});
|
|
2681
2875
|
}
|
|
2682
|
-
|
|
2876
|
+
// R41: the same rule for the verification protocol, measured for the RECREATED worktree
|
|
2877
|
+
// the re-run would gate in (its project npmrc included) — but a row with NO stamp is an OLDER
|
|
2878
|
+
// row from a different (pre-stamp) implementation, never "unchanged": it is re-run.
|
|
2879
|
+
const replayVerification = verificationProtocol(process.env, wt);
|
|
2880
|
+
const replayedVerifications = priorEvents
|
|
2881
|
+
.filter((e) => e.event === "gate-result" && e.taskId === t.id && e.data.commit === replayedGates.commit)
|
|
2882
|
+
.map((e) => e.data.verification);
|
|
2883
|
+
const sameProtocol = replayedVerifications.every((recorded) => sameVerification(recorded, replayVerification));
|
|
2884
|
+
if (!sameProtocol) {
|
|
2885
|
+
journal.append("gate-replay-verification-changed", t.id, {
|
|
2886
|
+
commit: replayedGates.commit, recorded: replayedVerifications, resolved: replayVerification,
|
|
2887
|
+
});
|
|
2888
|
+
}
|
|
2889
|
+
let reusable = sameWorld && sameProtocol && (exactCurrentCommit || canonicalCurrentCommit || recreatedLegacyCommit);
|
|
2683
2890
|
const reused = [];
|
|
2684
2891
|
const declaredGates = GATE_NAMES.filter((gate) => t.gates.includes(gate));
|
|
2685
2892
|
for (const gate of declaredGates) {
|
|
@@ -2695,6 +2902,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2695
2902
|
}
|
|
2696
2903
|
const resumedTask = { ...t, gates: remainingGates };
|
|
2697
2904
|
gateLoop: while (true) {
|
|
2905
|
+
executionSignal()?.throwIfAborted();
|
|
2698
2906
|
const gated = await gitHead(wt);
|
|
2699
2907
|
gateSubject = {
|
|
2700
2908
|
commit: await gateCommitSubject(taskBase, gated, wt),
|
|
@@ -2713,7 +2921,6 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2713
2921
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
2714
2922
|
collateral: collateral.get(t.id) ?? [],
|
|
2715
2923
|
// a recheck re-verifies a human's release: it never selects tests down, it runs the suite.
|
|
2716
|
-
pipeline: "v185",
|
|
2717
2924
|
via: cfg.visibility.llm === "pane"
|
|
2718
2925
|
? {
|
|
2719
2926
|
driver: trackedDriver,
|
|
@@ -2725,6 +2932,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2725
2932
|
: undefined,
|
|
2726
2933
|
excludeReviewers: badReviewers,
|
|
2727
2934
|
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
2935
|
+
// Leg-2 (OBS-1052): the run-scoped two-strike tally — without it retirement is inert and a
|
|
2936
|
+
// flaking seat is re-asked on every task. (carriedAuthors is held back: see SURGEON-LEG2-REPORT.)
|
|
2937
|
+
reviewNoVerdicts,
|
|
2728
2938
|
onGate: async (e) => {
|
|
2729
2939
|
if (e.phase === "start") {
|
|
2730
2940
|
notePhaseStart(e);
|
|
@@ -2746,7 +2956,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2746
2956
|
}
|
|
2747
2957
|
});
|
|
2748
2958
|
},
|
|
2749
|
-
}));
|
|
2959
|
+
}, false));
|
|
2750
2960
|
results.forEach(classifySignalOnlyTest);
|
|
2751
2961
|
if (pendingRechecks(journal.read()).has(t.id)) {
|
|
2752
2962
|
journal.append("recheck-battery", t.id, {
|
|
@@ -2764,6 +2974,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2764
2974
|
await park(t, `${infra.gate}: ${infra.details}`, "infra", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2765
2975
|
return;
|
|
2766
2976
|
}
|
|
2977
|
+
const capTrip = results.find(isDiffCapPark);
|
|
2978
|
+
if (capTrip) {
|
|
2979
|
+
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2980
|
+
return;
|
|
2981
|
+
}
|
|
2767
2982
|
// OBS-547: disposition FIRST, and through the same helper the ordinary attempt path uses. A
|
|
2768
2983
|
// crash between a journaled scope red and its classification must not turn a predicted red
|
|
2769
2984
|
// into a charged one just because a resume is what observed it.
|
|
@@ -2776,7 +2991,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2776
2991
|
// restored. Ask the journal which state this is, and take the classified dispatch's own
|
|
2777
2992
|
// assignment back with it.
|
|
2778
2993
|
const classified = journal.classifiedDispatch(t.id);
|
|
2779
|
-
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode))
|
|
2994
|
+
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
2780
2995
|
return;
|
|
2781
2996
|
gateFails++;
|
|
2782
2997
|
// Observed green gates are only measurements. If the resumed suffix is red, preserve that
|
|
@@ -2827,6 +3042,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2827
3042
|
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
2828
3043
|
let supersededWorkerSlot;
|
|
2829
3044
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
3045
|
+
executionSignal()?.throwIfAborted();
|
|
2830
3046
|
if (attempt !== providerDeathAttempt) {
|
|
2831
3047
|
providerDeathRequeues = 0;
|
|
2832
3048
|
providerDeathAttempt = attempt;
|
|
@@ -3021,7 +3237,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3021
3237
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
3022
3238
|
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
3023
3239
|
const recreating = existsSync(priorWt);
|
|
3240
|
+
// Carry the interrupted worker's snapshot, never incidental files created by later gates.
|
|
3241
|
+
const preserved = recreating ? reapedWorktreeRef : undefined;
|
|
3242
|
+
reapedWorktreeRef = undefined;
|
|
3024
3243
|
const commitsToCarry = recreating ? await commitsAheadOf(taskBase, priorWt) : [];
|
|
3244
|
+
if (preserved)
|
|
3245
|
+
commitsToCarry.push(preserved);
|
|
3025
3246
|
const wt = await recreateTaskWorktree(taskBranch, taskBase, priorWt);
|
|
3026
3247
|
// OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
|
|
3027
3248
|
// cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
|
|
@@ -3090,14 +3311,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3090
3311
|
if (repairFindings) {
|
|
3091
3312
|
const raw = await shGit(`git diff ${shq(taskBase)}..HEAD`, wt);
|
|
3092
3313
|
const cap = cfg.gates.diffCap ?? DEFAULT_DIFF_CAP; // same fallback the measuring gates use
|
|
3093
|
-
|
|
3094
|
-
|
|
3314
|
+
// OBS-1034: a capped brief says WHAT it cut — the files whose hunks start at or past the cap
|
|
3315
|
+
// (the file the cut lands inside is partial too) — on the row and in the worker's prompt.
|
|
3316
|
+
const rawBytes = Buffer.from(raw.stdout, "utf8"); // the cap is BYTES: measure and cut in UTF-8, not code units
|
|
3317
|
+
const capped = rawBytes.length > cap;
|
|
3318
|
+
const droppedFiles = capped ? repairDiffFilesCut(rawBytes, cap, [...await diffPaths(taskBase, wt)]) : [];
|
|
3319
|
+
const diff = capped
|
|
3320
|
+
? `${new TextDecoder().decode(rawBytes.subarray(0, cap))}\n… diff truncated at gates.diffCap (${cap} bytes)`
|
|
3321
|
+
+ `\nPARTIAL DIFF: the diff above is incomplete; files cut by the cap: ${droppedFiles.join(", ") || "(unknown)"}`
|
|
3095
3322
|
: raw.stdout;
|
|
3096
3323
|
const brief = repairBrief(repairFindings, diff, taskBase);
|
|
3097
3324
|
// anything the live brief holds beyond the journaled findings (a consult's guidance) is kept:
|
|
3098
3325
|
// a repair adds the diff and the fix-only contract, it never subtracts what was already known.
|
|
3099
3326
|
feedback = feedback && !brief.includes(feedback) ? `${brief}\n\n${feedback}` : brief;
|
|
3100
|
-
journal.append("repair-dispatch", t.id, { diffBytes: diff
|
|
3327
|
+
journal.append("repair-dispatch", t.id, { diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
3101
3328
|
}
|
|
3102
3329
|
if (feedback || priorNamed.length > 0) {
|
|
3103
3330
|
feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
|
|
@@ -3159,7 +3386,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3159
3386
|
? (hasSeed ? ":" : icmd)
|
|
3160
3387
|
: adapter.invoke(t, wt, assignment, { promptFile }).command;
|
|
3161
3388
|
const dispatchScript = promptFile.replace(/\.md$/, ".sh");
|
|
3389
|
+
// Approval can reset the attempt counter while the old pane remains retained.
|
|
3390
|
+
// Its ownership claim must survive a new engagement reusing the script path.
|
|
3391
|
+
const groupFile = `${dispatchScript}.${nonce}.pgid`;
|
|
3392
|
+
workerOwners.set(slot, { taskId: t.id, attempt, groupFile });
|
|
3162
3393
|
writeFileSync(dispatchScript, [
|
|
3394
|
+
// A driver may launch inside the daemon's group. That group is never worker-owned.
|
|
3395
|
+
`worker_pgid=$(ps -o pgid= -p $$ 2>/dev/null); daemon_pgid=$(ps -o pgid= -p ${process.pid} 2>/dev/null)`,
|
|
3396
|
+
`if [ -n "$worker_pgid" ] && [ -n "$daemon_pgid" ] && [ "$worker_pgid" != "$daemon_pgid" ]; then printf '%s\\n' "$worker_pgid" > ${shq(groupFile)}; fi`,
|
|
3163
3397
|
"export BASH_SILENCE_DEPRECATION_WARNING=1",
|
|
3164
3398
|
bannerShell(),
|
|
3165
3399
|
workerCmd,
|
|
@@ -3226,7 +3460,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3226
3460
|
}
|
|
3227
3461
|
if (hasSeed || cpuAccountant !== undefined)
|
|
3228
3462
|
return;
|
|
3229
|
-
cpuAccountant = new WorkerTreeCpuAccountant(dispatchScript, wt);
|
|
3463
|
+
cpuAccountant = new WorkerTreeCpuAccountant(dispatchScript, wt, () => readOwnedProcessGroup(groupFile));
|
|
3230
3464
|
await cpuAccountant.start();
|
|
3231
3465
|
};
|
|
3232
3466
|
const readCpuLeg = () => {
|
|
@@ -3320,6 +3554,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3320
3554
|
// window that could not have killed anything and briefed the next worker to fix a stall that
|
|
3321
3555
|
// never happened. Fourth conflation in the taxonomy OBS-53 opened.
|
|
3322
3556
|
let deadChannelKilled = false;
|
|
3557
|
+
let hardTimedOut = false;
|
|
3558
|
+
let quotaBannerKilled = false;
|
|
3559
|
+
let driverProbeFailed = false;
|
|
3560
|
+
let heldLegs = [];
|
|
3561
|
+
const noteDriverUnreadable = (error) => {
|
|
3562
|
+
driverProbeFailed = true;
|
|
3563
|
+
heldLegs = ["quota:driver-unreadable", "stall:driver-unreadable"];
|
|
3564
|
+
journal.append("contact-unreadable", t.id, { slot: slot.name, attempt, source: "driver", concludes: false,
|
|
3565
|
+
error: error instanceof Error ? error.message : String(error) });
|
|
3566
|
+
};
|
|
3323
3567
|
// T2 review: print mode's "the exit marker appeared". Kept apart from `finished` (the
|
|
3324
3568
|
// trailer) but still needed by the keepPanes decision below, whose contract is about a
|
|
3325
3569
|
// subprocess tree that REACHED its exit marker, not about what the worker claimed.
|
|
@@ -3337,7 +3581,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3337
3581
|
const trailerFrames = interactive && adapter.busyFrameMarkers
|
|
3338
3582
|
? new SettledTrailerTracker(adapter.busyFrameMarkers) : undefined;
|
|
3339
3583
|
const sampleTrailer = (frame) => {
|
|
3340
|
-
const
|
|
3584
|
+
const parsed = adapter.parse(frame, nonce);
|
|
3585
|
+
const trailer = new RegExp(trailerPattern(nonce)).test(frame)
|
|
3586
|
+
&& parsed.summary !== NO_TRAILER_SUMMARY && parsed.summary !== UNPARSEABLE_TRAILER_SUMMARY;
|
|
3341
3587
|
return trailerFrames ? trailerFrames.sample(frame, trailer) : trailer;
|
|
3342
3588
|
};
|
|
3343
3589
|
let seedResult;
|
|
@@ -3351,11 +3597,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3351
3597
|
// DeliveryReadinessError, and that catch continues or returns with no result to read, so a
|
|
3352
3598
|
// latch built from one would start false with the key already sent, and the next modal on
|
|
3353
3599
|
// this slot would get the free Enter this contract exists to prevent.
|
|
3354
|
-
let trustAnswered =
|
|
3600
|
+
let trustAnswered = trustAnsweredSlots.has(slot.id);
|
|
3355
3601
|
const noteSeedTrustAnswered = () => {
|
|
3356
3602
|
if (trustAnswered)
|
|
3357
3603
|
return;
|
|
3358
3604
|
trustAnswered = true;
|
|
3605
|
+
trustAnsweredSlots.add(slot.id);
|
|
3359
3606
|
// Same audit line the loop below writes, so a live run shows the answer wherever it happened.
|
|
3360
3607
|
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id, phase: "seed" });
|
|
3361
3608
|
};
|
|
@@ -3409,7 +3656,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3409
3656
|
// failure rather than allowed to race the trailer wait.
|
|
3410
3657
|
try {
|
|
3411
3658
|
seedResult = await runInteractiveSeed({
|
|
3412
|
-
driver
|
|
3659
|
+
driver: trustAnswered ? {
|
|
3660
|
+
run: driver.run.bind(driver), waitOutput: driver.waitOutput.bind(driver), read: driver.read.bind(driver),
|
|
3661
|
+
} : driver,
|
|
3662
|
+
slot, adapter, assignment, promptFile, taskTimeoutMinutes,
|
|
3413
3663
|
onTrustAnswered: noteSeedTrustAnswered,
|
|
3414
3664
|
});
|
|
3415
3665
|
}
|
|
@@ -3456,6 +3706,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3456
3706
|
// pageRepeatMs elapses, so an operator who missed the first one is paged again.
|
|
3457
3707
|
let lastPagedStatus;
|
|
3458
3708
|
let lastPagedAt = 0;
|
|
3709
|
+
let pagesSuppressed = 0; // OBS-1006: pageable slices swallowed by the cadence since the last row
|
|
3459
3710
|
// T1 review: worker-status is journaled on CHANGE, not per slice — every journal append
|
|
3460
3711
|
// is narrated to the run's live surface (cli/commands/run.ts) and feeds activity.ts's
|
|
3461
3712
|
// `now:` cell, so a per-slice append wrote a status line per worker per ~30s slice and
|
|
@@ -3481,9 +3732,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3481
3732
|
let cpuHeld = false; // likewise for the CPU leg's stand-down (OBS-548)
|
|
3482
3733
|
let paneReadHeld = false;
|
|
3483
3734
|
let paneStatusHeld = false;
|
|
3484
|
-
|
|
3735
|
+
const hardDeadline = attemptStart + (attemptHardTimeoutMs ?? stallWindowMs * 4);
|
|
3736
|
+
await armCpuLeg(true);
|
|
3737
|
+
while (Date.now() < hardDeadline) {
|
|
3738
|
+
if (termSignal)
|
|
3739
|
+
throw new Error(`terminated by ${termSignal}`);
|
|
3740
|
+
executionSignal()?.throwIfAborted();
|
|
3741
|
+
driverProbeFailed = false;
|
|
3485
3742
|
const sliceStart = Date.now();
|
|
3486
|
-
const remaining = stallWindowMs - (sliceStart - lastProgressAt);
|
|
3743
|
+
const remaining = Math.max(100, Math.min(stallWindowMs - (sliceStart - lastProgressAt), hardDeadline - sliceStart));
|
|
3487
3744
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3488
3745
|
const startupLeft = workerLaunchedAt === undefined
|
|
3489
3746
|
? 0
|
|
@@ -3500,11 +3757,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3500
3757
|
// otherwise cost a minute per pair of CPU samples. At the shipped 5m gate this leaves
|
|
3501
3758
|
// every slice before the gate exactly as long as it already was.
|
|
3502
3759
|
slice = Math.min(slice, harvestSliceMs(sliceStart - lastProgressAt));
|
|
3503
|
-
|
|
3760
|
+
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
3761
|
+
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true }).catch((error) => { noteDriverUnreadable(error); return false; })) {
|
|
3504
3762
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
3505
3763
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
3506
3764
|
// parseable trailer or a digit-suffixed exit marker in the harvest is completion.
|
|
3507
|
-
output = await driver.read(slot, PANE_READ_ROWS); // TUI transcripts carry chrome — read deeper than print's 500
|
|
3765
|
+
output = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; }); // TUI transcripts carry chrome — read deeper than print's 500
|
|
3508
3766
|
finished = sampleTrailer(output);
|
|
3509
3767
|
const exit = exitRe.exec(output);
|
|
3510
3768
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3513,6 +3771,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3513
3771
|
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
3514
3772
|
break;
|
|
3515
3773
|
}
|
|
3774
|
+
if (adapter.parse(output, nonce).summary === UNPARSEABLE_TRAILER_SUMMARY)
|
|
3775
|
+
break;
|
|
3516
3776
|
if (trailerFrames && new RegExp(trailerPattern(nonce)).test(output)) {
|
|
3517
3777
|
// A matching busy frame is still a live turn. Preserve the rolling progress budget.
|
|
3518
3778
|
await sampleContext();
|
|
@@ -3529,19 +3789,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3529
3789
|
paneText = await driver.read(slot, PANE_READ_ROWS);
|
|
3530
3790
|
}
|
|
3531
3791
|
catch (error) {
|
|
3532
|
-
|
|
3533
|
-
// sees a pane. The outer attempt finally owns accountant cleanup on that path. Only
|
|
3534
|
-
// an undetectable status makes the read failure relevant to the death detector, and
|
|
3535
|
-
// that genuinely unmeasurable pair fails open to the rolling timeout.
|
|
3536
|
-
let paneUndetectable = true;
|
|
3537
|
-
try {
|
|
3538
|
-
paneUndetectable = await driver.status(slot) === "unknown";
|
|
3539
|
-
}
|
|
3540
|
-
catch {
|
|
3541
|
-
// Two unreadable pane probes are still unmeasurable, never proof of death.
|
|
3542
|
-
}
|
|
3543
|
-
if (!paneUndetectable)
|
|
3544
|
-
throw error;
|
|
3792
|
+
noteDriverUnreadable(error);
|
|
3545
3793
|
if (!paneReadHeld) {
|
|
3546
3794
|
paneReadHeld = true;
|
|
3547
3795
|
noteWorkerLiveness("worker-dead-held", {
|
|
@@ -3549,14 +3797,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3549
3797
|
error: error instanceof Error ? error.message : String(error),
|
|
3550
3798
|
});
|
|
3551
3799
|
}
|
|
3552
|
-
await armCpuLeg(
|
|
3800
|
+
await armCpuLeg(true);
|
|
3553
3801
|
const spent = Date.now() - sliceStart;
|
|
3554
3802
|
if (spent < slice)
|
|
3555
3803
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
3556
3804
|
continue;
|
|
3557
3805
|
}
|
|
3558
3806
|
// waitOutput is only a wake hint; a driver may miss a trailer already painted.
|
|
3559
|
-
if (
|
|
3807
|
+
if (sampleTrailer(paneText)) {
|
|
3560
3808
|
output = paneText;
|
|
3561
3809
|
finished = true;
|
|
3562
3810
|
const exit = exitRe.exec(output);
|
|
@@ -3565,6 +3813,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3565
3813
|
await sampleContext();
|
|
3566
3814
|
break;
|
|
3567
3815
|
}
|
|
3816
|
+
if (adapter.parse(paneText, nonce).summary === UNPARSEABLE_TRAILER_SUMMARY) {
|
|
3817
|
+
output = paneText;
|
|
3818
|
+
break;
|
|
3819
|
+
}
|
|
3820
|
+
if (driverProbeFailed) {
|
|
3821
|
+
const spent = Date.now() - sliceStart;
|
|
3822
|
+
if (spent < slice)
|
|
3823
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(slice - spent, 1_000)));
|
|
3824
|
+
continue;
|
|
3825
|
+
}
|
|
3568
3826
|
if (paneText.length > 0)
|
|
3569
3827
|
everHadOutput = true;
|
|
3570
3828
|
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
@@ -3623,14 +3881,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3623
3881
|
// filtered by identity, never by novelty — a banner already on screen at launch
|
|
3624
3882
|
// classifies exactly like one printed mid-attempt (T1 review: a novelty baseline
|
|
3625
3883
|
// exculpated the launch-throttle case forever).
|
|
3626
|
-
|
|
3884
|
+
const bannerMatch = QUOTA_RE.exec(stallSnapshotBannerRows(paneText));
|
|
3885
|
+
if (bannerMatch)
|
|
3627
3886
|
quotaStreak++;
|
|
3628
3887
|
else
|
|
3629
3888
|
quotaStreak = 0;
|
|
3630
|
-
if (
|
|
3889
|
+
if (bannerMatch && !stallProgress.rowSignalSaturated && !nudgeFailed
|
|
3890
|
+
&& !(driver.nudge && NUDGEABLE_ADAPTERS.has(adapter.id) && (!nudged || nudgeDeadline !== undefined))
|
|
3891
|
+
&& readCpuLeg().state === "flat"
|
|
3892
|
+
&& quotaStreak >= 2 && sliceNow - lastProgressAt >= quotaBannerSilentMs) {
|
|
3631
3893
|
// no `output =` here: the post-loop no-trailer tail re-reads the pane anyway, so an
|
|
3632
3894
|
// assignment would only split the classification read from the verdict read.
|
|
3633
|
-
|
|
3895
|
+
quotaBannerKilled = true;
|
|
3896
|
+
journal.append("quota-banner", t.id, { slot: slot.name, attempt, silentMs: sliceNow - lastProgressAt, matched: bannerMatch[0], excerpt: bannerMatch.input, regex: QUOTA_RE.source });
|
|
3634
3897
|
break;
|
|
3635
3898
|
}
|
|
3636
3899
|
// T1 (OBS-262): the `paged` latch is deleted — status is sampled EVERY slice (and
|
|
@@ -3643,6 +3906,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3643
3906
|
st = await driver.status(slot);
|
|
3644
3907
|
}
|
|
3645
3908
|
catch (error) {
|
|
3909
|
+
noteDriverUnreadable(error);
|
|
3646
3910
|
if (!paneStatusHeld) {
|
|
3647
3911
|
paneStatusHeld = true;
|
|
3648
3912
|
noteWorkerLiveness("worker-dead-held", {
|
|
@@ -3650,7 +3914,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3650
3914
|
error: error instanceof Error ? error.message : String(error),
|
|
3651
3915
|
});
|
|
3652
3916
|
}
|
|
3653
|
-
await armCpuLeg(
|
|
3917
|
+
await armCpuLeg(true);
|
|
3654
3918
|
const spent = Date.now() - sliceStart;
|
|
3655
3919
|
if (spent < slice)
|
|
3656
3920
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
@@ -3668,6 +3932,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3668
3932
|
const paneText = await driver.read(slot, 80);
|
|
3669
3933
|
if (matchesTrustDialog(paneText, adapter.trustDialog)) {
|
|
3670
3934
|
trustAnswered = true;
|
|
3935
|
+
trustAnsweredSlots.add(slot.id);
|
|
3671
3936
|
// v1.25 T1: audit trail for live runs — prove the dialog appeared and was answered.
|
|
3672
3937
|
// Latch + sendKey + no-page continue stay byte-identical; this append is additive only.
|
|
3673
3938
|
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id });
|
|
@@ -3793,20 +4058,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3793
4058
|
// lastProgressAt. Either one advancing is output growth.
|
|
3794
4059
|
const lastOutputGrowthAt = Math.max(stallProgress.lastRowGrowthAt ?? 0, lastProgressAt);
|
|
3795
4060
|
// Everything the fast-kill can decide from the CHANNEL, evaluated before the CPU leg is
|
|
3796
|
-
// asked for
|
|
3797
|
-
// slice must not pay for it. The CPU reading is consulted at the kill itself, below.
|
|
4061
|
+
// asked for a conclusion. The shared CPU reading is consulted at the kill below.
|
|
3798
4062
|
const fastKillEligible = !nudgePending && !nudgeFailed
|
|
3799
4063
|
&& sliceNow - lastOutputGrowthAt >= deadChannelFastKillMs
|
|
3800
4064
|
&& worktreeSinceLaunch === "unchanged";
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
// slice from BOTH windows. Arming per-reader let the harvest's own "not silent enough
|
|
3805
|
-
// yet" branch stop the sampler the fast-kill had just started, and a restarted
|
|
3806
|
-
// accountant re-bases its monotonic total — the leg could then never read flat. The
|
|
3807
|
-
// accountant also carries the readers' nudge holds: when neither reader may conclude,
|
|
3808
|
-
// there is no reason to fork its 10 Hz process snapshots through the remaining window.
|
|
3809
|
-
await armCpuLeg(harvestCpuEligible || fastKillEligible);
|
|
4065
|
+
// All silent conclusions share the attempt's continuous CPU ledger. A nudge
|
|
4066
|
+
// hold or a saturated pane must not reset its history before the stall boundary.
|
|
4067
|
+
await armCpuLeg(true);
|
|
3810
4068
|
// T2 (OBS-264): the liveness triad CONCLUDES the wait on finished work — commits ahead
|
|
3811
4069
|
// of the task base (this attempt's own AND any carried forward — see the eligibility
|
|
3812
4070
|
// comment at the dispatch site), a flat worker-tree CPU delta, and >= harvestSilentMs
|
|
@@ -3864,7 +4122,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3864
4122
|
// concluded here". A live CPU delta is a live worker, by construction; an UNMEASURABLE
|
|
3865
4123
|
// reading keeps the fail-open contract the probe already states (undefined is never
|
|
3866
4124
|
// evidence a worker stopped), so on a host whose `ps` cannot be read the fast-kill stands
|
|
3867
|
-
// down and the
|
|
4125
|
+
// down and the fixed attempt ceiling bounds the hold.
|
|
3868
4126
|
if (fastKillEligible) {
|
|
3869
4127
|
const leg = readCpuLeg();
|
|
3870
4128
|
if (leg.state === "flat") {
|
|
@@ -3883,6 +4141,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3883
4141
|
});
|
|
3884
4142
|
}
|
|
3885
4143
|
}
|
|
4144
|
+
const stallCpu = readCpuLeg();
|
|
4145
|
+
heldLegs = [
|
|
4146
|
+
...(stallProgress.rowSignalSaturated ? ["quota:row-signal-saturated", "stall:row-signal-saturated"] : []),
|
|
4147
|
+
...(stallCpu.state !== "flat" ? [`stall:cpu-${stallCpu.state}`] : []),
|
|
4148
|
+
...(nudgeFailed || nudgePending ? ["quota:nudge", "stall:nudge"] : []),
|
|
4149
|
+
...(worktreeSincePriorRead === "unreadable" ? ["stall:worktree-unreadable"] : []),
|
|
4150
|
+
];
|
|
4151
|
+
if (sliceNow - lastProgressAt >= stallWindowMs && heldLegs.length === 0) {
|
|
4152
|
+
timedOut = true;
|
|
4153
|
+
break;
|
|
4154
|
+
}
|
|
3886
4155
|
if (nudgeable && !nudged && sliceNow - lastProgressAt >= nudgeAfterSilentMs) {
|
|
3887
4156
|
nudged = true;
|
|
3888
4157
|
// T1 review: a false return is a driver-delivery outcome (missing pin, readiness
|
|
@@ -3890,16 +4159,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3890
4159
|
// one failure is a flake class, retried once in-slice after a short settle. Only a
|
|
3891
4160
|
// failed RETRY condemns the channel. Both failures happen inside this slice, so the
|
|
3892
4161
|
// latch stays immediate and exactly one failure is journaled per attempt.
|
|
3893
|
-
let delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE);
|
|
4162
|
+
let delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE).catch((error) => { noteDriverUnreadable(error); return false; });
|
|
3894
4163
|
if (!delivered) {
|
|
3895
4164
|
await new Promise((r) => setTimeout(r, NUDGE_REDELIVER_MS));
|
|
3896
|
-
delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE);
|
|
4165
|
+
delivered = await driver.nudge(slot, WORKER_NUDGE_MESSAGE).catch((error) => { noteDriverUnreadable(error); return false; });
|
|
3897
4166
|
}
|
|
3898
4167
|
if (delivered) {
|
|
3899
4168
|
// absorb the nudge's own echo BEFORE arming the grace timer — post-nudge progress
|
|
3900
4169
|
// is measured against this baseline, not against the echo.
|
|
3901
|
-
const echo = await driver.read(slot, PANE_READ_ROWS);
|
|
3902
|
-
|
|
4170
|
+
const echo = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4171
|
+
if (echo !== undefined)
|
|
4172
|
+
stallProgress.observe({ paneText: echo, contextTokens });
|
|
3903
4173
|
nudgeDeadline = Date.now() + workerNudgeGraceMs;
|
|
3904
4174
|
journal.append("worker-nudge", t.id, { slot: slot.name, attempt });
|
|
3905
4175
|
}
|
|
@@ -3907,15 +4177,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3907
4177
|
// Delivery failure is journal evidence about the daemon's contact attempt only.
|
|
3908
4178
|
// The fast-kill's worktree clause remains wholly filesystem/git-derived above.
|
|
3909
4179
|
nudgeFailed = true;
|
|
3910
|
-
journal.append("worker-nudge-failed", t.id, { slot: slot.name, attempt });
|
|
4180
|
+
journal.append("worker-nudge-failed", t.id, { slot: slot.name, attempt, cause: "delivery-refused-or-unconfirmed" });
|
|
3911
4181
|
}
|
|
3912
4182
|
}
|
|
3913
|
-
if (nudged && nudgeDeadline !== undefined && sliceNow >= nudgeDeadline && sliceNow - lastProgressAt >= workerNudgeGraceMs) {
|
|
4183
|
+
if (!stallProgress.rowSignalSaturated && readCpuLeg().state === "flat" && nudged && nudgeDeadline !== undefined && sliceNow >= nudgeDeadline && sliceNow - lastProgressAt >= workerNudgeGraceMs) {
|
|
3914
4184
|
// grace spent, still no post-nudge progress: re-harvest once (the trailer may have
|
|
3915
4185
|
// landed between polls), then conclude the wait as a stall NOW — the consult sees the
|
|
3916
4186
|
// un-answered nudge instead of the remainder of the window.
|
|
4187
|
+
const finalPane = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return undefined; });
|
|
4188
|
+
if (finalPane === undefined)
|
|
4189
|
+
continue;
|
|
3917
4190
|
nudgeDeadline = undefined;
|
|
3918
|
-
output =
|
|
4191
|
+
output = finalPane;
|
|
3919
4192
|
finished = sampleTrailer(output);
|
|
3920
4193
|
const exit = exitRe.exec(output);
|
|
3921
4194
|
if (finished || (exit && !(trailerFrames && new RegExp(trailerPattern(nonce)).test(output)))) {
|
|
@@ -3925,8 +4198,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3925
4198
|
break;
|
|
3926
4199
|
}
|
|
3927
4200
|
journal.append("worker-nudge-expired", t.id, { slot: slot.name, attempt, graceMs: workerNudgeGraceMs });
|
|
3928
|
-
|
|
3929
|
-
|
|
4201
|
+
timedOut = true;
|
|
4202
|
+
break;
|
|
3930
4203
|
}
|
|
3931
4204
|
// Unlatched page (T1): the page DECISION fires and is journaled every slice the
|
|
3932
4205
|
// operator is the right actor — i.e. the nudge path doesn't own it (non-allowlisted
|
|
@@ -3939,23 +4212,32 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3939
4212
|
// once pageRepeatMs elapses, so a missed first page is not the last one.
|
|
3940
4213
|
const pageable = (st === "blocked" || st === "idle")
|
|
3941
4214
|
&& (!nudgePending || nudgeFailed);
|
|
4215
|
+
// OBS-1006: the row is appended INSIDE the delivery guard — one row per page delivered, never
|
|
4216
|
+
// one per slice — and it carries how many pageable slices were suppressed since the last one.
|
|
3942
4217
|
if (pageable) {
|
|
3943
|
-
journal.append("operator-page", t.id, { slot: slot.name, attempt, status: st });
|
|
3944
4218
|
if (st !== lastPagedStatus || sliceNow - lastPagedAt >= pageRepeatMs) {
|
|
4219
|
+
journal.append("operator-page", t.id, { slot: slot.name, attempt, status: st, suppressed: pagesSuppressed });
|
|
4220
|
+
pagesSuppressed = 0;
|
|
3945
4221
|
lastPagedStatus = st;
|
|
3946
4222
|
lastPagedAt = sliceNow;
|
|
3947
4223
|
const why = st === "blocked" ? "is blocked on a prompt — approve in its pane" : "looks idle without finishing — check its pane";
|
|
3948
4224
|
await driver.notify(`tickmarkr ${runId}: ${slot.name} ${why}`, { tier: "attention" });
|
|
3949
4225
|
}
|
|
4226
|
+
else {
|
|
4227
|
+
pagesSuppressed++;
|
|
4228
|
+
}
|
|
3950
4229
|
}
|
|
3951
4230
|
// a dead pane or a false-positive marker display returns fast — sleep the unspent slice, never hot-spin
|
|
3952
4231
|
const spent = Date.now() - sliceStart;
|
|
3953
4232
|
if (spent < slice)
|
|
3954
4233
|
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
3955
4234
|
}
|
|
3956
|
-
if (!finished && exitCode === null) {
|
|
4235
|
+
if (!finished && exitCode === null && adapter.parse(output, nonce).summary !== UNPARSEABLE_TRAILER_SUMMARY) {
|
|
3957
4236
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
3958
|
-
|
|
4237
|
+
hardTimedOut = Date.now() >= hardDeadline;
|
|
4238
|
+
timedOut ||= hardTimedOut;
|
|
4239
|
+
if (hardTimedOut)
|
|
4240
|
+
journal.append("worker-hard-timeout", t.id, { slot: slot.name, attempt, heldLegs });
|
|
3959
4241
|
try {
|
|
3960
4242
|
output = await driver.read(slot, PANE_READ_ROWS);
|
|
3961
4243
|
}
|
|
@@ -3963,14 +4245,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3963
4245
|
// The poll loop already recorded the unreadable pane. Retain the last readable bytes
|
|
3964
4246
|
// so this ambiguous path still reaches the ordinary timeout/consult backstop.
|
|
3965
4247
|
}
|
|
3966
|
-
finished =
|
|
4248
|
+
finished = sampleTrailer(output);
|
|
3967
4249
|
const exit = exitRe.exec(output);
|
|
3968
4250
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3969
4251
|
processExited = exit !== null;
|
|
3970
4252
|
}
|
|
3971
4253
|
if (finished && !trailerFrames) {
|
|
3972
|
-
await driver.waitAgentStatus(slot, "idle", 5_000);
|
|
3973
|
-
|
|
4254
|
+
await driver.waitAgentStatus(slot, "idle", 5_000).catch(noteDriverUnreadable);
|
|
4255
|
+
const settled = await driver.read(slot, PANE_READ_ROWS).catch((error) => { noteDriverUnreadable(error); return output; });
|
|
4256
|
+
if (sampleTrailer(settled))
|
|
4257
|
+
output = settled;
|
|
3974
4258
|
}
|
|
3975
4259
|
// T5 / OBS-111: an interactive harvest can race the TUI's final paint. When the pane
|
|
3976
4260
|
// contains the nonce token but the JSON hasn't balanced yet, settle and re-read through
|
|
@@ -4028,6 +4312,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4028
4312
|
// there is the trailer regex; the exit marker only sets exitCode), and the cause taxonomy
|
|
4029
4313
|
// already names this shape "clean-exit-no-trailer" — unreachable in print mode until now.
|
|
4030
4314
|
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
4315
|
+
executionSignal()?.throwIfAborted();
|
|
4031
4316
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
4032
4317
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
4033
4318
|
const startupLeft = workerLaunchedAt === undefined
|
|
@@ -4042,6 +4327,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4042
4327
|
}
|
|
4043
4328
|
// T2 (OBS-264): same probe cadence the interactive loop uses — see harvestSliceMs.
|
|
4044
4329
|
slice = Math.min(slice, harvestSliceMs(Date.now() - lastProgressAt));
|
|
4330
|
+
slice = Math.min(slice, remainingExecutionMs() ?? slice);
|
|
4045
4331
|
if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
4046
4332
|
processExited = true;
|
|
4047
4333
|
break;
|
|
@@ -4089,7 +4375,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4089
4375
|
// interactive attempts to the same exit marker before close and the post-hoc usage disk read
|
|
4090
4376
|
// so a writer never races the reader (real CLIs can flush usage asynchronously after the trailer).
|
|
4091
4377
|
if (interactive && finished && !exitRe.test(output)) {
|
|
4092
|
-
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true });
|
|
4378
|
+
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true }).catch(noteDriverUnreadable);
|
|
4093
4379
|
}
|
|
4094
4380
|
// Free availability reroutes deliberately reuse the charged-attempt ordinal. Stream artifacts
|
|
4095
4381
|
// cannot: every dispatch owns bytes that must remain independently recoverable, so derive the
|
|
@@ -4126,8 +4412,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4126
4412
|
try {
|
|
4127
4413
|
await closeSlot(slot);
|
|
4128
4414
|
if (!workerFinished) {
|
|
4415
|
+
const ref = await preserveWorktree(wt);
|
|
4416
|
+
if (ref) {
|
|
4417
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
4418
|
+
reapedWorktreeRef = ref;
|
|
4419
|
+
}
|
|
4129
4420
|
journal.append("worker-reaped-before-harvest", t.id, {
|
|
4130
|
-
slot: slot.name, attempt, cause: workerCause ?? "stall-timeout",
|
|
4421
|
+
slot: slot.name, attempt, cause: hardTimedOut ? "hard-timeout" : workerCause ?? "stall-timeout",
|
|
4422
|
+
...reapReports.get(slot),
|
|
4423
|
+
...(hardTimedOut ? { heldLegs } : {}),
|
|
4131
4424
|
...(startupEvidence ? { evidence: startupEvidence } : {}),
|
|
4132
4425
|
});
|
|
4133
4426
|
}
|
|
@@ -4175,6 +4468,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4175
4468
|
// commits survive via the existing commitsToCarry/cherryPickCommits carry-forward.
|
|
4176
4469
|
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4177
4470
|
return;
|
|
4471
|
+
if (!workerFinished && !processExited && (timedOut || deadChannelKilled || quotaBannerKilled)) {
|
|
4472
|
+
const seat = channelKey(assignment);
|
|
4473
|
+
stallReaps = stallSeat === seat ? stallReaps + 1 : 1;
|
|
4474
|
+
stallSeat = seat;
|
|
4475
|
+
if (stallReaps >= 2) {
|
|
4476
|
+
const ref = await preserveWorktree(wt);
|
|
4477
|
+
if (ref)
|
|
4478
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
4479
|
+
await park(t, `two consecutive stall reaps without a gate on seat ${seat}`, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { seat, stallReaps });
|
|
4480
|
+
return;
|
|
4481
|
+
}
|
|
4482
|
+
}
|
|
4483
|
+
else {
|
|
4484
|
+
stallReaps = 0;
|
|
4485
|
+
stallSeat = undefined;
|
|
4486
|
+
}
|
|
4178
4487
|
const preHarvestResult = result;
|
|
4179
4488
|
// T2 (OBS-264): recognize committed no-trailer work BEFORE any no-trailer streak, provider,
|
|
4180
4489
|
// quota or dead-channel routing. Gates never trusted the trailer, so this successful synthesis
|
|
@@ -4366,6 +4675,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4366
4675
|
}
|
|
4367
4676
|
// Provisioning can mutate the shared dependency tree the pristine capture is still using.
|
|
4368
4677
|
await waitForBaseline(t.id);
|
|
4678
|
+
stallReaps = 0;
|
|
4679
|
+
stallSeat = undefined;
|
|
4369
4680
|
graph = setStatus(graph, t.id, "gated");
|
|
4370
4681
|
saveGraph(repoRoot, graph);
|
|
4371
4682
|
// OBS-47: re-assert the node_modules link BEFORE gates run on any attempt. A worker may have
|
|
@@ -4430,6 +4741,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4430
4741
|
let results = [];
|
|
4431
4742
|
let commits = [];
|
|
4432
4743
|
gateLoop: while (true) {
|
|
4744
|
+
executionSignal()?.throwIfAborted();
|
|
4433
4745
|
const gated = await gitHead(wt);
|
|
4434
4746
|
gateSubject = { commit: await gateCommitSubject(taskBase, gated, wt), attempt };
|
|
4435
4747
|
await trackedDriver.project?.(t.id, "in-review");
|
|
@@ -4469,7 +4781,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4469
4781
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
4470
4782
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4471
4783
|
collateral: collateral.get(t.id) ?? [],
|
|
4472
|
-
|
|
4784
|
+
selectTests: !testGateFailed,
|
|
4473
4785
|
via: cfg.visibility.llm === "pane"
|
|
4474
4786
|
? {
|
|
4475
4787
|
driver: trackedDriver,
|
|
@@ -4488,13 +4800,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4488
4800
|
: undefined,
|
|
4489
4801
|
excludeReviewers: badReviewers,
|
|
4490
4802
|
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
4803
|
+
reviewNoVerdicts, // Leg-2 (OBS-1052): see the recheck site above
|
|
4491
4804
|
onGate,
|
|
4492
4805
|
})));
|
|
4493
4806
|
}
|
|
4494
4807
|
results.forEach(classifySignalOnlyTest);
|
|
4495
4808
|
graph = addEvidence(graph, t.id, { commits, gateResults: results, artifacts: [promptFile] });
|
|
4496
4809
|
saveGraph(repoRoot, graph);
|
|
4497
|
-
if (results.some((g) => g.gate === "test" && !g.pass
|
|
4810
|
+
if (results.some((g) => g.gate === "test" && !g.pass
|
|
4811
|
+
&& !(cfg.executionPolicy?.boundedInfrastructure && failureDisposition(g) === "infrastructure")))
|
|
4498
4812
|
testGateFailed = true;
|
|
4499
4813
|
if (results.every(gateSatisfied)) {
|
|
4500
4814
|
// T6: every gate — the review included — is satisfied on this commit, so the failure brief
|
|
@@ -4506,6 +4820,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4506
4820
|
// gate-fail brief is, so prior-RUN evidence (retired by its own rule, not by this reviewer)
|
|
4507
4821
|
// survives and only this run's settled findings go.
|
|
4508
4822
|
feedback = withCarriedEvidence("");
|
|
4823
|
+
executionSignal()?.throwIfAborted();
|
|
4509
4824
|
const m = await mergeSerial(taskBranch, t, gated);
|
|
4510
4825
|
if (m.tipMoved) {
|
|
4511
4826
|
journal.append("tip-moved", t.id, m.tipMoved);
|
|
@@ -4555,12 +4870,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4555
4870
|
// flowing through the chargeable quality path below.
|
|
4556
4871
|
const infraFailure = results.find((g) => gateFailed(g) && g.meta?.infra === true);
|
|
4557
4872
|
if (infraFailure) {
|
|
4558
|
-
await park(t, `${infraFailure.gate}: ${infraFailure.details}`, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4873
|
+
await park(t, `${infraFailure.gate}: ${infraFailure.details}${infraFailure.meta?.recoveryBlocked ? ` — ${infraFailure.meta.recoveryBlocked}` : ""}`, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4874
|
+
return;
|
|
4875
|
+
}
|
|
4876
|
+
// OBS-1007: a cap trip is a typed park of its own — the diff cannot shrink by retrying and the
|
|
4877
|
+
// only honest verb is a recheck under a raised cap. Decided BEFORE the authoring classifier, which
|
|
4878
|
+
// used to lift `gates.diffCap` out of the cap prose as a files[] hint.
|
|
4879
|
+
const capTrip = results.find(isDiffCapPark);
|
|
4880
|
+
if (capTrip) {
|
|
4881
|
+
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4559
4882
|
return;
|
|
4560
4883
|
}
|
|
4561
4884
|
// OBS-547: who pays for this red is decided by the run's collateral prediction (see
|
|
4562
4885
|
// dispositionScopeRed) — an authoring defect parks unchargeable before any accounting below.
|
|
4563
|
-
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4886
|
+
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
4564
4887
|
return;
|
|
4565
4888
|
gateFails++; // this attempt's gates failed — the one place quality degradation is verified (never inferred from attempts)
|
|
4566
4889
|
// v1.53 T3: prefer the CLI's own session id captured from this attempt's output (kimi's resume
|
|
@@ -4720,6 +5043,39 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4720
5043
|
return;
|
|
4721
5044
|
}
|
|
4722
5045
|
};
|
|
5046
|
+
const executeWithPolicy = async (t) => {
|
|
5047
|
+
const policy = cfg.executionPolicy;
|
|
5048
|
+
if (!policy || (t.humanGate && !approved.has(t.id)))
|
|
5049
|
+
return execTask(t);
|
|
5050
|
+
await withExecutionBudget({ limitMs: policy.taskExecutionLimitMs, taskId: t.id,
|
|
5051
|
+
readEvents: () => journal.read(), append: journal.append.bind(journal) }, async () => {
|
|
5052
|
+
const signal = executionSignal();
|
|
5053
|
+
const slots = new Set();
|
|
5054
|
+
budgetSlots.set(signal, slots);
|
|
5055
|
+
let closing;
|
|
5056
|
+
const closeOwned = () => withoutExecutionBudget(async () => {
|
|
5057
|
+
for (const slot of slots)
|
|
5058
|
+
await closeSlot(slot);
|
|
5059
|
+
});
|
|
5060
|
+
const stop = () => {
|
|
5061
|
+
closing = closeOwned();
|
|
5062
|
+
// Awaited again below; prevent an unhandled rejection while the worker unwinds.
|
|
5063
|
+
void closing.catch(() => { });
|
|
5064
|
+
};
|
|
5065
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
5066
|
+
try {
|
|
5067
|
+
await execTask(t);
|
|
5068
|
+
}
|
|
5069
|
+
finally {
|
|
5070
|
+
signal.removeEventListener("abort", stop);
|
|
5071
|
+
if (signal.aborted) {
|
|
5072
|
+
await closing;
|
|
5073
|
+
await closeOwned();
|
|
5074
|
+
}
|
|
5075
|
+
budgetSlots.delete(signal);
|
|
5076
|
+
}
|
|
5077
|
+
});
|
|
5078
|
+
};
|
|
4723
5079
|
taskLoopStarted = true;
|
|
4724
5080
|
const inflight = new Map();
|
|
4725
5081
|
// A settled worker can release a dependency or an approval in the same poll tick.
|
|
@@ -4752,8 +5108,33 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4752
5108
|
.filter((t) => !inflight.has(t.id))
|
|
4753
5109
|
.slice(0, Math.max(0, concurrency - inflight.size));
|
|
4754
5110
|
for (const t of ready) {
|
|
4755
|
-
const p =
|
|
5111
|
+
const p = executeWithPolicy(t)
|
|
4756
5112
|
.catch(async (err) => {
|
|
5113
|
+
const cleanupErrors = [];
|
|
5114
|
+
for (const [slot, owner] of workerOwners) {
|
|
5115
|
+
if (owner.taskId !== t.id || !liveSlots.has(slot))
|
|
5116
|
+
continue;
|
|
5117
|
+
try {
|
|
5118
|
+
await withoutExecutionBudget(() => closeSlot(slot));
|
|
5119
|
+
}
|
|
5120
|
+
catch (error) {
|
|
5121
|
+
// Cleanup evidence must not replace the task's original failure or abort siblings.
|
|
5122
|
+
cleanupErrors.push({ slot: slot.name, attempt: owner.attempt, error: String(error) });
|
|
5123
|
+
}
|
|
5124
|
+
}
|
|
5125
|
+
const cleanupEvidence = cleanupErrors.length ? { cleanupErrors } : {};
|
|
5126
|
+
if (err instanceof ExecutionBudgetExceeded) {
|
|
5127
|
+
const wt = worktreePath(repoRoot, `${branch}--${t.id}`);
|
|
5128
|
+
const ref = await preserveWorktree(wt);
|
|
5129
|
+
if (ref)
|
|
5130
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
5131
|
+
const dispatch = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "task-dispatch");
|
|
5132
|
+
await park(t, err.message, "infra", null, 0, Date.now(), 0, 0, undefined, 0, "fresh", { disposition: "execution-budget-exhausted", limitMs: cfg.executionPolicy.taskExecutionLimitMs,
|
|
5133
|
+
...cleanupEvidence,
|
|
5134
|
+
...(Number.isInteger(dispatch?.data.attempt) ? { attempt: dispatch.data.attempt } : {}),
|
|
5135
|
+
...(dispatch?.data.assignment ? { assignment: dispatch.data.assignment } : {}) });
|
|
5136
|
+
return;
|
|
5137
|
+
}
|
|
4757
5138
|
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
4758
5139
|
const dispatch = [...taskEvents].reverse().find((e) => e.event === "task-dispatch");
|
|
4759
5140
|
// OBS-206: shared rule with `resume --retry-failed` — see classifyTaskFailure.
|
|
@@ -4761,7 +5142,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4761
5142
|
const attempts = dispatch && Number.isInteger(dispatch.data.attempt) ? dispatch.data.attempt : 0;
|
|
4762
5143
|
graph = setStatus(graph, t.id, "failed");
|
|
4763
5144
|
saveGraph(repoRoot, graph);
|
|
4764
|
-
journal.append("task-failed", t.id, { error: String(err), kind, attempts });
|
|
5145
|
+
journal.append("task-failed", t.id, { error: String(err), kind, attempts, ...cleanupEvidence });
|
|
4765
5146
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
4766
5147
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
4767
5148
|
})
|