tickmarkr 2.5.3 → 2.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/registry.js +6 -1
- package/dist/adapters/types.d.ts +3 -0
- package/dist/cli/commands/approve.d.ts +1 -0
- package/dist/cli/commands/approve.js +54 -6
- package/dist/cli/commands/doctor.d.ts +4 -0
- package/dist/cli/commands/doctor.js +44 -29
- package/dist/cli/commands/fleet.js +195 -33
- package/dist/cli/commands/init.js +196 -6
- package/dist/cli/commands/plan.js +144 -3
- package/dist/cli/commands/resume.js +6 -2
- package/dist/cli/commands/run.js +21 -2
- package/dist/cli/commands/verify.js +1 -1
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.js +3 -1
- package/dist/compile/collateral.d.ts +2 -0
- package/dist/compile/collateral.js +50 -9
- package/dist/compile/ownership.d.ts +7 -0
- package/dist/compile/ownership.js +59 -22
- package/dist/config/config.d.ts +23 -8
- package/dist/config/config.js +42 -28
- package/dist/config/fleet-overlay.d.ts +3 -9
- package/dist/config/fleet-overlay.js +68 -11
- package/dist/config/fleet-why.d.ts +7 -0
- package/dist/config/fleet-why.js +5 -0
- package/dist/drivers/herdr.d.ts +5 -0
- package/dist/drivers/herdr.js +14 -3
- package/dist/drivers/index.d.ts +15 -1
- package/dist/drivers/index.js +38 -10
- package/dist/drivers/orca.d.ts +99 -10
- package/dist/drivers/orca.js +586 -97
- package/dist/drivers/types.d.ts +1 -0
- package/dist/gates/baseline.d.ts +3 -0
- package/dist/gates/baseline.js +2 -1
- package/dist/gates/cache.d.ts +100 -0
- package/dist/gates/cache.js +389 -0
- package/dist/gates/review.d.ts +35 -2
- package/dist/gates/review.js +64 -16
- package/dist/gates/run-gates.d.ts +5 -0
- package/dist/gates/run-gates.js +134 -18
- package/dist/gates/test-manifest.d.ts +99 -0
- package/dist/gates/test-manifest.js +389 -0
- package/dist/gates/test-reporter.d.ts +4 -0
- package/dist/gates/test-reporter.js +49 -0
- package/dist/route/preference.d.ts +22 -1
- package/dist/route/preference.js +123 -25
- package/dist/route/router.d.ts +13 -0
- package/dist/route/router.js +95 -17
- package/dist/run/daemon.d.ts +13 -0
- package/dist/run/daemon.js +398 -96
- package/dist/run/git.d.ts +35 -1
- package/dist/run/git.js +111 -10
- package/dist/run/journal.d.ts +14 -2
- package/dist/run/journal.js +78 -10
- package/dist/run/lease.d.ts +14 -0
- package/dist/run/lease.js +87 -0
- package/dist/run/merge.d.ts +2 -0
- package/dist/run/merge.js +91 -3
- package/dist/run/operator-state.d.ts +11 -0
- package/dist/run/operator-state.js +17 -3
- package/dist/tui/cockpit/board.d.ts +96 -0
- package/dist/tui/cockpit/board.js +346 -0
- package/dist/tui/cockpit/decision-actions.js +2 -0
- package/dist/tui/cockpit/layout.d.ts +5 -1
- package/dist/tui/cockpit/layout.js +8 -3
- package/dist/tui/cockpit/live-runtime.js +83 -31
- package/dist/tui/cockpit/run-view.d.ts +7 -5
- package/dist/tui/cockpit/run-view.js +12 -11
- package/dist/tui/ink/fleet-app.d.ts +41 -27
- package/dist/tui/ink/fleet-app.js +204 -31
- package/package.json +1 -1
- package/skills/tickmarkr-overseer/SKILL.md +180 -99
package/dist/run/daemon.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
1
2
|
import { execFileSync, spawn } from "node:child_process";
|
|
2
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
4
|
import { shq } from "../adapters/types.js";
|
|
@@ -9,7 +10,7 @@ import { fileURLToPath } from "node:url";
|
|
|
9
10
|
import { stringify } from "yaml";
|
|
10
11
|
import { classifyDeadChannel, NO_TRAILER_SUMMARY, trailerPattern, UNPARSEABLE_TRAILER_SUMMARY, writePrompt } from "../adapters/prompt.js";
|
|
11
12
|
import { allAdapters, getAdapter, probeAll, readDoctor, rolePools } from "../adapters/registry.js";
|
|
12
|
-
import { SettledTrailerTracker, addUsage, channelKey, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
13
|
+
import { SettledTrailerTracker, addUsage, channelKey, matchesInputBox, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
13
14
|
import { bannerShell, paneDispatchCommand } from "../brand.js";
|
|
14
15
|
import { collateralHits } from "../compile/collateral.js";
|
|
15
16
|
import { DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
@@ -27,12 +28,13 @@ import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
|
27
28
|
import { runEnvironment } from "./environment.js";
|
|
28
29
|
import { cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, preserveWorktree, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sh, shGit, SUITE_PARENT_ENV, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
29
30
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
30
|
-
import { activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
31
|
+
import { applyScopeAmendments, activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
31
32
|
import { isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
32
33
|
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
33
34
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
34
|
-
import { nextChannel, route } from "../route/router.js";
|
|
35
|
+
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
35
36
|
import { desiredPanes } from "./reconcile.js";
|
|
37
|
+
import { readTierLiveness, readWatchBoard, supervisionBeatPath } from "./supervision.js";
|
|
36
38
|
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, } from "./stall.js";
|
|
37
39
|
// The live set is also the ownership claim shared by task cleanup and termination.
|
|
38
40
|
export async function closeLiveSlot(liveSlots, driver, slot) {
|
|
@@ -378,14 +380,67 @@ export function setWorkerStartupWindowMsForTests(ms) {
|
|
|
378
380
|
export function resetWorkerStartupWindowMsForTests() {
|
|
379
381
|
workerStartupWindowMs = WORKER_STARTUP_WINDOW_MS;
|
|
380
382
|
}
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
383
|
+
/** One seat, one startup prefix. Once execution or its input box is seen, later panes are ineligible. */
|
|
384
|
+
export class StartupFailureDetector {
|
|
385
|
+
adapter;
|
|
386
|
+
closed = false;
|
|
387
|
+
constructor(adapter) {
|
|
388
|
+
this.adapter = adapter;
|
|
389
|
+
}
|
|
390
|
+
sample(output, launchedAt) {
|
|
391
|
+
if (this.closed || launchedAt === undefined || Date.now() - launchedAt > workerStartupWindowMs)
|
|
392
|
+
return;
|
|
393
|
+
// read() returns a tail, not a guaranteed transcript origin. A saturated read cannot
|
|
394
|
+
// establish startup ownership: the first tool frame may already have scrolled out.
|
|
395
|
+
if (Buffer.byteLength(output) >= WORKER_STARTUP_WINDOW_BYTES || output.split("\n").length >= 500) {
|
|
396
|
+
this.closed = true;
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
const bounded = output;
|
|
400
|
+
const rows = bounded.split("\n");
|
|
401
|
+
let inputStart = rows.length;
|
|
402
|
+
const input = this.adapter.inputBox;
|
|
403
|
+
if (input && matchesInputBox(bounded, input)) {
|
|
404
|
+
// Locate the beginning, including multi-row composers whose footer proves the matcher.
|
|
405
|
+
let end = rows.length;
|
|
406
|
+
let low = 1;
|
|
407
|
+
while (low < end) {
|
|
408
|
+
const mid = Math.floor((low + end) / 2);
|
|
409
|
+
if (matchesInputBox(rows.slice(0, mid).join("\n"), input))
|
|
410
|
+
end = mid;
|
|
411
|
+
else
|
|
412
|
+
low = mid + 1;
|
|
413
|
+
}
|
|
414
|
+
inputStart = 0;
|
|
415
|
+
let high = end;
|
|
416
|
+
while (inputStart + 1 < high) {
|
|
417
|
+
const mid = Math.floor((inputStart + high) / 2);
|
|
418
|
+
if (matchesInputBox(rows.slice(mid, end).join("\n"), input))
|
|
419
|
+
inputStart = mid;
|
|
420
|
+
else
|
|
421
|
+
high = mid;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
let offset = 0;
|
|
425
|
+
for (const [index, row] of rows.entries()) {
|
|
426
|
+
const cleanRow = row.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "");
|
|
427
|
+
// Structured tool frames and the terminal's tool headings close the prefix BEFORE their body.
|
|
428
|
+
const toolFrame = /"(?:type|role)"\s*:\s*"(?:tool[^" ]*|function_call[^" ]*|command_execution)"|^\s*(?:[•⏺]\s*(?:Ran|Explored|Searched|Called|Bash|Read|Write|Edit|Shell|Exec|Tool)\b|⎋ |(?:tool[-_ ](?:call|output|result)|function_call)\b)/i.test(cleanRow);
|
|
429
|
+
if (toolFrame || index >= inputStart) {
|
|
430
|
+
this.closed = true;
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
if (!this.adapter.harnessBannerRows?.includes(cleanRow)) {
|
|
434
|
+
const match = WORKER_STARTUP_FAILURE_RE.exec(row);
|
|
435
|
+
if (match)
|
|
436
|
+
return {
|
|
437
|
+
matchedBytes: match[0], offset: offset + Buffer.byteLength(row.slice(0, match.index)),
|
|
438
|
+
row, rowNumber: index + 1,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
offset += Buffer.byteLength(row) + 1;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
389
444
|
}
|
|
390
445
|
// OBS-901/906: one daemon-owned writer for every execution surface. Drivers expose one retained
|
|
391
446
|
// stream through read(); the daemon persists that exact surface before any close/reconcile can make
|
|
@@ -569,17 +624,54 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
569
624
|
});
|
|
570
625
|
const { verifyIntegrationTip } = await import(${JSON.stringify(new URL(`./merge.${extension}`, import.meta.url).href)});
|
|
571
626
|
const { runWithVerificationBudget } = await import(${JSON.stringify(new URL(`./git.${extension}`, import.meta.url).href)});
|
|
627
|
+
const { runWithCommandLease } = await import(${JSON.stringify(new URL(`./lease.${extension}`, import.meta.url).href)});
|
|
628
|
+
let sequence = 0;
|
|
629
|
+
const grants = new Map();
|
|
630
|
+
const commandCapacities = new Map();
|
|
631
|
+
process.on('message', (message) => {
|
|
632
|
+
if (message.type === 'lease-grant') { grants.get(message.id)?.(message); grants.delete(message.id); }
|
|
633
|
+
});
|
|
634
|
+
const lease = async (command, execute) => {
|
|
635
|
+
const id = ++sequence;
|
|
636
|
+
const grant = new Promise(resolve => grants.set(id, resolve));
|
|
637
|
+
process.send({ type: 'lease-acquire', id, command });
|
|
638
|
+
const { capacity, token } = await grant;
|
|
639
|
+
commandCapacities.set(command, capacity);
|
|
640
|
+
try { return await runWithVerificationBudget(capacity, () => execute(token)); }
|
|
641
|
+
finally { process.send({ type: 'lease-release', id }); }
|
|
642
|
+
};
|
|
572
643
|
let input = ''; for await (const chunk of process.stdin) input += chunk;
|
|
573
644
|
const { intWt, commands, runDir, baseline, capacity } = JSON.parse(input);
|
|
574
645
|
try {
|
|
575
|
-
const result = await runWithVerificationBudget(capacity, () => verifyIntegrationTip(intWt, commands, runDir, baseline));
|
|
576
|
-
if (!cancelled) process.stdout.write(JSON.stringify(result));
|
|
646
|
+
const result = await runWithVerificationBudget(capacity, () => runWithCommandLease(lease, () => verifyIntegrationTip(intWt, commands, runDir, baseline)));
|
|
647
|
+
if (!cancelled) process.stdout.write(JSON.stringify(result.map(row => ({ ...row, capacity: [...commandCapacities].reverse().find(([command]) => command === row.cmd || command.startsWith(row.cmd + " "))?.[1] ?? capacity }))));
|
|
577
648
|
} catch (error) { if (!cancelled) throw error; }
|
|
649
|
+
finally { process.disconnect(); }
|
|
578
650
|
`;
|
|
579
651
|
const child = spawn(process.execPath, [
|
|
580
652
|
...(extension === "ts" ? ["--import", createRequire(import.meta.url).resolve("tsx")] : []),
|
|
581
653
|
"--input-type=module", "-e", script,
|
|
582
|
-
], { stdio: ["pipe", "pipe", "pipe"], detached: true });
|
|
654
|
+
], { env: commandLeaseEnvironment(process.env), stdio: ["pipe", "pipe", "pipe", "ipc"], detached: true });
|
|
655
|
+
const releases = new Map();
|
|
656
|
+
const leases = new Set();
|
|
657
|
+
let closed = false;
|
|
658
|
+
child.on("message", (message) => {
|
|
659
|
+
if (message.type === "lease-release") {
|
|
660
|
+
releases.get(message.id)?.();
|
|
661
|
+
releases.delete(message.id);
|
|
662
|
+
}
|
|
663
|
+
if (message.type !== "lease-acquire")
|
|
664
|
+
return;
|
|
665
|
+
const held = withCommandLease(message.command, async () => {
|
|
666
|
+
if (closed || signal.aborted)
|
|
667
|
+
return;
|
|
668
|
+
const released = new Promise((resolve) => releases.set(message.id, resolve));
|
|
669
|
+
child.send({ type: "lease-grant", id: message.id, capacity: resolvedCapacity(), token: currentCommandLeaseToken() });
|
|
670
|
+
await released;
|
|
671
|
+
});
|
|
672
|
+
leases.add(held);
|
|
673
|
+
void held.catch(() => { }).finally(() => leases.delete(held));
|
|
674
|
+
});
|
|
583
675
|
let output = "";
|
|
584
676
|
let errors = "";
|
|
585
677
|
child.stdout.on("data", (chunk) => { output += chunk; });
|
|
@@ -588,6 +680,10 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
588
680
|
const finished = new Promise((resolve, reject) => {
|
|
589
681
|
child.once("error", reject);
|
|
590
682
|
child.once("close", (code) => {
|
|
683
|
+
closed = true;
|
|
684
|
+
for (const release of releases.values())
|
|
685
|
+
release();
|
|
686
|
+
releases.clear();
|
|
591
687
|
if (signal.aborted && (code === 143 || code === null))
|
|
592
688
|
reject(signal.reason);
|
|
593
689
|
else if (code !== 0)
|
|
@@ -607,6 +703,7 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
607
703
|
}
|
|
608
704
|
finally {
|
|
609
705
|
signal.removeEventListener("abort", cancel);
|
|
706
|
+
await Promise.allSettled(leases);
|
|
610
707
|
}
|
|
611
708
|
}
|
|
612
709
|
/**
|
|
@@ -661,9 +758,10 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
661
758
|
for (const r of await (opts.signal
|
|
662
759
|
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal)
|
|
663
760
|
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline))) {
|
|
761
|
+
const measuredCapacity = r.capacity ?? capacity;
|
|
664
762
|
if (r.pass) {
|
|
665
763
|
// Q121s: a forgiven pass journals its fingerprints — honest about what was carried, never a silent green.
|
|
666
|
-
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity });
|
|
764
|
+
journal.append("tip-verify", undefined, { gate: r.gate, cmd: r.cmd, pass: true, exitCode: r.exitCode, details: r.details, ...(r.forgiven ? { forgiven: true, fingerprints: r.fingerprints } : {}), tip, cmdHash, capacity: measuredCapacity });
|
|
667
765
|
}
|
|
668
766
|
else {
|
|
669
767
|
journal.append("tip-verify-failed", undefined, {
|
|
@@ -676,7 +774,7 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
676
774
|
lastMergedTask: opts.lastMergedTask,
|
|
677
775
|
tip,
|
|
678
776
|
cmdHash,
|
|
679
|
-
capacity,
|
|
777
|
+
capacity: measuredCapacity,
|
|
680
778
|
});
|
|
681
779
|
tipFailed = true;
|
|
682
780
|
}
|
|
@@ -733,7 +831,6 @@ async function observeWorkerProcessTree(marker, cwd) {
|
|
|
733
831
|
}
|
|
734
832
|
return tree.size === 0 ? "empty" : "running";
|
|
735
833
|
}
|
|
736
|
-
const SUITE_COMMAND_RE = /(?:^|[\s/])(vitest(?:\.mjs)?|jest|mocha)(?:[\s/]|$)|\bnpm(?:\s+run)?\s+test\b/i;
|
|
737
834
|
function processCwd(pid) {
|
|
738
835
|
try {
|
|
739
836
|
return realpathSync(readlinkSync(`/proc/${pid}/cwd`));
|
|
@@ -801,7 +898,7 @@ export function countLiveSuites(snapshot, repoRoot, daemonPid = process.pid, cwd
|
|
|
801
898
|
// runner ("…as a vitest test whose…"), so a finished interactive worker counted as a live suite and
|
|
802
899
|
// held a task's gates for 9 min 46 s. A runner is named in a command's HEAD — `node <bin>`,
|
|
803
900
|
// `npm test`, `npx vitest`, `sh -c npm test` — never 140 KB into it: read the first four tokens only.
|
|
804
|
-
const candidates = rows.filter((row) => !ancestors.has(row.pid) &&
|
|
901
|
+
const candidates = rows.filter((row) => !ancestors.has(row.pid) && isRunnerCommand(row.command));
|
|
805
902
|
const attributable = new Set(candidates.filter((row) => {
|
|
806
903
|
if (descendants.has(row.pid))
|
|
807
904
|
return true;
|
|
@@ -1346,7 +1443,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1346
1443
|
const rm = resolveRunMode(repoRoot, { flag: opts.mode, spec: graph.mode, globalDir: opts.globalDir });
|
|
1347
1444
|
const cfg = rm.cfg;
|
|
1348
1445
|
// Resolve both budgets once: worker fan-out divides by dispatch concurrency; verification
|
|
1349
|
-
// divides by one
|
|
1446
|
+
// divides by one runner command holding a lease. Explicit operator caps still win.
|
|
1350
1447
|
const concurrency = opts.concurrency ?? cfg.concurrency;
|
|
1351
1448
|
return await runWithForkBudget(concurrency, async () => {
|
|
1352
1449
|
const conservativeCapacity = resolvedCapacity();
|
|
@@ -1358,7 +1455,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1358
1455
|
return await runWithVerificationBudget(occupancyCapacity, async () => {
|
|
1359
1456
|
// v1.51 T4: every dispatch provenance line begins with the mode and its source; when a pin won
|
|
1360
1457
|
// the route (the final "→ " segment is a pin, not a degraded-to-auto tail) it names the mode it bypassed.
|
|
1361
|
-
const dispatchProvenance = (p) => `mode ${rm.mode.mode} (${rm.source})${p.split("→ ").pop().startsWith("pin ") ? ` — pin bypasses mode ${rm.mode.mode}` : ""} · ${p}`;
|
|
1458
|
+
const dispatchProvenance = (p) => `mode ${rm.mode.mode} (${rm.source})${p.split("→ ").pop().startsWith("pin ") && !p.includes("not re-tried") ? ` — pin bypasses mode ${rm.mode.mode}` : ""} · ${p}`;
|
|
1362
1459
|
// v1.87 T2: one pool per seat role, built once. `channels` stays the WORKER pool — every routing
|
|
1363
1460
|
// call below reads it exactly as before — while the judge, review and consult seats each receive
|
|
1364
1461
|
// the pool their own deny scope allows, so routing.deny.workers benches a channel for dispatch
|
|
@@ -1395,6 +1492,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1395
1492
|
liveSlots.add(s);
|
|
1396
1493
|
return s;
|
|
1397
1494
|
} } : {}),
|
|
1495
|
+
...(driver.retireLostWatch ? { retireLostWatch: driver.retireLostWatch.bind(driver) } : {}),
|
|
1398
1496
|
...(driver.project ? { project: driver.project.bind(driver) } : {}),
|
|
1399
1497
|
...(driver.reconcile ? { reconcile: driver.reconcile.bind(driver) } : {}),
|
|
1400
1498
|
notify: (m, o) => driver.notify(m, o),
|
|
@@ -1468,6 +1566,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1468
1566
|
const reviewHistory = journal.read()
|
|
1469
1567
|
.filter((event) => event.event === "gate-result" && event.data.gate === "review" && typeof event.data.reviewer === "string")
|
|
1470
1568
|
.map((event) => event.data.reviewer);
|
|
1569
|
+
// RF-1 (OBS-922 add.3): the seats THIS task's review rows name — the prior reviewers a later round's
|
|
1570
|
+
// floor holds. Task-scoped on purpose: reviewHistory rotates seats run-wide and is never floor evidence.
|
|
1571
|
+
// Each row's journaled dispatch tier travels with it, so a seat that left the pool on resume still counts.
|
|
1572
|
+
// Leg-2 T4 M2: a `review-no-verdict` note is dispatch evidence too — it lands BEFORE the in-gate retry's
|
|
1573
|
+
// gate-result, so a daemon killed between the two leaves it as the task's only record of that seat.
|
|
1574
|
+
const taskReviewers = (taskId) => journal.read()
|
|
1575
|
+
.filter((event) => event.taskId === taskId && typeof event.data.reviewer === "string"
|
|
1576
|
+
&& ((event.event === "gate-result" && event.data.gate === "review") || event.event === "review-no-verdict"))
|
|
1577
|
+
.map((event) => ({ reviewer: event.data.reviewer, tier: event.data.reviewerTier }));
|
|
1471
1578
|
// Capture this before this observer appends run-resume. A prior run-end belongs to a completed
|
|
1472
1579
|
// lifecycle and must use the ordinary resume/redispatch rules; only an interrupted live attempt
|
|
1473
1580
|
// owns reusable measurements or unclean-death residue.
|
|
@@ -1499,66 +1606,124 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1499
1606
|
const commands = detectGateCommands(repoRoot, cfg);
|
|
1500
1607
|
const watchName = formatOwnedName({ role: "watch", taskId: "run", attempt: 0, runId });
|
|
1501
1608
|
let watchSlot;
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1609
|
+
let boardOpened = false; // WB-1: only a board this daemon placed is watched below
|
|
1610
|
+
// Leg-2 T7 M1 (OBS-988): a reopen first retires the ghost through the driver's seam so the narrator
|
|
1611
|
+
// launches a NEW pane; a driver that answers with the lost pane anyway (no seam, or a cache it
|
|
1612
|
+
// would not drop) has not reopened anything, and that answer is a FAILED reopen, never a success.
|
|
1613
|
+
const openBoard = async (ghost) => {
|
|
1507
1614
|
try {
|
|
1508
|
-
|
|
1615
|
+
if (ghost?.slot)
|
|
1616
|
+
await trackedDriver.retireLostWatch?.(ghost.slot);
|
|
1617
|
+
const slot = await trackedDriver.narrator(repoRoot, watchCommand(runId), runId);
|
|
1618
|
+
if (ghost && slot.id === ghost.pane) {
|
|
1619
|
+
liveSlots.delete(slot); // the ghost again: no close is ever aimed at it
|
|
1620
|
+
return { ok: false, error: `narrator answered with the lost pane ${ghost.pane}` };
|
|
1621
|
+
}
|
|
1622
|
+
watchSlot = slot;
|
|
1623
|
+
boardOpened = true;
|
|
1624
|
+
return { ok: true, slot };
|
|
1509
1625
|
}
|
|
1510
1626
|
catch (error) {
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1627
|
+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
1628
|
+
}
|
|
1629
|
+
};
|
|
1630
|
+
const placeBoard = async () => {
|
|
1631
|
+
if (!trackedDriver.narrator) {
|
|
1632
|
+
noteCapabilityAbsent("narrator");
|
|
1633
|
+
return undefined;
|
|
1514
1634
|
}
|
|
1635
|
+
const placed = await openBoard();
|
|
1636
|
+
if (placed.ok)
|
|
1637
|
+
return placed.slot;
|
|
1638
|
+
journal.append("watch-placement-failed", undefined, { error: placed.error });
|
|
1639
|
+
console.error(`tickmarkr: narrator not opened: ${placed.error}`);
|
|
1640
|
+
return undefined;
|
|
1515
1641
|
};
|
|
1516
|
-
//
|
|
1517
|
-
//
|
|
1518
|
-
//
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1642
|
+
// WB-1 (OBS-988): the daemon watches its own cockpit. A board is lost when the watch tier's beat
|
|
1643
|
+
// aged past the supervision stale bound, the recorded arm has no presence, or the owner pid is dead.
|
|
1644
|
+
// The beat and presence checks wait for the UI to have armed (armId recorded) — a freshly reopened
|
|
1645
|
+
// board that has not armed yet is not a second loss. Reopens are bounded per run; past the bound
|
|
1646
|
+
// the loss is journaled `boardless` and the narrator is never called again.
|
|
1647
|
+
// Leg-2 T7 M2: one `watch-board-lost` row per LOSS. A loss is identified by the pane and pid the
|
|
1648
|
+
// owner record names; a reopen that fails leaves that record in place, so the next poll sees the
|
|
1649
|
+
// same loss, journals only its own reopen attempt, and the bound retires the run boardless from the
|
|
1650
|
+
// last failed attempt rather than from a repeated lost row.
|
|
1651
|
+
const MAX_BOARD_REOPENS = 3;
|
|
1652
|
+
let boardReopens = journal.read().filter((e) => e.event === "watch-board-reopened" || e.event === "watch-board-reopen-failed").length;
|
|
1653
|
+
let boardless = false;
|
|
1654
|
+
let journaledLoss;
|
|
1655
|
+
const boardLoss = () => {
|
|
1656
|
+
const owner = readWatchBoard(repoRoot, runId);
|
|
1657
|
+
if (!owner)
|
|
1658
|
+
return undefined;
|
|
1659
|
+
const lost = { pane: owner.pane, ...(owner.pid !== undefined ? { pid: owner.pid } : {}) };
|
|
1660
|
+
if (owner.armId !== undefined) {
|
|
1661
|
+
const beat = readTierLiveness(repoRoot, "watch");
|
|
1662
|
+
if (beat.state === "STALE")
|
|
1663
|
+
return { ...lost, beatAgeMs: beat.beatAgeMs };
|
|
1664
|
+
// ponytail: presence path math mirrors supervision.ts's private helper (`<tier>.live.<armId>`)
|
|
1665
|
+
if (!existsSync(join(dirname(supervisionBeatPath(repoRoot, "watch")), `watch.live.${owner.armId}`)))
|
|
1666
|
+
return lost;
|
|
1531
1667
|
}
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
journal.append("suite-wait", taskId, { count });
|
|
1552
|
-
lastCount = count;
|
|
1553
|
-
await new Promise((wake) => setTimeout(wake, SUITE_POLL_MS));
|
|
1554
|
-
}
|
|
1555
|
-
return await run();
|
|
1668
|
+
if (owner.pid !== undefined && !isPidLive(owner.pid))
|
|
1669
|
+
return lost;
|
|
1670
|
+
return undefined;
|
|
1671
|
+
};
|
|
1672
|
+
const watchBoard = async () => {
|
|
1673
|
+
if (!boardOpened || boardless || !trackedDriver.narrator)
|
|
1674
|
+
return;
|
|
1675
|
+
const loss = boardLoss();
|
|
1676
|
+
if (!loss) {
|
|
1677
|
+
journaledLoss = undefined;
|
|
1678
|
+
return;
|
|
1679
|
+
}
|
|
1680
|
+
const identity = `${loss.pane}:${loss.pid ?? ""}`;
|
|
1681
|
+
if (identity !== journaledLoss) {
|
|
1682
|
+
journaledLoss = identity;
|
|
1683
|
+
boardless = boardReopens >= MAX_BOARD_REOPENS;
|
|
1684
|
+
journal.append("watch-board-lost", undefined, { ...loss, ...(boardless ? { boardless: true } : {}) });
|
|
1685
|
+
if (boardless)
|
|
1686
|
+
return;
|
|
1556
1687
|
}
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1688
|
+
const attempt = ++boardReopens;
|
|
1689
|
+
// The ghost pane is gone: drop it from this run's live set so no close is ever aimed at it.
|
|
1690
|
+
const ghost = watchSlot;
|
|
1691
|
+
if (ghost)
|
|
1692
|
+
liveSlots.delete(ghost);
|
|
1693
|
+
watchSlot = undefined;
|
|
1694
|
+
const reopened = await openBoard({ slot: ghost, pane: loss.pane });
|
|
1695
|
+
if (reopened.ok) {
|
|
1696
|
+
journal.append("watch-board-reopened", undefined, { pane: readWatchBoard(repoRoot, runId)?.pane ?? reopened.slot.id, attempt });
|
|
1697
|
+
return;
|
|
1560
1698
|
}
|
|
1699
|
+
boardless = attempt >= MAX_BOARD_REOPENS;
|
|
1700
|
+
journal.append("watch-board-reopen-failed", undefined, { pane: loss.pane, attempt, error: reopened.error, ...(boardless ? { boardless: true } : {}) });
|
|
1701
|
+
console.error(`tickmarkr: board not reopened (attempt ${attempt}): ${reopened.error}`);
|
|
1561
1702
|
};
|
|
1703
|
+
// Context carries attribution through gates and remote inference; only shell commands acquire.
|
|
1704
|
+
const commandLeases = new CommandLeases();
|
|
1705
|
+
const withCommandContext = (taskId, run, signal) => runWithCommandLease((_command, execute) => commandLeases.run(async () => {
|
|
1706
|
+
let lastCount = -1;
|
|
1707
|
+
const startedAt = Date.now();
|
|
1708
|
+
for (;;) {
|
|
1709
|
+
signal?.throwIfAborted();
|
|
1710
|
+
const count = await liveSuiteCount(repoRoot);
|
|
1711
|
+
if (count === 0)
|
|
1712
|
+
break;
|
|
1713
|
+
if (count !== lastCount)
|
|
1714
|
+
journal.append("suite-wait", taskId, { count });
|
|
1715
|
+
lastCount = count;
|
|
1716
|
+
if (Date.now() - startedAt >= suiteWaitCeilingMs) {
|
|
1717
|
+
journal.append("suite-wait-ceiling", taskId, { count, waitedMs: Date.now() - startedAt });
|
|
1718
|
+
journal.append("suite-budget", taskId, {
|
|
1719
|
+
count, occupancyCap: occupancyCapacity.forkCap, conservativeCap: conservativeCapacity.forkCap,
|
|
1720
|
+
});
|
|
1721
|
+
return await runWithVerificationBudget(conservativeCapacity, execute);
|
|
1722
|
+
}
|
|
1723
|
+
await new Promise((wake) => setTimeout(wake, SUITE_POLL_MS));
|
|
1724
|
+
}
|
|
1725
|
+
return await execute();
|
|
1726
|
+
}, (count) => journal.append("suite-wait", taskId, { count }), SUITE_POLL_MS, signal), run);
|
|
1562
1727
|
let baseRef;
|
|
1563
1728
|
let baseline;
|
|
1564
1729
|
let baselinePending = false;
|
|
@@ -1596,6 +1761,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1596
1761
|
// The SAME comparator status uses (engagementComparable); one decision, two consumers. Fail closed:
|
|
1597
1762
|
// no resume path silently accepts a mismatched or unbound journal. --graph-changed is the operator's
|
|
1598
1763
|
// audited release for the stop-amend-resume workflow, journaling a graph-rehash naming both hashes.
|
|
1764
|
+
graph = applyScopeAmendments(graph, journal, true);
|
|
1599
1765
|
const loadedHash = graphDefinitionHash(graph);
|
|
1600
1766
|
const cmp = engagementComparable(journal.read(), loadedHash);
|
|
1601
1767
|
if (!cmp.comparable) {
|
|
@@ -1632,16 +1798,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1632
1798
|
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
1633
1799
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
1634
1800
|
});
|
|
1635
|
-
await
|
|
1801
|
+
await placeBoard();
|
|
1636
1802
|
}
|
|
1637
1803
|
else {
|
|
1638
1804
|
baseRef = await gitHead(repoRoot);
|
|
1639
1805
|
journal.append("baseline-start", undefined, { baseRef, commands, capacity: resolvedCapacity() });
|
|
1640
|
-
await
|
|
1806
|
+
await placeBoard();
|
|
1641
1807
|
baselinePending = true;
|
|
1642
1808
|
// Workers can run beside capture, but no gate may observe an absent or partial baseline.
|
|
1643
1809
|
// Keep publication and warnings inside the same barrier as the suite's final verdict.
|
|
1644
|
-
baselineCapture =
|
|
1810
|
+
baselineCapture = withCommandContext(undefined, async () => {
|
|
1645
1811
|
const captured = await captureBaseline(repoRoot, commands);
|
|
1646
1812
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(captured, null, 2));
|
|
1647
1813
|
baseline = captured;
|
|
@@ -1773,15 +1939,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1773
1939
|
];
|
|
1774
1940
|
return `${identity} — release with ${commands.map((command) => `\`${command}\``).join(" or ")}`;
|
|
1775
1941
|
};
|
|
1942
|
+
const namedPaths = (text) => [...text.matchAll(/(?:^|[\s`'"(])((?:[A-Za-z0-9_@.()[\]-]+\/)+[A-Za-z0-9_@.[\]-]+|[A-Za-z0-9_@-]+(?:\.[A-Za-z0-9_-]+)+)(?=$|[\s`'"),:;.!?])/g)]
|
|
1943
|
+
.map((match) => match[1].replace(/^\.\//, "").replace(/\.$/, ""))
|
|
1944
|
+
.filter((path) => !path.split("/").includes("..")
|
|
1945
|
+
&& (/\.[A-Za-z][A-Za-z0-9_-]*$/.test(basename(path)) || existsSync(join(repoRoot, path))));
|
|
1776
1946
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
1777
1947
|
// OBS-979: a worker refusal can identify the missing authoring scope even when gates
|
|
1778
1948
|
// subsequently supply the park's disposition. Keep that actionable path on the park itself.
|
|
1779
1949
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
1780
1950
|
if (t.files.length > 0 && worker?.data.ok === false && typeof worker.data.summary === "string") {
|
|
1781
1951
|
const allowed = filesGlob(t.files);
|
|
1782
|
-
const paths =
|
|
1783
|
-
.map((match) => match[1].replace(/^\.\//, "").replace(/\.$/, ""))
|
|
1784
|
-
.filter((path) => !path.split("/").includes("..") && !allowed(path));
|
|
1952
|
+
const paths = namedPaths(worker.data.summary).filter((path) => !allowed(path));
|
|
1785
1953
|
if (paths.length)
|
|
1786
1954
|
reason += ` — files[] repair hint: ${[...new Set(paths)].join(", ")}`;
|
|
1787
1955
|
}
|
|
@@ -1814,6 +1982,35 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1814
1982
|
const dispositionScopeRed = async (t, results, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode) => {
|
|
1815
1983
|
const scopeRed = results.find((g) => g.gate === "scope" && gateFailed(g));
|
|
1816
1984
|
const verdict = scopeRed?.meta?.collateral;
|
|
1985
|
+
if (!verdict && t.files.length > 0) {
|
|
1986
|
+
const allowed = filesGlob(t.files);
|
|
1987
|
+
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
1988
|
+
const refusal = worker?.data.ok === false && typeof worker.data.summary === "string"
|
|
1989
|
+
&& /outside|out.of.scope|allowlist|scope expansion|not (?:in|own)|unowned/i.test(worker.data.summary)
|
|
1990
|
+
? namedPaths(worker.data.summary).filter((path) => !allowed(path)) : [];
|
|
1991
|
+
const reds = results.filter(gateFailed);
|
|
1992
|
+
// A scope verdict already attributes actual out-of-scope EDITS using the collateral map.
|
|
1993
|
+
// Path-only diagnostics from the other gates name code that needs fixing, not an edit the
|
|
1994
|
+
// worker made. Do not replace the scope gate's stronger attribution with that inference.
|
|
1995
|
+
const paths = reds.filter((g) => g.gate !== "scope" || !g.meta?.collateral).flatMap((g) => namedPaths(g.details));
|
|
1996
|
+
if (refusal.length || (paths.length > 0 && paths.every((path) => !allowed(path)))) {
|
|
1997
|
+
const classification = {
|
|
1998
|
+
gate: refusal.length ? "worker" : reds[0]?.gate,
|
|
1999
|
+
paths: [...new Set(refusal.length ? refusal : paths)],
|
|
2000
|
+
repair: `files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}`,
|
|
2001
|
+
attempt: attempts + 1, chargeable: false, source: "diagnostic",
|
|
2002
|
+
};
|
|
2003
|
+
if (refusal.length)
|
|
2004
|
+
journal.append("scope-request", t.id, classification);
|
|
2005
|
+
else
|
|
2006
|
+
journal.append("scope-authoring", t.id, classification);
|
|
2007
|
+
// OBS-547 (Leg-2 T3 material): unchargeable ⇒ metered 0, exactly as the predicted path below —
|
|
2008
|
+
// passing the physical count would write `meteredAttempts: 1` beside `attempts: 0`.
|
|
2009
|
+
const approveCommand = `tickmarkr approve ${runId} ${t.id} --files ${[...new Set(refusal)].map((path) => /^[\w./-]+$/.test(path) ? path : shq(path)).join(",")}`;
|
|
2010
|
+
await park(t, `${refusal.length ? "scope request" : "authoring defect"} — files[] repair hint: ${[...new Set(refusal.length ? refusal : paths)].join(", ")}; current files[]: ${t.files.join(", ")}`, refusal.length ? "scope-request" : "authoring", assignment, attempts, startMs, gateFails, consults, tokens, 0, retryMode, refusal.length ? { paths: [...new Set(refusal)], approveCommand, graphDefinitionHash: graphDefinitionHash(graph) } : {});
|
|
2011
|
+
return true;
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
1817
2014
|
if (!verdict)
|
|
1818
2015
|
return false;
|
|
1819
2016
|
if (verdict.authoring) {
|
|
@@ -1850,6 +2047,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1850
2047
|
approvalSweepCursor = events.length;
|
|
1851
2048
|
if (approvals.length === 0)
|
|
1852
2049
|
return;
|
|
2050
|
+
graph = applyScopeAmendments(graph, journal);
|
|
1853
2051
|
resume = journal.replayResumeState();
|
|
1854
2052
|
satisfiedGates = journal.replaySatisfiedGates();
|
|
1855
2053
|
replayedGateResults = resumeLifecycleOpen
|
|
@@ -1932,12 +2130,28 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1932
2130
|
// not re-tried first (consult bans / prior failovers survive the release).
|
|
1933
2131
|
const rs = resume.get(t.id);
|
|
1934
2132
|
const contentDigest = taskContentDigest(t);
|
|
1935
|
-
|
|
2133
|
+
const previousDispatch = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "task-dispatch");
|
|
2134
|
+
const recordedGraphPath = join(journal.dir, "graph.json");
|
|
2135
|
+
const recordedTask = existsSync(recordedGraphPath)
|
|
2136
|
+
? JSON.parse(readFileSync(recordedGraphPath, "utf8")).tasks.find((task) => task.id === t.id)
|
|
2137
|
+
: undefined;
|
|
2138
|
+
const previousHints = previousDispatch && "routingHints" in previousDispatch.data
|
|
2139
|
+
? previousDispatch.data.routingHints : recordedTask?.routingHints;
|
|
2140
|
+
const hintsChanged = !!rs && (!!recordedTask || !!previousDispatch && "routingHints" in previousDispatch.data)
|
|
2141
|
+
&& (JSON.stringify(previousHints?.pin) !== JSON.stringify(t.routingHints?.pin)
|
|
2142
|
+
|| previousHints?.floor !== t.routingHints?.floor);
|
|
2143
|
+
if (hintsChanged)
|
|
2144
|
+
journal.append("restore-rerouted", t.id, {
|
|
2145
|
+
from: rs.lastAssignment ? channelKey(rs.lastAssignment) : rs.tried.at(-1) ?? null,
|
|
2146
|
+
to: channelKey(assignment),
|
|
2147
|
+
reason: JSON.stringify(previousHints?.pin) !== JSON.stringify(t.routingHints?.pin) ? "pin changed" : "floor changed",
|
|
2148
|
+
});
|
|
2149
|
+
if (!hintsChanged && rs?.lastAssignment && rs.attempts > 0
|
|
1936
2150
|
&& channels.some((c) => channelKey(c) === channelKey(rs.lastAssignment))
|
|
1937
2151
|
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
1938
2152
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
1939
2153
|
}
|
|
1940
|
-
else if (rs && rs.tried.length) {
|
|
2154
|
+
else if (!hintsChanged && rs && rs.tried.length) {
|
|
1941
2155
|
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
1942
2156
|
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
1943
2157
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
@@ -1948,6 +2162,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1948
2162
|
// assignment and proceed. Dispatching on a previously-tried channel beats deadlocking a resumed
|
|
1949
2163
|
// run; a park-instead policy can come later if it ever bites.
|
|
1950
2164
|
}
|
|
2165
|
+
const taskHistory = journal.read().filter((e) => e.taskId === t.id);
|
|
2166
|
+
const lastApproval = taskHistory.map((e) => e.event).lastIndexOf("task-approved");
|
|
2167
|
+
const priorClimb = taskHistory.slice(lastApproval + 1).reverse().find((e) => e.event === "tier-escalated");
|
|
2168
|
+
if (!hintsChanged && priorClimb && taskHistory.indexOf(priorClimb) > taskHistory.map((e) => e.event).lastIndexOf("task-dispatch")) {
|
|
2169
|
+
const target = channels.find((c) => channelKey(c) === priorClimb.data.to && !demotedChannels.has(channelKey(c)));
|
|
2170
|
+
if (target)
|
|
2171
|
+
assignment = { adapter: target.adapter, model: target.model, channel: target.channel, tier: target.tier };
|
|
2172
|
+
}
|
|
1951
2173
|
// pre-kill invariant: tried always contains the current assignment. Spread, never alias the
|
|
1952
2174
|
// journal-derived array (no hidden mutation of replayed state).
|
|
1953
2175
|
const tried = rs?.tried.length ? [...rs.tried] : [channelKey(assignment)];
|
|
@@ -2070,7 +2292,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2070
2292
|
...(g.meta?.reapedGroup === true ? { reapedGroup: true } : {}),
|
|
2071
2293
|
...(g.gate === "review" && typeof g.meta?.reviewer === "string" ? {
|
|
2072
2294
|
reviewer: g.meta.reviewer,
|
|
2073
|
-
...Object.fromEntries(["cause", "seatAuthoredBytes", "bytes", "rawPath", "briefPath", "timeoutMs", "unparseable", "noVerdict", "resolved", "reraised"]
|
|
2295
|
+
...Object.fromEntries(["cause", "seatAuthoredBytes", "bytes", "rawPath", "briefPath", "timeoutMs", "unparseable", "noVerdict", "resolved", "reraised", "reviewerFloor", "reviewerFloorCause", "reviewerTier"]
|
|
2074
2296
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
2075
2297
|
...(typeof g.meta.vendor === "string" ? { vendor: g.meta.vendor } : {}),
|
|
2076
2298
|
...(typeof g.meta.provider === "string" ? { provider: g.meta.provider } : {}),
|
|
@@ -2170,6 +2392,43 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2170
2392
|
? `The operator UPHELD the reviewer's findings — address them without discarding landed work.\nreview: ${upheldFeedback}`
|
|
2171
2393
|
: "");
|
|
2172
2394
|
let ladderIdx = 0;
|
|
2395
|
+
const engagementRows = () => {
|
|
2396
|
+
const rows = journal.read().filter((e) => e.taskId === t.id);
|
|
2397
|
+
const approval = rows.map((e) => e.event).lastIndexOf("task-approved");
|
|
2398
|
+
return rows.slice(approval + 1);
|
|
2399
|
+
};
|
|
2400
|
+
let ownedGateFailures = engagementRows().filter((e) => e.event === "escalation" && e.data.workerAttributed === true).length;
|
|
2401
|
+
let climbProvenance = !hintsChanged && priorClimb
|
|
2402
|
+
? `tier-escalated ${priorClimb.data.from} → ${priorClimb.data.to} (${priorClimb.data.cause})` : "";
|
|
2403
|
+
const climbSkips = new Set(!hintsChanged && priorClimb
|
|
2404
|
+
? channels.filter((c) => c.tier === priorClimb.data.fromTier).map(channelKey) : []);
|
|
2405
|
+
const climb = async (cause, gate, attempt) => {
|
|
2406
|
+
if (cfg.routing.escalateTier === "off" || engagementRows().some((e) => e.event === "tier-escalated"))
|
|
2407
|
+
return;
|
|
2408
|
+
const pin = t.routingHints?.pin;
|
|
2409
|
+
if (pin) {
|
|
2410
|
+
await park(t, `pin ${pin.via}:${pin.model} held: ${cause}`, "gate-fail", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2411
|
+
return "parked";
|
|
2412
|
+
}
|
|
2413
|
+
const next = climbChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
|
|
2414
|
+
if (!next?.climbed)
|
|
2415
|
+
return;
|
|
2416
|
+
const from = channelKey(assignment);
|
|
2417
|
+
const to = channelKey(next);
|
|
2418
|
+
const poolBefore = channels.filter((c) => !tried.includes(channelKey(c)) && !demotedChannels.has(channelKey(c))).map(channelKey);
|
|
2419
|
+
journal.append("tier-escalated", t.id, {
|
|
2420
|
+
attempt, cause, gate: gate.gate, fingerprint: normalizeGateFailure(gate.details).slice(0, 500),
|
|
2421
|
+
from, to, fromTier: assignment.tier, toTier: next.tier, poolBefore,
|
|
2422
|
+
costDelta: marginalCostRank(channels.find((c) => channelKey(c) === to)) - marginalCostRank(channels.find((c) => channelKey(c) === from)),
|
|
2423
|
+
});
|
|
2424
|
+
for (const c of channels)
|
|
2425
|
+
if (c.tier === assignment.tier && channelKey(c) !== to)
|
|
2426
|
+
climbSkips.add(channelKey(c));
|
|
2427
|
+
climbProvenance = `tier-escalated ${from} → ${to} (${cause})`;
|
|
2428
|
+
assignment = { adapter: next.adapter, model: next.model, channel: next.channel, tier: next.tier };
|
|
2429
|
+
tried.push(to);
|
|
2430
|
+
return "climbed";
|
|
2431
|
+
};
|
|
2173
2432
|
let modeFallbackNoted = false; // v1.2: journal the interactive→print fallback once per task, not per attempt
|
|
2174
2433
|
let gateFails = 0; // TEL-02: incremented ONLY where feedback is built from failing gates — never derived from attempts (quota failovers bump attempts too, Pitfall 6)
|
|
2175
2434
|
let consults = 0; // TEL-02: bumped in the runConsult wrapper so one counter covers all three trigger sites, across the attempt loop
|
|
@@ -2448,7 +2707,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2448
2707
|
await trackedDriver.project?.(t.id, "in-review");
|
|
2449
2708
|
await waitForBaseline(t.id);
|
|
2450
2709
|
journal.phaseStart(t.id, "gates");
|
|
2451
|
-
const { results } = await
|
|
2710
|
+
const { results } = await withCommandContext(t.id, () => runReviewRecovery(resumedTask, {
|
|
2452
2711
|
carriedFindings: outstandingReviewFindings(journal.read(), t.id),
|
|
2453
2712
|
worktree: wt, baseRef: taskBase, result: priorResult, author: gateAuthor,
|
|
2454
2713
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
@@ -2465,7 +2724,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2465
2724
|
}
|
|
2466
2725
|
: undefined,
|
|
2467
2726
|
excludeReviewers: badReviewers,
|
|
2468
|
-
reviewHistory, demotedReviewers,
|
|
2727
|
+
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
2469
2728
|
onGate: async (e) => {
|
|
2470
2729
|
if (e.phase === "start") {
|
|
2471
2730
|
notePhaseStart(e);
|
|
@@ -2714,7 +2973,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2714
2973
|
feedback = feedback ? `${feedback}\n\n${consultBrief}` : consultBrief;
|
|
2715
2974
|
}
|
|
2716
2975
|
}
|
|
2717
|
-
|
|
2976
|
+
const scopeApproval = [...journaledSoFar].reverse().find((e) => e.taskId === t.id && (e.event === "task-approved" || e.event === "task-dispatch"));
|
|
2977
|
+
retryMode = scopeApproval?.data.release === "scope-request" ? "fresh" : repairFindings
|
|
2718
2978
|
? "repair"
|
|
2719
2979
|
: priorSession
|
|
2720
2980
|
&& priorSession.channel === channelKey(assignment)
|
|
@@ -2741,8 +3001,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2741
3001
|
// ledger cannot tell a carried dispatch from an amnesiac one — the exact question a run that
|
|
2742
3002
|
// spends two frontier attempts re-deriving a known defect has to be able to answer afterwards.
|
|
2743
3003
|
journal.append("task-dispatch", t.id, {
|
|
2744
|
-
|
|
2745
|
-
|
|
3004
|
+
...(scopeApproval?.data.release === "scope-request" ? { files: t.files, graphDefinitionHash: graphDefinitionHash(graph) } : {}),
|
|
3005
|
+
assignment, attempt, provenance: dispatchProvenance([
|
|
3006
|
+
channelKey(assignment) === channelKey(r.assignment) ? r.provenance
|
|
3007
|
+
: t.routingHints?.pin ? `pin ${t.routingHints.pin.via}:${t.routingHints.pin.model} not re-tried` : "ladder assignment",
|
|
3008
|
+
`dispatch ${channelKey(assignment)}`, climbProvenance,
|
|
3009
|
+
].filter(Boolean).join(" · ")), retryMode,
|
|
3010
|
+
routingHints: t.routingHints ?? {},
|
|
3011
|
+
excludedChannels: [...new Set([...demotedChannels, ...tried, ...climbSkips])]
|
|
3012
|
+
.filter((key) => key !== channelKey(assignment)).sort(),
|
|
3013
|
+
exclusionReasons: Object.fromEntries([...new Set([...demotedChannels, ...tried, ...climbSkips])]
|
|
3014
|
+
.filter((key) => key !== channelKey(assignment))
|
|
3015
|
+
.map((key) => [key, demotedChannels.has(key) ? "demoted" : tried.includes(key) ? "already tried" : "tier climb skipped"])),
|
|
2746
3016
|
...(outstandingFindings.length > 0 ? { carriedFindings: outstandingFindings } : {}),
|
|
2747
3017
|
...(carriedConsultGuidance ? { carriedConsultGuidance } : {}),
|
|
2748
3018
|
});
|
|
@@ -3056,6 +3326,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3056
3326
|
let processExited = false;
|
|
3057
3327
|
let earlyLaunchDead = false;
|
|
3058
3328
|
let startupFailure = false;
|
|
3329
|
+
let startupEvidence;
|
|
3330
|
+
const startupDetector = new StartupFailureDetector(adapter);
|
|
3331
|
+
const startupFailureInWindow = (text, launchedAt) => {
|
|
3332
|
+
startupEvidence ??= startupDetector.sample(text, launchedAt);
|
|
3333
|
+
return startupEvidence !== undefined;
|
|
3334
|
+
};
|
|
3059
3335
|
let deadWorkerPark;
|
|
3060
3336
|
let settleParsed;
|
|
3061
3337
|
const trailerFrames = interactive && adapter.busyFrameMarkers
|
|
@@ -3852,6 +4128,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3852
4128
|
if (!workerFinished) {
|
|
3853
4129
|
journal.append("worker-reaped-before-harvest", t.id, {
|
|
3854
4130
|
slot: slot.name, attempt, cause: workerCause ?? "stall-timeout",
|
|
4131
|
+
...(startupEvidence ? { evidence: startupEvidence } : {}),
|
|
3855
4132
|
});
|
|
3856
4133
|
}
|
|
3857
4134
|
}
|
|
@@ -3896,6 +4173,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3896
4173
|
// (provider outage, quota banner, dead CLI) must still route; the harvest synthesis only
|
|
3897
4174
|
// decides whether THIS attempt's worktree goes to gates, and when routing wins instead, the
|
|
3898
4175
|
// commits survive via the existing commitsToCarry/cherryPickCommits carry-forward.
|
|
4176
|
+
if (workerFinished && !result.ok && await dispositionScopeRed(t, [], assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode))
|
|
4177
|
+
return;
|
|
3899
4178
|
const preHarvestResult = result;
|
|
3900
4179
|
// T2 (OBS-264): recognize committed no-trailer work BEFORE any no-trailer streak, provider,
|
|
3901
4180
|
// quota or dead-channel routing. Gates never trusted the trailer, so this successful synthesis
|
|
@@ -4185,7 +4464,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4185
4464
|
}
|
|
4186
4465
|
}
|
|
4187
4466
|
else {
|
|
4188
|
-
({ results, commits } = await
|
|
4467
|
+
({ results, commits } = await withCommandContext(t.id, () => runReviewRecovery(t, {
|
|
4189
4468
|
carriedFindings: outstandingFindings,
|
|
4190
4469
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
4191
4470
|
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
@@ -4208,7 +4487,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4208
4487
|
}
|
|
4209
4488
|
: undefined,
|
|
4210
4489
|
excludeReviewers: badReviewers,
|
|
4211
|
-
reviewHistory, demotedReviewers,
|
|
4490
|
+
reviewHistory, demotedReviewers, priorReviewers: taskReviewers(t.id),
|
|
4212
4491
|
onGate,
|
|
4213
4492
|
})));
|
|
4214
4493
|
}
|
|
@@ -4300,6 +4579,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4300
4579
|
&& !isDiffCapPark(reviewFail)
|
|
4301
4580
|
&& results.every((g) => gateSatisfied(g) || g.gate === "review");
|
|
4302
4581
|
const failing = results.filter(gateFailed);
|
|
4582
|
+
const deterministic = failing.find(isDeterministicFailure);
|
|
4583
|
+
if (deterministic)
|
|
4584
|
+
ownedGateFailures++;
|
|
4303
4585
|
// Decided before the cap, because the cap's question is whether the NEXT move would re-buy a
|
|
4304
4586
|
// measurement already made — and a funded repair is one of the moves that would.
|
|
4305
4587
|
const landed = await commitsAheadOf(taskBase, wt);
|
|
@@ -4341,13 +4623,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4341
4623
|
channel: channelKey(assignment), // the ban is bound to the channel that produced the repeat
|
|
4342
4624
|
attempt: attempt + 1,
|
|
4343
4625
|
});
|
|
4344
|
-
await driver.notify(`tickmarkr ${runId}: ${t.id} ${repeated.gate} failed identically twice —
|
|
4626
|
+
await driver.notify(`tickmarkr ${runId}: ${t.id} ${repeated.gate} failed identically twice — identical retry banned`, { tier: "attention" });
|
|
4345
4627
|
// The cap takes the ladder's MOVE, never its accounting: this failure still spends the rung it
|
|
4346
4628
|
// would have spent, so a task that cannot converge still reaches ladder exhaustion on exactly
|
|
4347
4629
|
// the budget it always had and the cap can never hand a stuck task extra rounds.
|
|
4348
4630
|
capStep = r.ladder[Math.min(ladderIdx++, r.ladder.length - 1)];
|
|
4349
|
-
journal.append("escalation", t.id, { step: capStep, attempt: attempt + 1, fingerprintCap: true });
|
|
4631
|
+
journal.append("escalation", t.id, { step: capStep, attempt: attempt + 1, fingerprintCap: true, workerAttributed: !!deterministic });
|
|
4350
4632
|
await driver.notify(`tickmarkr ${runId}: ${t.id} escalation: ${capStep}`, { tier: "attention" });
|
|
4633
|
+
const climbed = await climb("gate-fingerprint-cap", repeated, attempt + 1);
|
|
4634
|
+
if (climbed === "parked")
|
|
4635
|
+
return;
|
|
4636
|
+
if (climbed === "climbed")
|
|
4637
|
+
continue;
|
|
4351
4638
|
const v = await runConsult("gate-fail-repeat", output, feedback, results);
|
|
4352
4639
|
// Rule: a terminal cap consult vetoes a same-channel retry, but cannot veto an `escalate`
|
|
4353
4640
|
// rung that already satisfies retry-same-banned by changing channel. Thus terminal+retry
|
|
@@ -4397,12 +4684,21 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4397
4684
|
: r.ladder[Math.min(ladderIdx++, r.ladder.length - 1)]);
|
|
4398
4685
|
if (!capStep) { // a capped failure already journaled and announced the rung it spent
|
|
4399
4686
|
journal.append("escalation", t.id, {
|
|
4400
|
-
step, attempt: attempt + 1,
|
|
4687
|
+
step, attempt: attempt + 1, workerAttributed: !!deterministic,
|
|
4401
4688
|
...(reviewFixRetry && !repairExhausted ? { reviewFix: true } : {}),
|
|
4402
4689
|
...(repair ? { repair: repairHistory.length + 1, repairCharge: repairsDrawn + 1 } : {}),
|
|
4403
4690
|
});
|
|
4404
4691
|
await driver.notify(`tickmarkr ${runId}: ${t.id} escalation: ${step}`, { tier: "attention" });
|
|
4405
4692
|
}
|
|
4693
|
+
const climbCause = repairExhausted ? "repair-exhausted"
|
|
4694
|
+
: deterministic && ownedGateFailures === 2 ? "gate-fail" : undefined;
|
|
4695
|
+
if (climbCause) {
|
|
4696
|
+
const climbed = await climb(climbCause, deterministic ?? failing[0], attempt + 1);
|
|
4697
|
+
if (climbed === "parked")
|
|
4698
|
+
return;
|
|
4699
|
+
if (climbed === "climbed")
|
|
4700
|
+
continue;
|
|
4701
|
+
}
|
|
4406
4702
|
if (step === "retry")
|
|
4407
4703
|
continue;
|
|
4408
4704
|
if (step === "escalate") {
|
|
@@ -4428,14 +4724,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4428
4724
|
const inflight = new Map();
|
|
4429
4725
|
// A settled worker can release a dependency or an approval in the same poll tick.
|
|
4430
4726
|
// Audit the current graph at every empty-flight boundary, never a prior ready snapshot.
|
|
4431
|
-
|
|
4727
|
+
// WB-1 rider: `end-condition-held` names a CLOSE the daemon refused, so only a close attempt journals
|
|
4728
|
+
// it — the post-race sweep still dispatches what it finds but says nothing.
|
|
4729
|
+
const holdEndCondition = (closing = true) => {
|
|
4432
4730
|
sweepLiveApprovals();
|
|
4433
4731
|
const freeSlots = Math.max(0, concurrency - inflight.size);
|
|
4434
4732
|
const dispatchable = readyTasks(graph).filter((t) => !inflight.has(t.id));
|
|
4435
4733
|
if (freeSlots === 0 || dispatchable.length === 0)
|
|
4436
4734
|
return false;
|
|
4437
|
-
|
|
4438
|
-
|
|
4735
|
+
if (closing) {
|
|
4736
|
+
for (const task of dispatchable) {
|
|
4737
|
+
journal.append("end-condition-held", task.id, { deps: task.deps, freeSlots });
|
|
4738
|
+
}
|
|
4439
4739
|
}
|
|
4440
4740
|
return true;
|
|
4441
4741
|
};
|
|
@@ -4446,6 +4746,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4446
4746
|
// must still stop the run before it can dispatch more work or write run-end.
|
|
4447
4747
|
if (termSignal)
|
|
4448
4748
|
throw new Error(`terminated by ${termSignal}`);
|
|
4749
|
+
await watchBoard();
|
|
4449
4750
|
sweepLiveApprovals();
|
|
4450
4751
|
const ready = readyTasks(graph)
|
|
4451
4752
|
.filter((t) => !inflight.has(t.id))
|
|
@@ -4500,12 +4801,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4500
4801
|
const waiters = [...inflight.values(), aborted];
|
|
4501
4802
|
// A free slot is itself a scheduling boundary: poll the append-only approval stream instead of
|
|
4502
4803
|
// sleeping until an unrelated long-running task settles.
|
|
4503
|
-
|
|
4804
|
+
// An open board is polled on the same cadence, so a dead cockpit is noticed mid-task.
|
|
4805
|
+
if (inflight.size < concurrency || boardOpened) {
|
|
4504
4806
|
waiters.push(new Promise((wake) => setTimeout(wake, APPROVAL_POLL_MS)));
|
|
4505
4807
|
}
|
|
4506
4808
|
await Promise.race(waiters); // aborted rejects on termination — unwinds the run
|
|
4507
4809
|
if (inflight.size === 0)
|
|
4508
|
-
holdEndCondition();
|
|
4810
|
+
holdEndCondition(false);
|
|
4509
4811
|
}
|
|
4510
4812
|
// D-07: the sweep now closes only what's LEFT in keptSlots — done-closed worker slots were removed
|
|
4511
4813
|
// (no double-close) and self-cleaned LLM/consult panes were never added under keepLlm:false. This
|
|
@@ -4555,7 +4857,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4555
4857
|
poll = setTimeout(pollApprovals, APPROVAL_POLL_MS);
|
|
4556
4858
|
let tipFailed;
|
|
4557
4859
|
try {
|
|
4558
|
-
const verification =
|
|
4860
|
+
const verification = withCommandContext(undefined, () => verifyIntegrationTipCached(intWt, commands, journal, { lastMergedTask, baseline, signal: controller.signal }), controller.signal);
|
|
4559
4861
|
activeTipVerify = { controller, settled: verification.then(() => { }, () => { }) };
|
|
4560
4862
|
tipFailed = await verification;
|
|
4561
4863
|
}
|