tickmarkr 2.1.5 → 2.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import { createHash, randomBytes } from "node:crypto";
2
2
  import { shq } from "../adapters/types.js";
3
- import { appendFileSync, closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readlinkSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
3
+ import { appendFileSync, closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, readlinkSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
4
4
  import { tmpdir } from "node:os";
5
5
  import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
6
6
  import { stringify } from "yaml";
@@ -15,7 +15,7 @@ import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js
15
15
  import { formatOwnedName } from "../drivers/types.js";
16
16
  import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../gates/baseline.js";
17
17
  import { runGates } from "../gates/run-gates.js";
18
- import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus, taskContentDigest } from "../graph/graph.js";
18
+ import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus, taskContentDigest, tickmarkrDir } from "../graph/graph.js";
19
19
  import { GATE_NAMES } from "../graph/schema.js";
20
20
  import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
21
21
  import { runEnvironment } from "./environment.js";
@@ -23,7 +23,7 @@ import { cleanupRunWorktrees, gitHead, linkNodeModules, npmDependencyInstallComm
23
23
  import { runInteractiveSeed } from "./interactive-seed.js";
24
24
  import { activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingReviewFindings, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, renderStructuredReviewFinding, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
25
25
  import { isDiffCapPark } from "../gates/review.js";
26
- import { acquireApprovalSerialization, acquireRunLock, releaseRunLock } from "./lock.js";
26
+ import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
27
27
  import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
28
28
  import { nextChannel, route } from "../route/router.js";
29
29
  import { desiredPanes } from "./reconcile.js";
@@ -521,6 +521,35 @@ async function gateCommitSubject(base, head, wt) {
521
521
  return head; // fail closed to the exact object id if canonicalization fails
522
522
  return createHash("sha256").update(history.stdout).digest("hex");
523
523
  }
524
+ // A CPU total cannot prove a tree empty: a newly launched live process can still have a measured
525
+ // total of zero at the host's clock resolution. This probe asks the narrower cardinality question
526
+ // OBS-737 needs. A readable process table with no marker root is measured empty; a failed or empty
527
+ // table is unmeasurable. Descendants are closed over PPID because not every child retains the
528
+ // dispatch-script marker in its own argv.
529
+ async function observeWorkerProcessTree(marker, cwd) {
530
+ const snapshot = await shGit("ps -Awwo pid=,ppid=,command=", cwd, 15_000);
531
+ if (snapshot.code !== 0)
532
+ return "unmeasurable";
533
+ const rows = [];
534
+ for (const line of snapshot.stdout.split("\n")) {
535
+ const match = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line);
536
+ if (match)
537
+ rows.push({ pid: match[1], ppid: match[2], command: match[3] });
538
+ }
539
+ if (rows.length === 0)
540
+ return "unmeasurable";
541
+ const tree = new Set(rows.filter((row) => row.command.includes(marker)).map((row) => row.pid));
542
+ for (let grew = true; grew;) {
543
+ grew = false;
544
+ for (const row of rows) {
545
+ if (!tree.has(row.pid) && tree.has(row.ppid)) {
546
+ tree.add(row.pid);
547
+ grew = true;
548
+ }
549
+ }
550
+ }
551
+ return tree.size === 0 ? "empty" : "running";
552
+ }
524
553
  const OBSERVE_CHUNK_BYTES = 64 * 1024;
525
554
  const OBSERVE_BUDGET_BYTES = 256 * 1024 * 1024;
526
555
  let observeBudgetBytes = OBSERVE_BUDGET_BYTES;
@@ -605,6 +634,20 @@ async function boundedGitObservation(command, worktree, budget) {
605
634
  return undefined;
606
635
  return result.stdout;
607
636
  }
637
+ // "No worktree delta" means no staged/unstaged/untracked bytes AND no commits beyond the task's
638
+ // dispatch base. Both reads are bounded and preserve the observer's third state: a failed probe is
639
+ // unreadable, never clean.
640
+ async function observeWorktreeDelta(base, worktree) {
641
+ let budget = observeBudgetBytes;
642
+ const status = await boundedGitObservation("GIT_OPTIONAL_LOCKS=0 git status --porcelain=v1 -z --untracked-files=all", worktree, budget);
643
+ if (status === undefined)
644
+ return "unreadable";
645
+ budget -= Buffer.byteLength(status);
646
+ const ahead = await boundedGitObservation(`GIT_OPTIONAL_LOCKS=0 git rev-list --count ${shq(base)}..HEAD`, worktree, budget);
647
+ if (ahead === undefined || !/^\d+$/.test(ahead.trim()))
648
+ return "unreadable";
649
+ return status.length === 0 && ahead.trim() === "0" ? "unchanged" : "changed";
650
+ }
608
651
  // The signature has four git-owned inputs: HEAD, staged blob/mode/path identity, porcelain path and
609
652
  // state, and the set of worktree paths whose bytes Git cannot supply. Those paths contribute their
610
653
  // filesystem identity, mode, symlink text, and content. A failed or over-budget leg is a third
@@ -721,6 +764,37 @@ function recordFatalRunEnd(journal, runId, branch, err) {
721
764
  console.error(`tickmarkr ${runId}: run crashed — no terminal record written; journal sink unwritable at ${journalPath} (${retryErr instanceof Error ? retryErr.message : String(retryErr)}) — original error: ${original}`);
722
765
  }
723
766
  }
767
+ // OBS-777: the fleet fold cannot distinguish an orphan from another repository's live worker. Give
768
+ // it one repository-scoped snapshot instead: terminal journals are ended, and an open lifecycle is
769
+ // ended only when the shared lock predicate can prove its last recorded daemon pid dead.
770
+ function endedRunIdsForReconcile(repoRoot) {
771
+ const ended = new Set();
772
+ const dir = join(tickmarkrDir(repoRoot), "runs");
773
+ if (!existsSync(dir))
774
+ return ended;
775
+ let runIds;
776
+ try {
777
+ runIds = readdirSync(dir);
778
+ }
779
+ catch {
780
+ return ended;
781
+ }
782
+ for (const runId of runIds) {
783
+ try {
784
+ const events = Journal.open(repoRoot, runId).read();
785
+ if (runHasEnded(events)) {
786
+ ended.add(runId);
787
+ continue;
788
+ }
789
+ const lifecycle = [...events].reverse().find((event) => (event.event === "run-start" || event.event === "run-resume")
790
+ && Number.isInteger(event.data.pid) && event.data.pid > 0);
791
+ if (lifecycle && !isPidLive(lifecycle.data.pid))
792
+ ended.add(runId);
793
+ }
794
+ catch { /* incomplete or foreign run directory — it proves nothing */ }
795
+ }
796
+ return ended;
797
+ }
724
798
  export async function runDaemon(repoRoot, opts = {}) {
725
799
  // v1.51 T2 / OBS-89 (v1.60): retired --quality env seam. Mode resolution owns premium routing;
726
800
  // route() no longer reads the retired env at all, so the old entrypoint scrub is gone with it.
@@ -738,6 +812,7 @@ export async function runDaemon(repoRoot, opts = {}) {
738
812
  // subprocess, so the optional-chain open below is a no-op there). Cosmetic-only: any failure is
739
813
  // swallowed (never affects the run); the operator closes a surviving watch pane.
740
814
  const lock = acquireRunLock(repoRoot, runId);
815
+ const endedRunIds = endedRunIdsForReconcile(repoRoot); // one snapshot for every sweep this caller makes
741
816
  // D10: the lock is this daemon's liveness record and already carries its pid; status prints that
742
817
  // identity beside the supervision row. The `orchestrator` tier belongs exclusively to the seated
743
818
  // supervisor, so a run never beats or stands down that seat's record on the daemon's behalf.
@@ -835,7 +910,7 @@ export async function runDaemon(repoRoot, opts = {}) {
835
910
  // Termination (SIGINT/SIGTERM): record the daemon-controlled exit before closing every live slot,
836
911
  // reconcile owned panes against an EMPTY
837
912
  // desired set (herdr panes not in memory; panesToClose spares foreign names, watch panes, and
838
- // other runs' panes by construction), release the run lock, then exit. There is still no run-end,
913
+ // live/unknown other runs' panes by construction), release the run lock, then exit. There is still no run-end,
839
914
  // so stop-amend-resume keeps resuming; exit-cause distinguishes this deliberate stop from an
840
915
  // observer-classified abrupt death. keepPanes:"forever" (the
841
916
  // keep-everything debug override) preserves panes but still releases the lock and exits.
@@ -864,7 +939,7 @@ export async function runDaemon(repoRoot, opts = {}) {
864
939
  catch { /* cosmetic — reconcile is the backstop */ }
865
940
  }
866
941
  try {
867
- await driver.reconcile?.(new Set(), runId);
942
+ await driver.reconcile?.(new Set(), runId, { endedRunIds });
868
943
  }
869
944
  catch { /* cosmetic — visibility is never a gate */ }
870
945
  }
@@ -1050,7 +1125,7 @@ export async function runDaemon(repoRoot, opts = {}) {
1050
1125
  // herdr's watches bookkeeping lives in close(), and a raw pane-close in the sweep would
1051
1126
  // leave narrator() a stale cache) — the driver always sees it as desired; its lifecycle is
1052
1127
  // decided here from the fold alone.
1053
- await driver.reconcile?.(new Set([...desired, watchName]), runId, opts);
1128
+ await driver.reconcile?.(new Set([...desired, watchName]), runId, { ...opts, endedRunIds });
1054
1129
  // OBS-103: when the fold retires the watch (run-end boundary), close the narrator. The
1055
1130
  // decision keys on the run identity in the pane name — narrator() adopts a prior daemon
1056
1131
  // instance's pane under the same owned name, so a stop→resume cycle's leftover narrator
@@ -1066,7 +1141,10 @@ export async function runDaemon(repoRoot, opts = {}) {
1066
1141
  /* cosmetic — visibility is never a gate */
1067
1142
  }
1068
1143
  };
1069
- await reconcile(); // run start/resume boundary: nothing in flight — full sweep, incl. older runs' leftovers
1144
+ // run start/resume boundary: nothing in flight, so the sweep takes this run's judge/review/consult
1145
+ // panes too. OBS-777: the one startup snapshot lets it reclaim only older runs proven ended, while
1146
+ // every live or unknown run remains spared.
1147
+ await reconcile();
1070
1148
  // v1.4 self-reference guard: a random nonce on the worker trailer AND exit marker. Displayed
1071
1149
  // source/diffs (e.g. a worker editing tickmarkr's own prompt.ts/daemon.ts) can't know it, so an echoed
1072
1150
  // TICKMARKR_RESULT/TICKMARKR_EXIT literal can never premature-harvest the worker. Quote-split keeps the
@@ -1091,10 +1169,10 @@ export async function runDaemon(repoRoot, opts = {}) {
1091
1169
  };
1092
1170
  // gateFails/consults are execTask-scoped counters passed in so a park row is a rich verified-failure
1093
1171
  // observation (e.g. ladder-exhausted + gateFails:4); every task-human row has a closed kind, never prose alone.
1094
- const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
1172
+ const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
1095
1173
  graph = setStatus(graph, t.id, "human");
1096
1174
  saveGraph(repoRoot, graph);
1097
- journal.append("task-human", t.id, { reason, kind });
1175
+ journal.append("task-human", t.id, { ...details, reason, kind });
1098
1176
  if (assignment) {
1099
1177
  // OBS-547: `metered` counts CHARGEABLE metered attempts, so an unchargeable dispatch passes 0 and
1100
1178
  // the count is omitted rather than written as 0 or as `1` beside `attempts: 0` — a row claiming
@@ -1172,6 +1250,22 @@ export async function runDaemon(repoRoot, opts = {}) {
1172
1250
  journal.append("worktree-preserved", t.id, { ref });
1173
1251
  return driver.worktree(repoRoot, taskBranch, taskBase);
1174
1252
  };
1253
+ // A dead worker with a clean checkout still needs a durable recovery handle: there may be no
1254
+ // pane left to identify even the commit it was dispatched from. preserveWorktree deliberately
1255
+ // creates no ref for ordinary clean recreations, so this exceptional terminal path pins HEAD
1256
+ // explicitly under the same recovery namespace. Reconfirm the delta immediately before the
1257
+ // ref write; a change or unreadable recheck withdraws the park.
1258
+ const preserveDeadWorker = async (worktree, taskBase) => {
1259
+ const state = await observeWorktreeDelta(taskBase, worktree);
1260
+ if (state !== "unchanged")
1261
+ return { state };
1262
+ const head = await gitHead(worktree);
1263
+ const ref = `refs/tickmarkr/preserved/${head}`;
1264
+ const updated = await shGit(`git update-ref ${shq(ref)} ${shq(head)}`, worktree);
1265
+ if (updated.code !== 0)
1266
+ throw new Error(`could not preserve dead worker HEAD at ${ref}: ${updated.stderr || updated.stdout}`);
1267
+ return { state, ref };
1268
+ };
1175
1269
  const r = route(t, cfg, channels, profile, undefined, demotedChannels);
1176
1270
  for (const lint of r.lints)
1177
1271
  journal.append("routing-lint", t.id, { lint });
@@ -2184,6 +2278,7 @@ export async function runDaemon(repoRoot, opts = {}) {
2184
2278
  // subprocess tree that REACHED its exit marker, not about what the worker claimed.
2185
2279
  let processExited = false;
2186
2280
  let earlyLaunchDead = false;
2281
+ let deadWorkerPark;
2187
2282
  let settleParsed;
2188
2283
  let seedResult;
2189
2284
  // v1.22 T5 / OBS-19: auto-answer a fingerprint-matched trust dialog exactly once per slot.
@@ -2323,6 +2418,8 @@ export async function runDaemon(repoRoot, opts = {}) {
2323
2418
  let quotaStreak = 0;
2324
2419
  let rowSaturationHeld = false; // journaled once per attempt when the kill stands down
2325
2420
  let cpuHeld = false; // likewise for the CPU leg's stand-down (OBS-548)
2421
+ let paneReadHeld = false;
2422
+ let paneStatusHeld = false;
2326
2423
  while (Date.now() - lastProgressAt < stallWindowMs) {
2327
2424
  const sliceStart = Date.now();
2328
2425
  const remaining = stallWindowMs - (sliceStart - lastProgressAt);
@@ -2350,7 +2447,39 @@ export async function runDaemon(repoRoot, opts = {}) {
2350
2447
  break;
2351
2448
  }
2352
2449
  }
2353
- const paneText = await driver.read(slot, PANE_READ_ROWS);
2450
+ // A failed pane read is absence of evidence, never evidence of an absent pane. Keep the
2451
+ // rolling taskTimeoutMinutes window as the backstop and name the held probe once.
2452
+ let paneText;
2453
+ try {
2454
+ paneText = await driver.read(slot, PANE_READ_ROWS);
2455
+ }
2456
+ catch (error) {
2457
+ // Preserve the pre-existing exception path when the independent status probe still
2458
+ // sees a pane. The outer attempt finally owns accountant cleanup on that path. Only
2459
+ // an undetectable status makes the read failure relevant to the death detector, and
2460
+ // that genuinely unmeasurable pair fails open to the rolling timeout.
2461
+ let paneUndetectable = true;
2462
+ try {
2463
+ paneUndetectable = await driver.status(slot) === "unknown";
2464
+ }
2465
+ catch {
2466
+ // Two unreadable pane probes are still unmeasurable, never proof of death.
2467
+ }
2468
+ if (!paneUndetectable)
2469
+ throw error;
2470
+ if (!paneReadHeld) {
2471
+ paneReadHeld = true;
2472
+ journal.append("worker-dead-held", t.id, {
2473
+ slot: slot.name, attempt, reason: "pane-read-unreadable",
2474
+ error: error instanceof Error ? error.message : String(error),
2475
+ });
2476
+ }
2477
+ await armCpuLeg(false);
2478
+ const spent = Date.now() - sliceStart;
2479
+ if (spent < slice)
2480
+ await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
2481
+ continue;
2482
+ }
2354
2483
  if (paneText.length > 0)
2355
2484
  everHadOutput = true;
2356
2485
  // OBS-117 (v1.71 T6): zero raw output by the early-launch deadline is a dead channel now.
@@ -2419,7 +2548,24 @@ export async function runDaemon(repoRoot, opts = {}) {
2419
2548
  // gate held. page on "idle" too: herdr's blocked-scrape is strict and proved flaky
2420
2549
  // for TUI dialogs (live check: cursor's trust dialog scraped as idle).
2421
2550
  // "unknown"/"working" never page.
2422
- const st = await driver.status(slot);
2551
+ let st;
2552
+ try {
2553
+ st = await driver.status(slot);
2554
+ }
2555
+ catch (error) {
2556
+ if (!paneStatusHeld) {
2557
+ paneStatusHeld = true;
2558
+ journal.append("worker-dead-held", t.id, {
2559
+ slot: slot.name, attempt, reason: "pane-status-unreadable",
2560
+ error: error instanceof Error ? error.message : String(error),
2561
+ });
2562
+ }
2563
+ await armCpuLeg(false);
2564
+ const spent = Date.now() - sliceStart;
2565
+ if (spent < slice)
2566
+ await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
2567
+ continue;
2568
+ }
2423
2569
  if (st !== lastStatus) {
2424
2570
  lastStatus = st;
2425
2571
  journal.append("worker-status", t.id, { slot: slot.name, status: st, attempt });
@@ -2459,6 +2605,95 @@ export async function runDaemon(repoRoot, opts = {}) {
2459
2605
  // daemon has nothing left to do — the pane falls back under the fast-kill and page
2460
2606
  // watchdogs like any other, instead of riding the whole rolling window untended.
2461
2607
  const nudgePending = nudgeable && (!nudged || nudgeDeadline !== undefined);
2608
+ // OBS-737: disposition for the one fully measured death state. `unknown` alone is not
2609
+ // absence (status parsing can fail), and an empty read alone is not absence (a live pane
2610
+ // can be quiet); together they are the existing driver-level pane absence witness. The
2611
+ // process probe and both worktree observations retain their own third states. Only the
2612
+ // explicit conjunction parks; every other state falls through to the unchanged rolling
2613
+ // timeout below. Seeded launches are excluded because their process marker is knowingly
2614
+ // unmeasurable (armCpuLeg documents that contract above).
2615
+ const paneAbsentCandidate = st === "unknown" && paneText.trim().length === 0;
2616
+ // A subprocess can exit between waitOutput and read while its stdout is still draining.
2617
+ // Confirm emptiness in this same poll before paying for `ps`; then confirm once more
2618
+ // after the process probe yielded the event loop. Any bytes or read error withdraw the
2619
+ // absence witness, so a fast completed worker cannot be parked in that drain race.
2620
+ let paneAbsent = paneAbsentCandidate;
2621
+ if (paneAbsent) {
2622
+ try {
2623
+ const confirmation = await driver.read(slot, PANE_READ_ROWS);
2624
+ paneAbsent = confirmation.trim().length === 0;
2625
+ if (!paneAbsent) {
2626
+ everHadOutput = true;
2627
+ if (stallProgress.observe({ paneText: confirmation, contextTokens }))
2628
+ lastProgressAt = Date.now();
2629
+ }
2630
+ }
2631
+ catch (error) {
2632
+ paneAbsent = false;
2633
+ if (!paneReadHeld) {
2634
+ paneReadHeld = true;
2635
+ journal.append("worker-dead-held", t.id, {
2636
+ slot: slot.name, attempt, reason: "pane-read-unreadable",
2637
+ error: error instanceof Error ? error.message : String(error),
2638
+ });
2639
+ }
2640
+ }
2641
+ }
2642
+ const processTree = paneAbsent && worktreeSinceLaunch === "unchanged" && !hasSeed
2643
+ ? await observeWorkerProcessTree(dispatchScript, wt)
2644
+ : "unmeasurable";
2645
+ if (processTree === "empty") {
2646
+ try {
2647
+ const confirmation = await driver.read(slot, PANE_READ_ROWS);
2648
+ paneAbsent = confirmation.trim().length === 0;
2649
+ if (!paneAbsent) {
2650
+ everHadOutput = true;
2651
+ if (stallProgress.observe({ paneText: confirmation, contextTokens }))
2652
+ lastProgressAt = Date.now();
2653
+ }
2654
+ }
2655
+ catch (error) {
2656
+ paneAbsent = false;
2657
+ if (!paneReadHeld) {
2658
+ paneReadHeld = true;
2659
+ journal.append("worker-dead-held", t.id, {
2660
+ slot: slot.name, attempt, reason: "pane-read-unreadable",
2661
+ error: error instanceof Error ? error.message : String(error),
2662
+ });
2663
+ }
2664
+ }
2665
+ }
2666
+ const worktreeDelta = processTree === "empty"
2667
+ && paneAbsent ? await observeWorktreeDelta(taskBase, wt)
2668
+ : "unreadable";
2669
+ // The first process snapshot can race a just-starting child after the dispatch pane
2670
+ // disappeared. Re-read it after the pane and worktree legs have both held: preservation
2671
+ // is terminal, so a process appearing in that interval must withdraw the park rather
2672
+ // than be orphaned by it. The final worktree recheck remains inside preserveDeadWorker.
2673
+ const confirmedProcessTree = worktreeDelta === "unchanged"
2674
+ ? await observeWorkerProcessTree(dispatchScript, wt)
2675
+ : "unmeasurable";
2676
+ const deathCertain = paneAbsent
2677
+ && processTree === "empty"
2678
+ && confirmedProcessTree === "empty"
2679
+ && worktreeDelta === "unchanged";
2680
+ if (deathCertain) {
2681
+ const preservation = await preserveDeadWorker(wt, taskBase);
2682
+ if (!preservation.ref) {
2683
+ journal.append("worker-dead-held", t.id, {
2684
+ slot: slot.name, attempt, reason: `worktree-${preservation.state}`,
2685
+ });
2686
+ continue;
2687
+ }
2688
+ const ref = preservation.ref;
2689
+ const reason = `worker is unambiguously dead: pane absent, process tree empty, and worktree unchanged; preserved at ${ref}`;
2690
+ deadWorkerPark = { ref, reason };
2691
+ journal.append("worktree-preserved", t.id, { ref });
2692
+ journal.append("worker-dead-held", t.id, {
2693
+ slot: slot.name, attempt, reason: "unambiguous-worker-death", ref,
2694
+ });
2695
+ break;
2696
+ }
2462
2697
  // T1 review fix: the kill's "no output growth" leg clocks off the RAW growth signals,
2463
2698
  // never lastProgressAt alone — the flat-token rule (stall.ts) deliberately suppresses
2464
2699
  // the re-arm report on row growth once tokens stick, and contextTokens is sticky across
@@ -2632,7 +2867,13 @@ export async function runDaemon(repoRoot, opts = {}) {
2632
2867
  if (!finished && exitCode === null) {
2633
2868
  // timed out (or only ever saw false positives): harvest whatever the pane holds now
2634
2869
  timedOut = Date.now() - lastProgressAt >= stallWindowMs;
2635
- output = await driver.read(slot, PANE_READ_ROWS);
2870
+ try {
2871
+ output = await driver.read(slot, PANE_READ_ROWS);
2872
+ }
2873
+ catch {
2874
+ // The poll loop already recorded the unreadable pane. Retain the last readable bytes
2875
+ // so this ambiguous path still reaches the ordinary timeout/consult backstop.
2876
+ }
2636
2877
  finished = new RegExp(trailerPattern(nonce)).test(output);
2637
2878
  const exit = exitRe.exec(output);
2638
2879
  exitCode = exit ? Number(exit[1]) : null;
@@ -2766,6 +3007,10 @@ export async function runDaemon(repoRoot, opts = {}) {
2766
3007
  tokens = addUsage(tokens, attemptUsage);
2767
3008
  metered++;
2768
3009
  }
3010
+ if (deadWorkerPark) {
3011
+ await park(t, deadWorkerPark.reason, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { ref: deadWorkerPark.ref });
3012
+ return;
3013
+ }
2769
3014
  let result = settleParsed ?? adapter.parse(output, nonce);
2770
3015
  const workerFinished = finished;
2771
3016
  const workerCause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut, deadChannel: deadChannelKilled });
@@ -29,6 +29,11 @@ export declare const ATTEMPT_CAP_RELEASE: "attempt-cap";
29
29
  export declare const GATE_SATISFIED_RELEASE: "gate-satisfied";
30
30
  export declare const REVIEW_UPHELD_RELEASE: "review-upheld";
31
31
  export declare const RECHECK_RELEASE: "recheck";
32
+ export interface PreservedRef {
33
+ ref: string;
34
+ diffCommand: string;
35
+ }
36
+ export declare function preservedRefsByTask(events: JournalEvent[]): Map<string, PreservedRef[]>;
32
37
  export declare function reviewRoundsSinceApproval(events: JournalEvent[], taskId: string): number;
33
38
  export declare function upheldFeedbackByTask(events: JournalEvent[]): Map<string, string>;
34
39
  export interface StructuredFinding {
@@ -2,11 +2,12 @@ import { AsyncLocalStorage } from "node:async_hooks";
2
2
  import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
3
3
  import { join } from "node:path";
4
4
  import { z } from "zod";
5
- import { channelKey, TokenUsageSchema } from "../adapters/types.js";
5
+ import { channelKey, shq, TokenUsageSchema } from "../adapters/types.js";
6
6
  import { stateDirName, taskContentDigest, tickmarkrDir } from "../graph/graph.js";
7
7
  import { GATE_NAMES, TIERS } from "../graph/schema.js";
8
8
  import { buildProfile, classify } from "../route/profile.js";
9
9
  import { DecisionEventSchema, trackJournalRows, } from "./protocol.js";
10
+ import { normalizeGateOutcome } from "./outcome.js";
10
11
  import { redactSecrets } from "./redact.js";
11
12
  export function phaseForGate(gate) {
12
13
  if (gate === "acceptance")
@@ -15,6 +16,23 @@ export function phaseForGate(gate) {
15
16
  return "review";
16
17
  return `gate:${gate}`;
17
18
  }
19
+ /** The pipe projects the same normalized gate outcome as the TTY rail. The selected-test field only
20
+ * chooses the held outcome's operator noun after the accessor has classified it; it is not a second
21
+ * verdict read. */
22
+ const journalGateDetail = (data) => {
23
+ if (typeof data.gate !== "string")
24
+ return undefined;
25
+ switch (normalizeGateOutcome(data).kind) {
26
+ case "passed":
27
+ return `${data.gate} passed`;
28
+ case "failed":
29
+ return `${data.gate} failed`;
30
+ case "held":
31
+ return Array.isArray(data.selectedTests) ? `${data.gate} selected-test screen` : `${data.gate} held`;
32
+ default:
33
+ return data.gate;
34
+ }
35
+ };
18
36
  export function formatJournalNarration({ event, taskId, data }) {
19
37
  const assignment = data.assignment;
20
38
  const direct = [data.summary, data.reason, data.error, data.step, data.action, data.lint, data.branch, data.from]
@@ -26,7 +44,9 @@ export function formatJournalNarration({ event, taskId, data }) {
26
44
  ? `${data.gate} failed${typeof data.lastMergedTask === "string" ? ` after ${data.lastMergedTask}` : ""}`
27
45
  : event === "tip-verify"
28
46
  ? `${data.gate} passed`
29
- : `${data.gate}${data.pass === true ? " passed" : data.pass === false ? " failed" : ""}`
47
+ : event === "gate-result"
48
+ ? journalGateDetail(data)
49
+ : `${data.gate}`
30
50
  : typeof data.code === "number" ? `exit ${data.code}`
31
51
  : typeof data.pid === "number" ? `pid ${data.pid}`
32
52
  : typeof data.baseRef === "string" ? `base ${data.baseRef.slice(0, 12)}`
@@ -56,6 +76,24 @@ export const REVIEW_UPHELD_RELEASE = "review-upheld";
56
76
  // so the corrected declaration is the thing that earns the green. Budget semantics match attempt-cap
57
77
  // (fresh attempts, tried survives) because the park cost the task its remaining budget.
58
78
  export const RECHECK_RELEASE = "recheck";
79
+ // OBS-738: one authority for every recovery surface. The ref is accepted only from the row that
80
+ // preservation itself writes; task-human prose, branch heads and commit history are deliberately
81
+ // absent from this fold. Keep every row in journal order — one task can be recreated more than once,
82
+ // and the terminal record owes the operator every resulting recovery handle, not merely the newest.
83
+ export function preservedRefsByTask(events) {
84
+ const byTask = new Map();
85
+ for (const event of events) {
86
+ if (event.event !== "worktree-preserved" || !event.taskId || typeof event.data.ref !== "string"
87
+ || event.data.ref === "")
88
+ continue;
89
+ const ref = event.data.ref;
90
+ byTask.set(event.taskId, [
91
+ ...(byTask.get(event.taskId) ?? []),
92
+ { ref, diffCommand: `git diff ${shq(`${ref}^!`)}` },
93
+ ]);
94
+ }
95
+ return byTask;
96
+ }
59
97
  // OBS-189: review rounds are scoped to the current ENGAGEMENT — the stretch since the newest operator
60
98
  // approval for the task. A whole-journal count re-parks an upheld task before its funded attempt can
61
99
  // dispatch (measured live on run-20260726-213539), making a fresh journal the only escape. A T15
@@ -1069,19 +1107,36 @@ export class Journal {
1069
1107
  ? undefined
1070
1108
  : DecisionEventSchema.parse({ ...eventOrDecision, ts: new Date().toISOString() });
1071
1109
  const event = decisionRow?.event ?? eventOrDecision;
1110
+ const rowTaskId = decisionRow && "taskId" in decisionRow ? decisionRow.taskId : taskId;
1072
1111
  const inputData = decisionRow?.data ?? data;
1112
+ // OBS-738: terminal and resume records reduce the journal that precedes them. Neither re-derives
1113
+ // recovery facts from task-human prose: preserved refs come from preservedRefsByTask, and the
1114
+ // upheld brief comes from the established prompt/replay reducer.
1115
+ const priorEvents = event === "run-end" || event === "resume-restore" ? this.read() : [];
1116
+ const reducedData = event === "run-end"
1117
+ ? (() => {
1118
+ const preservedRefs = [...preservedRefsByTask(priorEvents)].flatMap(([preservedTaskId, refs]) => refs.map(({ ref, diffCommand }) => ({ taskId: preservedTaskId, ref, diffCommand })));
1119
+ return preservedRefs.length > 0 ? { ...inputData, preservedRefs } : inputData;
1120
+ })()
1121
+ : event === "resume-restore" && rowTaskId && upheldFeedbackByTask(priorEvents).has(rowTaskId)
1122
+ ? {
1123
+ ...inputData,
1124
+ upheldFeedbackRestoredFor: rowTaskId,
1125
+ summary: `upheld feedback restored for ${rowTaskId}`,
1126
+ }
1127
+ : inputData;
1073
1128
  const evidence = judgePersistence.getStore();
1074
1129
  const failed = evidence?.invocations.filter((invocation) => invocation.transcript !== undefined) ?? [];
1075
1130
  const persistedData = event === "judge-retry" && failed.length > 0
1076
1131
  ? {
1077
- ...inputData,
1132
+ ...reducedData,
1078
1133
  transcript: failed[0].transcript,
1079
1134
  ...(failed[1] ? { retryTranscript: failed[1].transcript } : {}),
1080
1135
  }
1081
- : inputData;
1082
- const row = decisionRow ?? {
1083
- ts: new Date().toISOString(), event, ...(taskId ? { taskId } : {}), data: persistedData,
1084
- };
1136
+ : reducedData;
1137
+ const row = decisionRow
1138
+ ? { ...decisionRow, data: persistedData }
1139
+ : { ts: new Date().toISOString(), event, ...(taskId ? { taskId } : {}), data: persistedData };
1085
1140
  // T3 secret redaction: only the persisted bytes are masked — the caller's data stays untouched in
1086
1141
  // memory. The narrator receives the persisted (masked) row so a pane sink never shows a credential.
1087
1142
  const line = redactSecrets(JSON.stringify(row));
@@ -88,7 +88,14 @@ export function normalizeGateOutcome(source) {
88
88
  return { kind: "unavailable", reason: `malformed gate result: expected an object, read ${shape}` };
89
89
  }
90
90
  const row = source;
91
- // `kind` selects the canonical format. It must validate as exactly one canonical arm without any
91
+ // A gate-result data wrapper states its canonical verdict in `outcome`. Consumers may pass either
92
+ // the wrapper or the outcome itself; both must read the same kind even if stale bare fields remain.
93
+ if (hasOwn(row, "outcome")) {
94
+ if (isGateOutcome(row.outcome))
95
+ return row.outcome;
96
+ return { kind: "unavailable", reason: reasonOf(row, "malformed gate result: invalid explicit outcome") };
97
+ }
98
+ // `kind` selects the canonical outcome object. It must validate as exactly one canonical arm without any
92
99
  // legacy truth discriminator; otherwise continuing through the legacy branches could reinterpret a
93
100
  // contradictory dual-write as passed, failed or infra. Mixed and malformed canonical rows stay
94
101
  // explicitly unavailable, retaining any reason/details the producer did manage to state.
@@ -3,6 +3,8 @@ export declare const SUPERVISION_BEAT_MS = 10000;
3
3
  /** Ceiling before a beat reads STALE: SIX beats, lock.ts's ratio — five may be missed before alarm. */
4
4
  export declare const SUPERVISION_STALE_MS: number;
5
5
  export declare const SUPERVISION_FUTURE_GRACE_MS = 1000;
6
+ /** Context watchers act at 75% unless their invocation declares another threshold. */
7
+ export declare const SUPERVISION_DEFAULT_THRESHOLD_PCT = 75;
6
8
  export declare const SUPERVISION_TIERS: readonly ["orchestrator", "orchestrator-context", "overseer", "overseer-context", "watch"];
7
9
  export type SupervisionTier = (typeof SUPERVISION_TIERS)[number];
8
10
  export declare const SUPERVISION_SEAT_TIERS: readonly ["orchestrator", "orchestrator-context", "overseer", "overseer-context"];
@@ -17,16 +19,28 @@ export interface TierLiveness {
17
19
  beatAgeMs?: number;
18
20
  /** The seat the record names, when it names one. Always present on a seat tier that is not ABSENT. */
19
21
  seat?: string;
22
+ /** The alarm's durable duty, orthogonal to whether its watcher is armed, stale or stood down. */
23
+ clearOwedSince?: string;
24
+ /** The latch exists but cannot be decoded; never mistaken for no duty owed. */
25
+ clearOwedUnreadable?: true;
26
+ }
27
+ /** A watcher observation that can raise or discharge the seat's clear obligation. */
28
+ export interface SupervisionBeatObservation {
29
+ armId: string;
30
+ pct: number;
31
+ thresholdPct: number;
20
32
  }
21
33
  export declare const supervisionBeatPath: (repoRoot: string, tier: SupervisionTier) => string;
22
34
  /** Where a watcher records that it STOOD DOWN. Its own file: the beat keeps meaning only "alive". */
23
35
  export declare const supervisionStandDownPath: (repoRoot: string, tier: SupervisionTier) => string;
24
- export declare function beatSupervision(repoRoot: string, tier: SupervisionTier, seat?: string): void;
36
+ export declare function beatSupervision(repoRoot: string, tier: SupervisionTier, seat?: string, observation?: SupervisionBeatObservation): void;
25
37
  /** Handle a watcher holds for as long as it is supervising; disarm stands it down and is idempotent. */
26
38
  export interface ArmedSupervision {
27
39
  disarm: () => void;
28
40
  }
29
41
  export declare function armSupervision(repoRoot: string, tier: SupervisionTier, beatMs?: number, seat?: string): ArmedSupervision;
42
+ /** Arm the live status board's own tier through the normal supervision writer and lifecycle. */
43
+ export declare function armWatchSupervision(repoRoot: string, beatMs?: number): ArmedSupervision;
30
44
  export declare function readTierLiveness(repoRoot: string, tier: SupervisionTier, now?: number): TierLiveness;
31
45
  export declare function supervisionStatus(repoRoot: string, tier: SupervisionTier, now?: number): TierLiveness;
32
46
  /** Every KNOWN tier, always — a tier omitted from this list would read as one that is fine. */