tickmarkr 2.1.5 → 2.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/beat.js +37 -10
- package/dist/cli/commands/run.js +21 -2
- package/dist/cli/commands/stats.d.ts +21 -0
- package/dist/cli/commands/stats.js +200 -0
- package/dist/cli/commands/status.js +51 -19
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +3 -1
- package/dist/compile/index.d.ts +1 -1
- package/dist/compile/index.js +14 -5
- package/dist/compile/ownership.d.ts +25 -0
- package/dist/compile/ownership.js +142 -0
- package/dist/drivers/herdr.d.ts +1 -0
- package/dist/drivers/herdr.js +7 -6
- package/dist/drivers/types.d.ts +2 -0
- package/dist/drivers/types.js +54 -14
- package/dist/gates/baseline.js +39 -18
- package/dist/run/daemon.js +257 -12
- package/dist/run/journal.d.ts +5 -0
- package/dist/run/journal.js +62 -7
- package/dist/run/outcome.js +8 -1
- package/dist/run/supervision.d.ts +15 -1
- package/dist/run/supervision.js +112 -14
- package/package.json +1 -1
- package/skills/tickmarkr-auto/SKILL.md +2 -1
- package/skills/tickmarkr-loop/SKILL.md +2 -1
- package/skills/tickmarkr-overseer/SKILL.md +21 -1
- package/skills/tickmarkr-overseer/scripts/watch-context.sh +46 -32
package/dist/run/daemon.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
2
|
import { shq } from "../adapters/types.js";
|
|
3
|
-
import { appendFileSync, closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readlinkSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { appendFileSync, closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, readlinkSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
6
|
import { stringify } from "yaml";
|
|
@@ -15,7 +15,7 @@ import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js
|
|
|
15
15
|
import { formatOwnedName } from "../drivers/types.js";
|
|
16
16
|
import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../gates/baseline.js";
|
|
17
17
|
import { runGates } from "../gates/run-gates.js";
|
|
18
|
-
import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus, taskContentDigest } from "../graph/graph.js";
|
|
18
|
+
import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus, taskContentDigest, tickmarkrDir } from "../graph/graph.js";
|
|
19
19
|
import { GATE_NAMES } from "../graph/schema.js";
|
|
20
20
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
21
21
|
import { runEnvironment } from "./environment.js";
|
|
@@ -23,7 +23,7 @@ import { cleanupRunWorktrees, gitHead, linkNodeModules, npmDependencyInstallComm
|
|
|
23
23
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
24
24
|
import { activeRetryBan, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingReviewFindings, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, renderStructuredReviewFinding, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
25
25
|
import { isDiffCapPark } from "../gates/review.js";
|
|
26
|
-
import { acquireApprovalSerialization, acquireRunLock, releaseRunLock } from "./lock.js";
|
|
26
|
+
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
27
27
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
28
28
|
import { nextChannel, route } from "../route/router.js";
|
|
29
29
|
import { desiredPanes } from "./reconcile.js";
|
|
@@ -521,6 +521,35 @@ async function gateCommitSubject(base, head, wt) {
|
|
|
521
521
|
return head; // fail closed to the exact object id if canonicalization fails
|
|
522
522
|
return createHash("sha256").update(history.stdout).digest("hex");
|
|
523
523
|
}
|
|
524
|
+
// A CPU total cannot prove a tree empty: a newly launched live process can still have a measured
|
|
525
|
+
// total of zero at the host's clock resolution. This probe asks the narrower cardinality question
|
|
526
|
+
// OBS-737 needs. A readable process table with no marker root is measured empty; a failed or empty
|
|
527
|
+
// table is unmeasurable. Descendants are closed over PPID because not every child retains the
|
|
528
|
+
// dispatch-script marker in its own argv.
|
|
529
|
+
async function observeWorkerProcessTree(marker, cwd) {
|
|
530
|
+
const snapshot = await shGit("ps -Awwo pid=,ppid=,command=", cwd, 15_000);
|
|
531
|
+
if (snapshot.code !== 0)
|
|
532
|
+
return "unmeasurable";
|
|
533
|
+
const rows = [];
|
|
534
|
+
for (const line of snapshot.stdout.split("\n")) {
|
|
535
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line);
|
|
536
|
+
if (match)
|
|
537
|
+
rows.push({ pid: match[1], ppid: match[2], command: match[3] });
|
|
538
|
+
}
|
|
539
|
+
if (rows.length === 0)
|
|
540
|
+
return "unmeasurable";
|
|
541
|
+
const tree = new Set(rows.filter((row) => row.command.includes(marker)).map((row) => row.pid));
|
|
542
|
+
for (let grew = true; grew;) {
|
|
543
|
+
grew = false;
|
|
544
|
+
for (const row of rows) {
|
|
545
|
+
if (!tree.has(row.pid) && tree.has(row.ppid)) {
|
|
546
|
+
tree.add(row.pid);
|
|
547
|
+
grew = true;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
return tree.size === 0 ? "empty" : "running";
|
|
552
|
+
}
|
|
524
553
|
const OBSERVE_CHUNK_BYTES = 64 * 1024;
|
|
525
554
|
const OBSERVE_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
526
555
|
let observeBudgetBytes = OBSERVE_BUDGET_BYTES;
|
|
@@ -605,6 +634,20 @@ async function boundedGitObservation(command, worktree, budget) {
|
|
|
605
634
|
return undefined;
|
|
606
635
|
return result.stdout;
|
|
607
636
|
}
|
|
637
|
+
// "No worktree delta" means no staged/unstaged/untracked bytes AND no commits beyond the task's
|
|
638
|
+
// dispatch base. Both reads are bounded and preserve the observer's third state: a failed probe is
|
|
639
|
+
// unreadable, never clean.
|
|
640
|
+
async function observeWorktreeDelta(base, worktree) {
|
|
641
|
+
let budget = observeBudgetBytes;
|
|
642
|
+
const status = await boundedGitObservation("GIT_OPTIONAL_LOCKS=0 git status --porcelain=v1 -z --untracked-files=all", worktree, budget);
|
|
643
|
+
if (status === undefined)
|
|
644
|
+
return "unreadable";
|
|
645
|
+
budget -= Buffer.byteLength(status);
|
|
646
|
+
const ahead = await boundedGitObservation(`GIT_OPTIONAL_LOCKS=0 git rev-list --count ${shq(base)}..HEAD`, worktree, budget);
|
|
647
|
+
if (ahead === undefined || !/^\d+$/.test(ahead.trim()))
|
|
648
|
+
return "unreadable";
|
|
649
|
+
return status.length === 0 && ahead.trim() === "0" ? "unchanged" : "changed";
|
|
650
|
+
}
|
|
608
651
|
// The signature has four git-owned inputs: HEAD, staged blob/mode/path identity, porcelain path and
|
|
609
652
|
// state, and the set of worktree paths whose bytes Git cannot supply. Those paths contribute their
|
|
610
653
|
// filesystem identity, mode, symlink text, and content. A failed or over-budget leg is a third
|
|
@@ -721,6 +764,37 @@ function recordFatalRunEnd(journal, runId, branch, err) {
|
|
|
721
764
|
console.error(`tickmarkr ${runId}: run crashed — no terminal record written; journal sink unwritable at ${journalPath} (${retryErr instanceof Error ? retryErr.message : String(retryErr)}) — original error: ${original}`);
|
|
722
765
|
}
|
|
723
766
|
}
|
|
767
|
+
// OBS-777: the fleet fold cannot distinguish an orphan from another repository's live worker. Give
|
|
768
|
+
// it one repository-scoped snapshot instead: terminal journals are ended, and an open lifecycle is
|
|
769
|
+
// ended only when the shared lock predicate can prove its last recorded daemon pid dead.
|
|
770
|
+
function endedRunIdsForReconcile(repoRoot) {
|
|
771
|
+
const ended = new Set();
|
|
772
|
+
const dir = join(tickmarkrDir(repoRoot), "runs");
|
|
773
|
+
if (!existsSync(dir))
|
|
774
|
+
return ended;
|
|
775
|
+
let runIds;
|
|
776
|
+
try {
|
|
777
|
+
runIds = readdirSync(dir);
|
|
778
|
+
}
|
|
779
|
+
catch {
|
|
780
|
+
return ended;
|
|
781
|
+
}
|
|
782
|
+
for (const runId of runIds) {
|
|
783
|
+
try {
|
|
784
|
+
const events = Journal.open(repoRoot, runId).read();
|
|
785
|
+
if (runHasEnded(events)) {
|
|
786
|
+
ended.add(runId);
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
const lifecycle = [...events].reverse().find((event) => (event.event === "run-start" || event.event === "run-resume")
|
|
790
|
+
&& Number.isInteger(event.data.pid) && event.data.pid > 0);
|
|
791
|
+
if (lifecycle && !isPidLive(lifecycle.data.pid))
|
|
792
|
+
ended.add(runId);
|
|
793
|
+
}
|
|
794
|
+
catch { /* incomplete or foreign run directory — it proves nothing */ }
|
|
795
|
+
}
|
|
796
|
+
return ended;
|
|
797
|
+
}
|
|
724
798
|
export async function runDaemon(repoRoot, opts = {}) {
|
|
725
799
|
// v1.51 T2 / OBS-89 (v1.60): retired --quality env seam. Mode resolution owns premium routing;
|
|
726
800
|
// route() no longer reads the retired env at all, so the old entrypoint scrub is gone with it.
|
|
@@ -738,6 +812,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
738
812
|
// subprocess, so the optional-chain open below is a no-op there). Cosmetic-only: any failure is
|
|
739
813
|
// swallowed (never affects the run); the operator closes a surviving watch pane.
|
|
740
814
|
const lock = acquireRunLock(repoRoot, runId);
|
|
815
|
+
const endedRunIds = endedRunIdsForReconcile(repoRoot); // one snapshot for every sweep this caller makes
|
|
741
816
|
// D10: the lock is this daemon's liveness record and already carries its pid; status prints that
|
|
742
817
|
// identity beside the supervision row. The `orchestrator` tier belongs exclusively to the seated
|
|
743
818
|
// supervisor, so a run never beats or stands down that seat's record on the daemon's behalf.
|
|
@@ -835,7 +910,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
835
910
|
// Termination (SIGINT/SIGTERM): record the daemon-controlled exit before closing every live slot,
|
|
836
911
|
// reconcile owned panes against an EMPTY
|
|
837
912
|
// desired set (herdr panes not in memory; panesToClose spares foreign names, watch panes, and
|
|
838
|
-
// other runs' panes by construction), release the run lock, then exit. There is still no run-end,
|
|
913
|
+
// live/unknown other runs' panes by construction), release the run lock, then exit. There is still no run-end,
|
|
839
914
|
// so stop-amend-resume keeps resuming; exit-cause distinguishes this deliberate stop from an
|
|
840
915
|
// observer-classified abrupt death. keepPanes:"forever" (the
|
|
841
916
|
// keep-everything debug override) preserves panes but still releases the lock and exits.
|
|
@@ -864,7 +939,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
864
939
|
catch { /* cosmetic — reconcile is the backstop */ }
|
|
865
940
|
}
|
|
866
941
|
try {
|
|
867
|
-
await driver.reconcile?.(new Set(), runId);
|
|
942
|
+
await driver.reconcile?.(new Set(), runId, { endedRunIds });
|
|
868
943
|
}
|
|
869
944
|
catch { /* cosmetic — visibility is never a gate */ }
|
|
870
945
|
}
|
|
@@ -1050,7 +1125,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1050
1125
|
// herdr's watches bookkeeping lives in close(), and a raw pane-close in the sweep would
|
|
1051
1126
|
// leave narrator() a stale cache) — the driver always sees it as desired; its lifecycle is
|
|
1052
1127
|
// decided here from the fold alone.
|
|
1053
|
-
await driver.reconcile?.(new Set([...desired, watchName]), runId, opts);
|
|
1128
|
+
await driver.reconcile?.(new Set([...desired, watchName]), runId, { ...opts, endedRunIds });
|
|
1054
1129
|
// OBS-103: when the fold retires the watch (run-end boundary), close the narrator. The
|
|
1055
1130
|
// decision keys on the run identity in the pane name — narrator() adopts a prior daemon
|
|
1056
1131
|
// instance's pane under the same owned name, so a stop→resume cycle's leftover narrator
|
|
@@ -1066,7 +1141,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1066
1141
|
/* cosmetic — visibility is never a gate */
|
|
1067
1142
|
}
|
|
1068
1143
|
};
|
|
1069
|
-
|
|
1144
|
+
// run start/resume boundary: nothing in flight, so the sweep takes this run's judge/review/consult
|
|
1145
|
+
// panes too. OBS-777: the one startup snapshot lets it reclaim only older runs proven ended, while
|
|
1146
|
+
// every live or unknown run remains spared.
|
|
1147
|
+
await reconcile();
|
|
1070
1148
|
// v1.4 self-reference guard: a random nonce on the worker trailer AND exit marker. Displayed
|
|
1071
1149
|
// source/diffs (e.g. a worker editing tickmarkr's own prompt.ts/daemon.ts) can't know it, so an echoed
|
|
1072
1150
|
// TICKMARKR_RESULT/TICKMARKR_EXIT literal can never premature-harvest the worker. Quote-split keeps the
|
|
@@ -1091,10 +1169,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1091
1169
|
};
|
|
1092
1170
|
// gateFails/consults are execTask-scoped counters passed in so a park row is a rich verified-failure
|
|
1093
1171
|
// observation (e.g. ladder-exhausted + gateFails:4); every task-human row has a closed kind, never prose alone.
|
|
1094
|
-
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
|
|
1172
|
+
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh", details = {}) => {
|
|
1095
1173
|
graph = setStatus(graph, t.id, "human");
|
|
1096
1174
|
saveGraph(repoRoot, graph);
|
|
1097
|
-
journal.append("task-human", t.id, { reason, kind });
|
|
1175
|
+
journal.append("task-human", t.id, { ...details, reason, kind });
|
|
1098
1176
|
if (assignment) {
|
|
1099
1177
|
// OBS-547: `metered` counts CHARGEABLE metered attempts, so an unchargeable dispatch passes 0 and
|
|
1100
1178
|
// the count is omitted rather than written as 0 or as `1` beside `attempts: 0` — a row claiming
|
|
@@ -1172,6 +1250,22 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1172
1250
|
journal.append("worktree-preserved", t.id, { ref });
|
|
1173
1251
|
return driver.worktree(repoRoot, taskBranch, taskBase);
|
|
1174
1252
|
};
|
|
1253
|
+
// A dead worker with a clean checkout still needs a durable recovery handle: there may be no
|
|
1254
|
+
// pane left to identify even the commit it was dispatched from. preserveWorktree deliberately
|
|
1255
|
+
// creates no ref for ordinary clean recreations, so this exceptional terminal path pins HEAD
|
|
1256
|
+
// explicitly under the same recovery namespace. Reconfirm the delta immediately before the
|
|
1257
|
+
// ref write; a change or unreadable recheck withdraws the park.
|
|
1258
|
+
const preserveDeadWorker = async (worktree, taskBase) => {
|
|
1259
|
+
const state = await observeWorktreeDelta(taskBase, worktree);
|
|
1260
|
+
if (state !== "unchanged")
|
|
1261
|
+
return { state };
|
|
1262
|
+
const head = await gitHead(worktree);
|
|
1263
|
+
const ref = `refs/tickmarkr/preserved/${head}`;
|
|
1264
|
+
const updated = await shGit(`git update-ref ${shq(ref)} ${shq(head)}`, worktree);
|
|
1265
|
+
if (updated.code !== 0)
|
|
1266
|
+
throw new Error(`could not preserve dead worker HEAD at ${ref}: ${updated.stderr || updated.stdout}`);
|
|
1267
|
+
return { state, ref };
|
|
1268
|
+
};
|
|
1175
1269
|
const r = route(t, cfg, channels, profile, undefined, demotedChannels);
|
|
1176
1270
|
for (const lint of r.lints)
|
|
1177
1271
|
journal.append("routing-lint", t.id, { lint });
|
|
@@ -2184,6 +2278,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2184
2278
|
// subprocess tree that REACHED its exit marker, not about what the worker claimed.
|
|
2185
2279
|
let processExited = false;
|
|
2186
2280
|
let earlyLaunchDead = false;
|
|
2281
|
+
let deadWorkerPark;
|
|
2187
2282
|
let settleParsed;
|
|
2188
2283
|
let seedResult;
|
|
2189
2284
|
// v1.22 T5 / OBS-19: auto-answer a fingerprint-matched trust dialog exactly once per slot.
|
|
@@ -2323,6 +2418,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2323
2418
|
let quotaStreak = 0;
|
|
2324
2419
|
let rowSaturationHeld = false; // journaled once per attempt when the kill stands down
|
|
2325
2420
|
let cpuHeld = false; // likewise for the CPU leg's stand-down (OBS-548)
|
|
2421
|
+
let paneReadHeld = false;
|
|
2422
|
+
let paneStatusHeld = false;
|
|
2326
2423
|
while (Date.now() - lastProgressAt < stallWindowMs) {
|
|
2327
2424
|
const sliceStart = Date.now();
|
|
2328
2425
|
const remaining = stallWindowMs - (sliceStart - lastProgressAt);
|
|
@@ -2350,7 +2447,39 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2350
2447
|
break;
|
|
2351
2448
|
}
|
|
2352
2449
|
}
|
|
2353
|
-
|
|
2450
|
+
// A failed pane read is absence of evidence, never evidence of an absent pane. Keep the
|
|
2451
|
+
// rolling taskTimeoutMinutes window as the backstop and name the held probe once.
|
|
2452
|
+
let paneText;
|
|
2453
|
+
try {
|
|
2454
|
+
paneText = await driver.read(slot, PANE_READ_ROWS);
|
|
2455
|
+
}
|
|
2456
|
+
catch (error) {
|
|
2457
|
+
// Preserve the pre-existing exception path when the independent status probe still
|
|
2458
|
+
// sees a pane. The outer attempt finally owns accountant cleanup on that path. Only
|
|
2459
|
+
// an undetectable status makes the read failure relevant to the death detector, and
|
|
2460
|
+
// that genuinely unmeasurable pair fails open to the rolling timeout.
|
|
2461
|
+
let paneUndetectable = true;
|
|
2462
|
+
try {
|
|
2463
|
+
paneUndetectable = await driver.status(slot) === "unknown";
|
|
2464
|
+
}
|
|
2465
|
+
catch {
|
|
2466
|
+
// Two unreadable pane probes are still unmeasurable, never proof of death.
|
|
2467
|
+
}
|
|
2468
|
+
if (!paneUndetectable)
|
|
2469
|
+
throw error;
|
|
2470
|
+
if (!paneReadHeld) {
|
|
2471
|
+
paneReadHeld = true;
|
|
2472
|
+
journal.append("worker-dead-held", t.id, {
|
|
2473
|
+
slot: slot.name, attempt, reason: "pane-read-unreadable",
|
|
2474
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2475
|
+
});
|
|
2476
|
+
}
|
|
2477
|
+
await armCpuLeg(false);
|
|
2478
|
+
const spent = Date.now() - sliceStart;
|
|
2479
|
+
if (spent < slice)
|
|
2480
|
+
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
2481
|
+
continue;
|
|
2482
|
+
}
|
|
2354
2483
|
if (paneText.length > 0)
|
|
2355
2484
|
everHadOutput = true;
|
|
2356
2485
|
// OBS-117 (v1.71 T6): zero raw output by the early-launch deadline is a dead channel now.
|
|
@@ -2419,7 +2548,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2419
2548
|
// gate held. page on "idle" too: herdr's blocked-scrape is strict and proved flaky
|
|
2420
2549
|
// for TUI dialogs (live check: cursor's trust dialog scraped as idle).
|
|
2421
2550
|
// "unknown"/"working" never page.
|
|
2422
|
-
|
|
2551
|
+
let st;
|
|
2552
|
+
try {
|
|
2553
|
+
st = await driver.status(slot);
|
|
2554
|
+
}
|
|
2555
|
+
catch (error) {
|
|
2556
|
+
if (!paneStatusHeld) {
|
|
2557
|
+
paneStatusHeld = true;
|
|
2558
|
+
journal.append("worker-dead-held", t.id, {
|
|
2559
|
+
slot: slot.name, attempt, reason: "pane-status-unreadable",
|
|
2560
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2561
|
+
});
|
|
2562
|
+
}
|
|
2563
|
+
await armCpuLeg(false);
|
|
2564
|
+
const spent = Date.now() - sliceStart;
|
|
2565
|
+
if (spent < slice)
|
|
2566
|
+
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
2567
|
+
continue;
|
|
2568
|
+
}
|
|
2423
2569
|
if (st !== lastStatus) {
|
|
2424
2570
|
lastStatus = st;
|
|
2425
2571
|
journal.append("worker-status", t.id, { slot: slot.name, status: st, attempt });
|
|
@@ -2459,6 +2605,95 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2459
2605
|
// daemon has nothing left to do — the pane falls back under the fast-kill and page
|
|
2460
2606
|
// watchdogs like any other, instead of riding the whole rolling window untended.
|
|
2461
2607
|
const nudgePending = nudgeable && (!nudged || nudgeDeadline !== undefined);
|
|
2608
|
+
// OBS-737: disposition for the one fully measured death state. `unknown` alone is not
|
|
2609
|
+
// absence (status parsing can fail), and an empty read alone is not absence (a live pane
|
|
2610
|
+
// can be quiet); together they are the existing driver-level pane absence witness. The
|
|
2611
|
+
// process probe and both worktree observations retain their own third states. Only the
|
|
2612
|
+
// explicit conjunction parks; every other state falls through to the unchanged rolling
|
|
2613
|
+
// timeout below. Seeded launches are excluded because their process marker is knowingly
|
|
2614
|
+
// unmeasurable (armCpuLeg documents that contract above).
|
|
2615
|
+
const paneAbsentCandidate = st === "unknown" && paneText.trim().length === 0;
|
|
2616
|
+
// A subprocess can exit between waitOutput and read while its stdout is still draining.
|
|
2617
|
+
// Confirm emptiness in this same poll before paying for `ps`; then confirm once more
|
|
2618
|
+
// after the process probe yielded the event loop. Any bytes or read error withdraw the
|
|
2619
|
+
// absence witness, so a fast completed worker cannot be parked in that drain race.
|
|
2620
|
+
let paneAbsent = paneAbsentCandidate;
|
|
2621
|
+
if (paneAbsent) {
|
|
2622
|
+
try {
|
|
2623
|
+
const confirmation = await driver.read(slot, PANE_READ_ROWS);
|
|
2624
|
+
paneAbsent = confirmation.trim().length === 0;
|
|
2625
|
+
if (!paneAbsent) {
|
|
2626
|
+
everHadOutput = true;
|
|
2627
|
+
if (stallProgress.observe({ paneText: confirmation, contextTokens }))
|
|
2628
|
+
lastProgressAt = Date.now();
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
catch (error) {
|
|
2632
|
+
paneAbsent = false;
|
|
2633
|
+
if (!paneReadHeld) {
|
|
2634
|
+
paneReadHeld = true;
|
|
2635
|
+
journal.append("worker-dead-held", t.id, {
|
|
2636
|
+
slot: slot.name, attempt, reason: "pane-read-unreadable",
|
|
2637
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2638
|
+
});
|
|
2639
|
+
}
|
|
2640
|
+
}
|
|
2641
|
+
}
|
|
2642
|
+
const processTree = paneAbsent && worktreeSinceLaunch === "unchanged" && !hasSeed
|
|
2643
|
+
? await observeWorkerProcessTree(dispatchScript, wt)
|
|
2644
|
+
: "unmeasurable";
|
|
2645
|
+
if (processTree === "empty") {
|
|
2646
|
+
try {
|
|
2647
|
+
const confirmation = await driver.read(slot, PANE_READ_ROWS);
|
|
2648
|
+
paneAbsent = confirmation.trim().length === 0;
|
|
2649
|
+
if (!paneAbsent) {
|
|
2650
|
+
everHadOutput = true;
|
|
2651
|
+
if (stallProgress.observe({ paneText: confirmation, contextTokens }))
|
|
2652
|
+
lastProgressAt = Date.now();
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
catch (error) {
|
|
2656
|
+
paneAbsent = false;
|
|
2657
|
+
if (!paneReadHeld) {
|
|
2658
|
+
paneReadHeld = true;
|
|
2659
|
+
journal.append("worker-dead-held", t.id, {
|
|
2660
|
+
slot: slot.name, attempt, reason: "pane-read-unreadable",
|
|
2661
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2662
|
+
});
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
}
|
|
2666
|
+
const worktreeDelta = processTree === "empty"
|
|
2667
|
+
&& paneAbsent ? await observeWorktreeDelta(taskBase, wt)
|
|
2668
|
+
: "unreadable";
|
|
2669
|
+
// The first process snapshot can race a just-starting child after the dispatch pane
|
|
2670
|
+
// disappeared. Re-read it after the pane and worktree legs have both held: preservation
|
|
2671
|
+
// is terminal, so a process appearing in that interval must withdraw the park rather
|
|
2672
|
+
// than be orphaned by it. The final worktree recheck remains inside preserveDeadWorker.
|
|
2673
|
+
const confirmedProcessTree = worktreeDelta === "unchanged"
|
|
2674
|
+
? await observeWorkerProcessTree(dispatchScript, wt)
|
|
2675
|
+
: "unmeasurable";
|
|
2676
|
+
const deathCertain = paneAbsent
|
|
2677
|
+
&& processTree === "empty"
|
|
2678
|
+
&& confirmedProcessTree === "empty"
|
|
2679
|
+
&& worktreeDelta === "unchanged";
|
|
2680
|
+
if (deathCertain) {
|
|
2681
|
+
const preservation = await preserveDeadWorker(wt, taskBase);
|
|
2682
|
+
if (!preservation.ref) {
|
|
2683
|
+
journal.append("worker-dead-held", t.id, {
|
|
2684
|
+
slot: slot.name, attempt, reason: `worktree-${preservation.state}`,
|
|
2685
|
+
});
|
|
2686
|
+
continue;
|
|
2687
|
+
}
|
|
2688
|
+
const ref = preservation.ref;
|
|
2689
|
+
const reason = `worker is unambiguously dead: pane absent, process tree empty, and worktree unchanged; preserved at ${ref}`;
|
|
2690
|
+
deadWorkerPark = { ref, reason };
|
|
2691
|
+
journal.append("worktree-preserved", t.id, { ref });
|
|
2692
|
+
journal.append("worker-dead-held", t.id, {
|
|
2693
|
+
slot: slot.name, attempt, reason: "unambiguous-worker-death", ref,
|
|
2694
|
+
});
|
|
2695
|
+
break;
|
|
2696
|
+
}
|
|
2462
2697
|
// T1 review fix: the kill's "no output growth" leg clocks off the RAW growth signals,
|
|
2463
2698
|
// never lastProgressAt alone — the flat-token rule (stall.ts) deliberately suppresses
|
|
2464
2699
|
// the re-arm report on row growth once tokens stick, and contextTokens is sticky across
|
|
@@ -2632,7 +2867,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2632
2867
|
if (!finished && exitCode === null) {
|
|
2633
2868
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
2634
2869
|
timedOut = Date.now() - lastProgressAt >= stallWindowMs;
|
|
2635
|
-
|
|
2870
|
+
try {
|
|
2871
|
+
output = await driver.read(slot, PANE_READ_ROWS);
|
|
2872
|
+
}
|
|
2873
|
+
catch {
|
|
2874
|
+
// The poll loop already recorded the unreadable pane. Retain the last readable bytes
|
|
2875
|
+
// so this ambiguous path still reaches the ordinary timeout/consult backstop.
|
|
2876
|
+
}
|
|
2636
2877
|
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
2637
2878
|
const exit = exitRe.exec(output);
|
|
2638
2879
|
exitCode = exit ? Number(exit[1]) : null;
|
|
@@ -2766,6 +3007,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2766
3007
|
tokens = addUsage(tokens, attemptUsage);
|
|
2767
3008
|
metered++;
|
|
2768
3009
|
}
|
|
3010
|
+
if (deadWorkerPark) {
|
|
3011
|
+
await park(t, deadWorkerPark.reason, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { ref: deadWorkerPark.ref });
|
|
3012
|
+
return;
|
|
3013
|
+
}
|
|
2769
3014
|
let result = settleParsed ?? adapter.parse(output, nonce);
|
|
2770
3015
|
const workerFinished = finished;
|
|
2771
3016
|
const workerCause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut, deadChannel: deadChannelKilled });
|
package/dist/run/journal.d.ts
CHANGED
|
@@ -29,6 +29,11 @@ export declare const ATTEMPT_CAP_RELEASE: "attempt-cap";
|
|
|
29
29
|
export declare const GATE_SATISFIED_RELEASE: "gate-satisfied";
|
|
30
30
|
export declare const REVIEW_UPHELD_RELEASE: "review-upheld";
|
|
31
31
|
export declare const RECHECK_RELEASE: "recheck";
|
|
32
|
+
export interface PreservedRef {
|
|
33
|
+
ref: string;
|
|
34
|
+
diffCommand: string;
|
|
35
|
+
}
|
|
36
|
+
export declare function preservedRefsByTask(events: JournalEvent[]): Map<string, PreservedRef[]>;
|
|
32
37
|
export declare function reviewRoundsSinceApproval(events: JournalEvent[], taskId: string): number;
|
|
33
38
|
export declare function upheldFeedbackByTask(events: JournalEvent[]): Map<string, string>;
|
|
34
39
|
export interface StructuredFinding {
|
package/dist/run/journal.js
CHANGED
|
@@ -2,11 +2,12 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
2
2
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
|
-
import { channelKey, TokenUsageSchema } from "../adapters/types.js";
|
|
5
|
+
import { channelKey, shq, TokenUsageSchema } from "../adapters/types.js";
|
|
6
6
|
import { stateDirName, taskContentDigest, tickmarkrDir } from "../graph/graph.js";
|
|
7
7
|
import { GATE_NAMES, TIERS } from "../graph/schema.js";
|
|
8
8
|
import { buildProfile, classify } from "../route/profile.js";
|
|
9
9
|
import { DecisionEventSchema, trackJournalRows, } from "./protocol.js";
|
|
10
|
+
import { normalizeGateOutcome } from "./outcome.js";
|
|
10
11
|
import { redactSecrets } from "./redact.js";
|
|
11
12
|
export function phaseForGate(gate) {
|
|
12
13
|
if (gate === "acceptance")
|
|
@@ -15,6 +16,23 @@ export function phaseForGate(gate) {
|
|
|
15
16
|
return "review";
|
|
16
17
|
return `gate:${gate}`;
|
|
17
18
|
}
|
|
19
|
+
/** The pipe projects the same normalized gate outcome as the TTY rail. The selected-test field only
|
|
20
|
+
* chooses the held outcome's operator noun after the accessor has classified it; it is not a second
|
|
21
|
+
* verdict read. */
|
|
22
|
+
const journalGateDetail = (data) => {
|
|
23
|
+
if (typeof data.gate !== "string")
|
|
24
|
+
return undefined;
|
|
25
|
+
switch (normalizeGateOutcome(data).kind) {
|
|
26
|
+
case "passed":
|
|
27
|
+
return `${data.gate} passed`;
|
|
28
|
+
case "failed":
|
|
29
|
+
return `${data.gate} failed`;
|
|
30
|
+
case "held":
|
|
31
|
+
return Array.isArray(data.selectedTests) ? `${data.gate} selected-test screen` : `${data.gate} held`;
|
|
32
|
+
default:
|
|
33
|
+
return data.gate;
|
|
34
|
+
}
|
|
35
|
+
};
|
|
18
36
|
export function formatJournalNarration({ event, taskId, data }) {
|
|
19
37
|
const assignment = data.assignment;
|
|
20
38
|
const direct = [data.summary, data.reason, data.error, data.step, data.action, data.lint, data.branch, data.from]
|
|
@@ -26,7 +44,9 @@ export function formatJournalNarration({ event, taskId, data }) {
|
|
|
26
44
|
? `${data.gate} failed${typeof data.lastMergedTask === "string" ? ` after ${data.lastMergedTask}` : ""}`
|
|
27
45
|
: event === "tip-verify"
|
|
28
46
|
? `${data.gate} passed`
|
|
29
|
-
:
|
|
47
|
+
: event === "gate-result"
|
|
48
|
+
? journalGateDetail(data)
|
|
49
|
+
: `${data.gate}`
|
|
30
50
|
: typeof data.code === "number" ? `exit ${data.code}`
|
|
31
51
|
: typeof data.pid === "number" ? `pid ${data.pid}`
|
|
32
52
|
: typeof data.baseRef === "string" ? `base ${data.baseRef.slice(0, 12)}`
|
|
@@ -56,6 +76,24 @@ export const REVIEW_UPHELD_RELEASE = "review-upheld";
|
|
|
56
76
|
// so the corrected declaration is the thing that earns the green. Budget semantics match attempt-cap
|
|
57
77
|
// (fresh attempts, tried survives) because the park cost the task its remaining budget.
|
|
58
78
|
export const RECHECK_RELEASE = "recheck";
|
|
79
|
+
// OBS-738: one authority for every recovery surface. The ref is accepted only from the row that
|
|
80
|
+
// preservation itself writes; task-human prose, branch heads and commit history are deliberately
|
|
81
|
+
// absent from this fold. Keep every row in journal order — one task can be recreated more than once,
|
|
82
|
+
// and the terminal record owes the operator every resulting recovery handle, not merely the newest.
|
|
83
|
+
export function preservedRefsByTask(events) {
|
|
84
|
+
const byTask = new Map();
|
|
85
|
+
for (const event of events) {
|
|
86
|
+
if (event.event !== "worktree-preserved" || !event.taskId || typeof event.data.ref !== "string"
|
|
87
|
+
|| event.data.ref === "")
|
|
88
|
+
continue;
|
|
89
|
+
const ref = event.data.ref;
|
|
90
|
+
byTask.set(event.taskId, [
|
|
91
|
+
...(byTask.get(event.taskId) ?? []),
|
|
92
|
+
{ ref, diffCommand: `git diff ${shq(`${ref}^!`)}` },
|
|
93
|
+
]);
|
|
94
|
+
}
|
|
95
|
+
return byTask;
|
|
96
|
+
}
|
|
59
97
|
// OBS-189: review rounds are scoped to the current ENGAGEMENT — the stretch since the newest operator
|
|
60
98
|
// approval for the task. A whole-journal count re-parks an upheld task before its funded attempt can
|
|
61
99
|
// dispatch (measured live on run-20260726-213539), making a fresh journal the only escape. A T15
|
|
@@ -1069,19 +1107,36 @@ export class Journal {
|
|
|
1069
1107
|
? undefined
|
|
1070
1108
|
: DecisionEventSchema.parse({ ...eventOrDecision, ts: new Date().toISOString() });
|
|
1071
1109
|
const event = decisionRow?.event ?? eventOrDecision;
|
|
1110
|
+
const rowTaskId = decisionRow && "taskId" in decisionRow ? decisionRow.taskId : taskId;
|
|
1072
1111
|
const inputData = decisionRow?.data ?? data;
|
|
1112
|
+
// OBS-738: terminal and resume records reduce the journal that precedes them. Neither re-derives
|
|
1113
|
+
// recovery facts from task-human prose: preserved refs come from preservedRefsByTask, and the
|
|
1114
|
+
// upheld brief comes from the established prompt/replay reducer.
|
|
1115
|
+
const priorEvents = event === "run-end" || event === "resume-restore" ? this.read() : [];
|
|
1116
|
+
const reducedData = event === "run-end"
|
|
1117
|
+
? (() => {
|
|
1118
|
+
const preservedRefs = [...preservedRefsByTask(priorEvents)].flatMap(([preservedTaskId, refs]) => refs.map(({ ref, diffCommand }) => ({ taskId: preservedTaskId, ref, diffCommand })));
|
|
1119
|
+
return preservedRefs.length > 0 ? { ...inputData, preservedRefs } : inputData;
|
|
1120
|
+
})()
|
|
1121
|
+
: event === "resume-restore" && rowTaskId && upheldFeedbackByTask(priorEvents).has(rowTaskId)
|
|
1122
|
+
? {
|
|
1123
|
+
...inputData,
|
|
1124
|
+
upheldFeedbackRestoredFor: rowTaskId,
|
|
1125
|
+
summary: `upheld feedback restored for ${rowTaskId}`,
|
|
1126
|
+
}
|
|
1127
|
+
: inputData;
|
|
1073
1128
|
const evidence = judgePersistence.getStore();
|
|
1074
1129
|
const failed = evidence?.invocations.filter((invocation) => invocation.transcript !== undefined) ?? [];
|
|
1075
1130
|
const persistedData = event === "judge-retry" && failed.length > 0
|
|
1076
1131
|
? {
|
|
1077
|
-
...
|
|
1132
|
+
...reducedData,
|
|
1078
1133
|
transcript: failed[0].transcript,
|
|
1079
1134
|
...(failed[1] ? { retryTranscript: failed[1].transcript } : {}),
|
|
1080
1135
|
}
|
|
1081
|
-
:
|
|
1082
|
-
const row = decisionRow
|
|
1083
|
-
|
|
1084
|
-
|
|
1136
|
+
: reducedData;
|
|
1137
|
+
const row = decisionRow
|
|
1138
|
+
? { ...decisionRow, data: persistedData }
|
|
1139
|
+
: { ts: new Date().toISOString(), event, ...(taskId ? { taskId } : {}), data: persistedData };
|
|
1085
1140
|
// T3 secret redaction: only the persisted bytes are masked — the caller's data stays untouched in
|
|
1086
1141
|
// memory. The narrator receives the persisted (masked) row so a pane sink never shows a credential.
|
|
1087
1142
|
const line = redactSecrets(JSON.stringify(row));
|
package/dist/run/outcome.js
CHANGED
|
@@ -88,7 +88,14 @@ export function normalizeGateOutcome(source) {
|
|
|
88
88
|
return { kind: "unavailable", reason: `malformed gate result: expected an object, read ${shape}` };
|
|
89
89
|
}
|
|
90
90
|
const row = source;
|
|
91
|
-
//
|
|
91
|
+
// A gate-result data wrapper states its canonical verdict in `outcome`. Consumers may pass either
|
|
92
|
+
// the wrapper or the outcome itself; both must read the same kind even if stale bare fields remain.
|
|
93
|
+
if (hasOwn(row, "outcome")) {
|
|
94
|
+
if (isGateOutcome(row.outcome))
|
|
95
|
+
return row.outcome;
|
|
96
|
+
return { kind: "unavailable", reason: reasonOf(row, "malformed gate result: invalid explicit outcome") };
|
|
97
|
+
}
|
|
98
|
+
// `kind` selects the canonical outcome object. It must validate as exactly one canonical arm without any
|
|
92
99
|
// legacy truth discriminator; otherwise continuing through the legacy branches could reinterpret a
|
|
93
100
|
// contradictory dual-write as passed, failed or infra. Mixed and malformed canonical rows stay
|
|
94
101
|
// explicitly unavailable, retaining any reason/details the producer did manage to state.
|
|
@@ -3,6 +3,8 @@ export declare const SUPERVISION_BEAT_MS = 10000;
|
|
|
3
3
|
/** Ceiling before a beat reads STALE: SIX beats, lock.ts's ratio — five may be missed before alarm. */
|
|
4
4
|
export declare const SUPERVISION_STALE_MS: number;
|
|
5
5
|
export declare const SUPERVISION_FUTURE_GRACE_MS = 1000;
|
|
6
|
+
/** Context watchers act at 75% unless their invocation declares another threshold. */
|
|
7
|
+
export declare const SUPERVISION_DEFAULT_THRESHOLD_PCT = 75;
|
|
6
8
|
export declare const SUPERVISION_TIERS: readonly ["orchestrator", "orchestrator-context", "overseer", "overseer-context", "watch"];
|
|
7
9
|
export type SupervisionTier = (typeof SUPERVISION_TIERS)[number];
|
|
8
10
|
export declare const SUPERVISION_SEAT_TIERS: readonly ["orchestrator", "orchestrator-context", "overseer", "overseer-context"];
|
|
@@ -17,16 +19,28 @@ export interface TierLiveness {
|
|
|
17
19
|
beatAgeMs?: number;
|
|
18
20
|
/** The seat the record names, when it names one. Always present on a seat tier that is not ABSENT. */
|
|
19
21
|
seat?: string;
|
|
22
|
+
/** The alarm's durable duty, orthogonal to whether its watcher is armed, stale or stood down. */
|
|
23
|
+
clearOwedSince?: string;
|
|
24
|
+
/** The latch exists but cannot be decoded; never mistaken for no duty owed. */
|
|
25
|
+
clearOwedUnreadable?: true;
|
|
26
|
+
}
|
|
27
|
+
/** A watcher observation that can raise or discharge the seat's clear obligation. */
|
|
28
|
+
export interface SupervisionBeatObservation {
|
|
29
|
+
armId: string;
|
|
30
|
+
pct: number;
|
|
31
|
+
thresholdPct: number;
|
|
20
32
|
}
|
|
21
33
|
export declare const supervisionBeatPath: (repoRoot: string, tier: SupervisionTier) => string;
|
|
22
34
|
/** Where a watcher records that it STOOD DOWN. Its own file: the beat keeps meaning only "alive". */
|
|
23
35
|
export declare const supervisionStandDownPath: (repoRoot: string, tier: SupervisionTier) => string;
|
|
24
|
-
export declare function beatSupervision(repoRoot: string, tier: SupervisionTier, seat?: string): void;
|
|
36
|
+
export declare function beatSupervision(repoRoot: string, tier: SupervisionTier, seat?: string, observation?: SupervisionBeatObservation): void;
|
|
25
37
|
/** Handle a watcher holds for as long as it is supervising; disarm stands it down and is idempotent. */
|
|
26
38
|
export interface ArmedSupervision {
|
|
27
39
|
disarm: () => void;
|
|
28
40
|
}
|
|
29
41
|
export declare function armSupervision(repoRoot: string, tier: SupervisionTier, beatMs?: number, seat?: string): ArmedSupervision;
|
|
42
|
+
/** Arm the live status board's own tier through the normal supervision writer and lifecycle. */
|
|
43
|
+
export declare function armWatchSupervision(repoRoot: string, beatMs?: number): ArmedSupervision;
|
|
30
44
|
export declare function readTierLiveness(repoRoot: string, tier: SupervisionTier, now?: number): TierLiveness;
|
|
31
45
|
export declare function supervisionStatus(repoRoot: string, tier: SupervisionTier, now?: number): TierLiveness;
|
|
32
46
|
/** Every KNOWN tier, always — a tier omitted from this list would read as one that is fine. */
|