tickmarkr 1.44.0 → 1.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  import { mkdirSync, writeFileSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
  import { getAdapter } from "../adapters/registry.js";
4
- import { bannerShell, paneDispatchCommand, paneDispatchScript } from "../brand.js";
5
- import { extractJson, gatePaneName } from "../gates/llm.js";
4
+ import { bannerShell, paneDispatchCommand } from "../brand.js";
5
+ import { augmentFakeVerdictOutput, extractVerdictJson, gateExitTrailer, gatePaneName, generateVerdictNonce, verdictNonceLine } from "../gates/llm.js";
6
6
  import { sh } from "./git.js";
7
7
  const MAX_RETRY_GUIDANCE_LINES = 10;
8
8
  function guidanceParts(text) {
@@ -19,8 +19,43 @@ export function renderRetryGuidance(v) {
19
19
  lines.push(part);
20
20
  return lines.slice(0, MAX_RETRY_GUIDANCE_LINES).map((l) => `- ${l}`).join("\n");
21
21
  }
22
+ const LANDED_COMMIT_PREMISE_RE = /\b(?:already\s+committed|is\s+already\s+committed|landed\s+commit|implementation\s+is\s+already)\b/i;
23
+ const COMMIT_HASH_RE = /\b[0-9a-f]{7,40}\b/gi;
24
+ // OBS-58: name prior attempt commits by hash and drop consult premises about landed work that the
25
+ // fresh worktree does not satisfy — regenerated against post-recreation state, not inherited verbatim.
26
+ export function augmentRetryBrief(feedback, opts) {
27
+ const { attempted, carried, present } = opts;
28
+ const named = [...new Set([...attempted, ...carried])];
29
+ const presentLower = new Set([...present].map((h) => h.toLowerCase()));
30
+ const parts = [];
31
+ if (named.length > 0) {
32
+ parts.push("## Prior attempt commits (by hash)\n"
33
+ + named.map((h) => `- ${h}${presentLower.has(h.toLowerCase()) ? " — present in this worktree" : " — not in this worktree"}`).join("\n"));
34
+ }
35
+ let body = feedback.trim();
36
+ if (body && LANDED_COMMIT_PREMISE_RE.test(body)) {
37
+ const falseHash = [...body.matchAll(COMMIT_HASH_RE)].some((m) => !presentLower.has(m[0].toLowerCase()));
38
+ const falseLanded = carried.length === 0 && attempted.length > 0;
39
+ if (falseHash || falseLanded) {
40
+ body = body
41
+ .split("\n")
42
+ .filter((line) => !LANDED_COMMIT_PREMISE_RE.test(line))
43
+ .join("\n")
44
+ .trim();
45
+ const correction = carried.length > 0
46
+ ? `Prior attempt work is present at: ${carried.join(", ")}.`
47
+ : attempted.length > 0
48
+ ? "Prior attempt commits could not be carried forward onto this worktree — re-land the work or continue from the integration tip."
49
+ : "No prior attempt commits are present in this worktree — implement from the integration tip.";
50
+ body = body ? `${correction}\n\n${body}` : correction;
51
+ }
52
+ }
53
+ if (body)
54
+ parts.push(body);
55
+ return parts.join("\n\n");
56
+ }
22
57
  const ACTIONS = ["retry", "reroute", "decompose", "human"];
23
- export function buildDossierPrompt(d) {
58
+ export function buildDossierPrompt(d, nonce) {
24
59
  return `TICKMARKR-CONSULT
25
60
  You are a senior engineering consult for the tickmarkr orchestrator. A worker task hit trouble.
26
61
  Read the dossier and return a verdict. Be decisive; cost is the lowest priority, quality the highest.
@@ -47,8 +82,10 @@ channel of that adapter for this task. Use it for environmental failures ("the C
47
82
  trust dialog, broken install) — not when a single model produced bad code. Omit for model-level
48
83
  reroutes so other models of the same adapter remain eligible.
49
84
 
85
+ ${verdictNonceLine(nonce)}
86
+
50
87
  Respond with ONLY this JSON:
51
- {"action": "retry" | "reroute" | "decompose" | "human", "reason": "why this action", "guidance": "imperative steps for the worker (newline-separated ok)", "excludeAdapter"?: "<adapter-id>"}
88
+ {"nonce": "${nonce}", "action": "retry" | "reroute" | "decompose" | "human", "reason": "why this action", "guidance": "imperative steps for the worker (newline-separated ok)", "excludeAdapter"?: "<adapter-id>"}
52
89
  `;
53
90
  }
54
91
  let consultSeq = 0;
@@ -57,15 +94,17 @@ export async function consult(d, cfg, adapters, driver, cwd, runDir,
57
94
  // via SlotOpts.owned; without it the legacy gatePaneName shape survives (non-daemon callers/tests).
58
95
  opts = {}) {
59
96
  const n = ++consultSeq;
97
+ const nonce = generateVerdictNonce();
60
98
  const dir = join(runDir, "consults");
61
99
  mkdirSync(dir, { recursive: true });
62
100
  const promptFile = join(dir, `${d.taskId}-${n}.md`);
63
- writeFileSync(promptFile, buildDossierPrompt(d));
101
+ writeFileSync(promptFile, buildDossierPrompt(d, nonce));
64
102
  const adapter = getAdapter(cfg.consult.adapter, adapters);
65
103
  let out;
66
104
  if (cfg.visibility.llm === "headless") {
67
105
  const r = await sh(adapter.headlessCommand(promptFile, cfg.consult.model), cwd, cfg.consult.stallMinutes * 60_000);
68
106
  out = r.stdout + r.stderr;
107
+ out = augmentFakeVerdictOutput(adapter, out, nonce);
69
108
  }
70
109
  else {
71
110
  // T8: role-first pane name for fleet visibility (consult · T2); consultSeq stays on the dossier artifact only
@@ -76,16 +115,21 @@ opts = {}) {
76
115
  opts.onSlot?.(slot);
77
116
  const scriptPath = join(dir, `${d.taskId}-${n}.sh`);
78
117
  // OBS-50: visible consult panes get the brand banner; headless path above stays banner-free (machine-parsed stdout)
79
- writeFileSync(scriptPath, paneDispatchScript([bannerShell(), adapter.headlessCommand(promptFile, cfg.consult.model)]));
118
+ writeFileSync(scriptPath, [
119
+ "export BASH_SILENCE_DEPRECATION_WARNING=1",
120
+ bannerShell(),
121
+ adapter.headlessCommand(promptFile, cfg.consult.model),
122
+ gateExitTrailer(nonce),
123
+ ].join("\n"));
80
124
  await driver.run(slot, paneDispatchCommand(scriptPath));
81
- // digit-suffixed marker only: a dossier quoting tickmarkr's own "TICKMARKR_EXIT:" literal must not
82
- // false-complete the consult pane before its verdict (same guard as daemon.ts:193 / llm.ts).
83
- await driver.waitOutput(slot, "TICKMARKR_EXIT:\\d", cfg.consult.stallMinutes * 60_000, { regex: true });
125
+ // nonce-suffixed exit only: a dossier quoting tickmarkr's own exit literal must not false-complete.
126
+ await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, cfg.consult.stallMinutes * 60_000, { regex: true });
84
127
  out = await driver.read(slot, 300);
85
128
  if (!opts.keep)
86
129
  await driver.close(slot);
130
+ out = augmentFakeVerdictOutput(adapter, out, nonce);
87
131
  }
88
- const v = extractJson(out);
132
+ const v = extractVerdictJson(out, nonce);
89
133
  if (!v || !ACTIONS.includes(v.action)) {
90
134
  return { action: "human", notes: "consult verdict unparseable — failing safe to human" };
91
135
  }
@@ -4,6 +4,7 @@ import { type JournalEvent } from "./journal.js";
4
4
  export interface RunOptions {
5
5
  runId?: string;
6
6
  resume?: boolean;
7
+ graphChanged?: boolean;
7
8
  concurrency?: number;
8
9
  driver?: ExecutorDriver;
9
10
  adapters?: WorkerAdapter[];
@@ -1,5 +1,6 @@
1
1
  import { randomBytes } from "node:crypto";
2
- import { readFileSync, writeFileSync } from "node:fs";
2
+ import { shq } from "../adapters/types.js";
3
+ import { existsSync, readFileSync, writeFileSync } from "node:fs";
3
4
  import { join } from "node:path";
4
5
  import { trailerPattern, writePrompt } from "../adapters/prompt.js";
5
6
  import { allAdapters, discoverChannels, getAdapter, probeAll, readDoctor } from "../adapters/registry.js";
@@ -9,10 +10,10 @@ import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js
9
10
  import { formatOwnedName } from "../drivers/types.js";
10
11
  import { captureBaseline, detectGateCommands } from "../gates/baseline.js";
11
12
  import { runGates } from "../gates/run-gates.js";
12
- import { addEvidence, attributeBlocked, blockedTasks, getTask, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
13
- import { consult, renderRetryGuidance } from "./consult.js";
14
- import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, WORKTREE_LAYOUT_CONTRACT } from "./git.js";
15
- import { Journal, loadRoutingProfile, newRunId } from "./journal.js";
13
+ import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
14
+ import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
15
+ import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, shGit, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
16
+ import { classifyWorkerResultCause, engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
16
17
  import { acquireRunLock, releaseRunLock } from "./lock.js";
17
18
  import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
18
19
  import { nextChannel, route } from "../route/router.js";
@@ -27,6 +28,30 @@ export function formatSummary(s) {
27
28
  }
28
29
  const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
29
30
  const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
31
+ const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
32
+ const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
33
+ const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
34
+ async function commitsAheadOf(base, wt) {
35
+ const head = await gitHead(wt);
36
+ if (head === base)
37
+ return [];
38
+ const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, wt);
39
+ if (r.code !== 0)
40
+ return [];
41
+ return r.stdout.trim().split("\n").filter(Boolean);
42
+ }
43
+ async function cherryPickCommits(wt, commits) {
44
+ const carried = [];
45
+ for (const hash of commits) {
46
+ const r = await shGit(`git cherry-pick --no-gpg-sign ${shq(hash)}`, wt);
47
+ if (r.code !== 0) {
48
+ await shGit("git cherry-pick --abort", wt);
49
+ break;
50
+ }
51
+ carried.push(hash);
52
+ }
53
+ return carried;
54
+ }
30
55
  export async function runDaemon(repoRoot, opts = {}) {
31
56
  const cfg = loadConfig(repoRoot, { globalDir: opts.globalDir });
32
57
  const adapters = opts.adapters ?? allAdapters();
@@ -74,6 +99,24 @@ export async function runDaemon(repoRoot, opts = {}) {
74
99
  if (!start)
75
100
  throw new Error(`journal for ${runId} has no run-start event`);
76
101
  baseRef = start.data.baseRef;
102
+ // T3 (Sol #2 / Fable F2): refuse to replay this journal's task states onto a graph it does not
103
+ // belong to — overlapping ids would inherit foreign done/human/approval state, missing ids throw.
104
+ // The SAME comparator status uses (engagementComparable); one decision, two consumers. Fail closed:
105
+ // no resume path silently accepts a mismatched or unbound journal. --graph-changed is the operator's
106
+ // audited release for the stop-amend-resume workflow, journaling a graph-rehash naming both hashes.
107
+ const loadedHash = graphDefinitionHash(graph);
108
+ const cmp = engagementComparable(journal.read(), loadedHash);
109
+ if (!cmp.comparable) {
110
+ if (!opts.graphChanged) {
111
+ throw new Error(cmp.reason === "unbound"
112
+ ? `refusing to resume ${runId}: journal has no recorded graph definition hash (older tickmarkr) — pass --graph-changed to override`
113
+ : `refusing to resume ${runId}: graph changed since this run (recorded ${cmp.recorded} ≠ loaded ${loadedHash}) — pass --graph-changed to override`);
114
+ }
115
+ journal.append("graph-rehash", undefined, {
116
+ from: cmp.reason === "mismatch" ? cmp.recorded : null,
117
+ to: loadedHash,
118
+ });
119
+ }
77
120
  baseline = JSON.parse(readFileSync(join(journal.dir, "baseline.json"), "utf8"));
78
121
  for (const [id, st] of journal.replayStatuses()) {
79
122
  // operator release: a graph.json edit back to "pending" beats a replayed human/failed park (locked decision 12)
@@ -87,7 +130,7 @@ export async function runDaemon(repoRoot, opts = {}) {
87
130
  baseRef = await gitHead(repoRoot);
88
131
  baseline = await captureBaseline(repoRoot, commands);
89
132
  writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(baseline, null, 2));
90
- journal.append("run-start", undefined, { pid: process.pid, baseRef, commands, channels: channels.map(channelKey), branch, graphHash: graph.spec.hash }); // graphHash: OBS-52 status join guard; pid: v1.13 (VIS-11) liveness
133
+ journal.append("run-start", undefined, { pid: process.pid, baseRef, commands, channels: channels.map(channelKey), branch, graphDefinitionHash: graphDefinitionHash(graph) }); // graphDefinitionHash: T3 engagement identity (status+resume share it); pid: v1.13 (VIS-11) liveness
91
134
  }
92
135
  // T6: open the narrator AFTER run-start/run-resume is journaled so the watch surface has a run to
93
136
  // show. driver.narrator is undefined on subprocess → no-op (subprocess spawns nothing). Swallowed:
@@ -110,6 +153,9 @@ export async function runDaemon(repoRoot, opts = {}) {
110
153
  const keepLlm = keepForever;
111
154
  const keptSlots = [];
112
155
  const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
156
+ // OBS-57: per-run in-run demotion — channels that burn consecutive no-trailer windows route around for later attempts.
157
+ const demotedChannels = new Set();
158
+ const noTrailerStreak = new Map();
113
159
  // OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
114
160
  // terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
115
161
  // owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
@@ -145,14 +191,14 @@ export async function runDaemon(repoRoot, opts = {}) {
145
191
  mergeChain = next.catch(() => undefined);
146
192
  return next;
147
193
  };
148
- // kind is null only at the humanGate call (assignment null no row anyway); gateFails/consults are
149
- // execTask-scoped counters passed in so a park row is a rich verified-failure observation (e.g. ladder-exhausted + gateFails:4).
194
+ // gateFails/consults are execTask-scoped counters passed in so a park row is a rich verified-failure
195
+ // observation (e.g. ladder-exhausted + gateFails:4); every task-human row has a closed kind, never prose alone.
150
196
  const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
151
197
  graph = setStatus(graph, t.id, "human");
152
198
  saveGraph(repoRoot, graph);
153
- journal.append("task-human", t.id, { reason });
199
+ journal.append("task-human", t.id, { reason, kind });
154
200
  if (assignment) {
155
- journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind ?? undefined, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
201
+ journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
156
202
  }
157
203
  await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
158
204
  await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}`, { tier: "attention" });
@@ -165,10 +211,10 @@ export async function runDaemon(repoRoot, opts = {}) {
165
211
  // term) would silently dispatch every unapproved gate that becomes ready during a resume — pinned
166
212
  // by the resume-path guard pin in tests/run/daemon.test.ts (the only test that reaches this guard
167
213
  // on the resume path; a park-then-resume task is filtered out by readyTasks() and never gets here).
168
- await park(t, `humanGate: "${t.title}" requires approval before dispatch`, null, null, 0, startMs);
214
+ await park(t, `humanGate: "${t.title}" requires approval before dispatch`, "human-gate", null, 0, startMs);
169
215
  return;
170
216
  }
171
- const r = route(t, cfg, channels, profile);
217
+ const r = route(t, cfg, channels, profile, undefined, demotedChannels);
172
218
  for (const lint of r.lints)
173
219
  journal.append("routing-lint", t.id, { lint });
174
220
  // VIS-02: journal a deviation from the static choice ONLY when one occurred (greppable absence = no deviation)
@@ -193,7 +239,7 @@ export async function runDaemon(repoRoot, opts = {}) {
193
239
  // trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
194
240
  // release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
195
241
  // EXISTING nextChannel `tried` parameter — zero router changes (D-03).
196
- const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile);
242
+ const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
197
243
  if (next)
198
244
  assignment = next;
199
245
  // ponytail: nextChannel null (every channel already tried / none available) — keep the static
@@ -227,9 +273,9 @@ export async function runDaemon(repoRoot, opts = {}) {
227
273
  // ROUTE-13: learned within-band failover + deviation audit. nextChannel stays pure (route/ never
228
274
  // journals); the daemon compares the learned pick against the static pick and owns the journal write.
229
275
  const failover = (site) => {
230
- const next = nextChannel(assignment, t, cfg, channels, tried, profile);
276
+ const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
231
277
  if (profile && next) {
232
- const staticNext = nextChannel(assignment, t, cfg, channels, tried);
278
+ const staticNext = nextChannel(assignment, t, cfg, channels, tried, undefined, demotedChannels);
233
279
  if (staticNext && channelKey(next) !== channelKey(staticNext)) {
234
280
  journal.append("failover-deviation", t.id, { site, static: channelKey(staticNext), chosen: channelKey(next) });
235
281
  }
@@ -293,7 +339,13 @@ export async function runDaemon(repoRoot, opts = {}) {
293
339
  // the existing attempt-cap check below with zero new code. Fresh path: rs is undefined ⇒ 0.
294
340
  // v1.24 OBS-18: after task-approved{release:attempt-cap}, replay zeros attempts so this loop
295
341
  // starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
342
+ let providerDeathRequeues = 0;
343
+ let providerDeathAttempt = -1;
296
344
  attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
345
+ if (attempt !== providerDeathAttempt) {
346
+ providerDeathRequeues = 0;
347
+ providerDeathAttempt = attempt;
348
+ }
297
349
  // v1.13 (VIS-09 safety): one FRESH nonce per attempt — see the run-scope comment above. A retained
298
350
  // prior-attempt trailer (herdr scrollback / subprocess buffer) must never satisfy this attempt.
299
351
  const nonce = randomBytes(4).toString("hex");
@@ -303,6 +355,16 @@ export async function runDaemon(repoRoot, opts = {}) {
303
355
  await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
304
356
  return;
305
357
  }
358
+ // OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
359
+ if (demotedChannels.has(channelKey(assignment))) {
360
+ const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
361
+ if (next) {
362
+ assignment = next;
363
+ const k = channelKey(next);
364
+ if (!tried.includes(k))
365
+ tried.push(k);
366
+ }
367
+ }
306
368
  // v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
307
369
  // + adapter capability resumes; every other path is today's fresh dispatch.
308
370
  const priorSession = retrySession;
@@ -329,7 +391,27 @@ export async function runDaemon(repoRoot, opts = {}) {
329
391
  journal.append("task-dispatch", t.id, { assignment, attempt, provenance: r.provenance, retryMode });
330
392
  const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
331
393
  const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
394
+ const priorWt = worktreePath(repoRoot, taskBranch);
395
+ const commitsToCarry = existsSync(priorWt) ? await commitsAheadOf(taskBase, priorWt) : [];
332
396
  const wt = await driver.worktree(repoRoot, taskBranch, taskBase);
397
+ // OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
398
+ // cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
399
+ // orphan work a consult already verified as landed.
400
+ let carriedCommits = [];
401
+ if (commitsToCarry.length > 0) {
402
+ carriedCommits = await cherryPickCommits(wt, commitsToCarry);
403
+ journal.append("worktree-recreation", t.id, { attempted: commitsToCarry, carried: carriedCommits });
404
+ }
405
+ const priorNamed = [...new Set([...commitsToCarry, ...carriedCommits])];
406
+ const presentCommits = new Set(carriedCommits);
407
+ for (const h of commitsToCarry) {
408
+ if (!presentCommits.has(h) && (await shGit(`git merge-base --is-ancestor ${shq(h)} HEAD`, wt)).code === 0) {
409
+ presentCommits.add(h);
410
+ }
411
+ }
412
+ if (feedback || priorNamed.length > 0) {
413
+ feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
414
+ }
333
415
  if (cfg.setup) {
334
416
  // v1.22 T3: setup runs inside the task worktree — seal herdr control vars so a setup script
335
417
  // cannot mutate the operator's panes. Worker/judge/review/consult are sealed at the driver
@@ -343,11 +425,13 @@ export async function runDaemon(repoRoot, opts = {}) {
343
425
  }
344
426
  }
345
427
  const promptFile = writePrompt(journal.dir, t, attempt, feedback, nonce);
428
+ // OBS-56: state the non-interactive, one-pass finish contract and the OBS-54 stall budget in every
429
+ // worker prompt, not only consult retry guidance. Prepended so prompt.ts's completion trailer stays last.
430
+ const workerContract = `## Harness contract\n- This harness is non-interactive: make one continuous pass; do not stop for questions or follow-up input.\n- You have a ${taskTimeoutMinutes} minute stall window. Budget the full suite once, then commit and emit the completion trailer before it expires.`;
346
431
  // OBS-47: state the worktree layout contract in the worker prompt (cheap-tier workers were
347
- // committing/deleting node_modules and tripping the scope gate). Prepended, not appended, so the
348
- // completion trailer stays the structural last line (prompt.ts's design). The harness re-asserts
349
- // the link itself before gates regardless of what the worker does with it.
350
- writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${readFileSync(promptFile, "utf8")}`);
432
+ // committing/deleting node_modules and tripping the scope gate). The harness re-asserts the link
433
+ // itself before gates regardless of what the worker does with it.
434
+ writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${workerContract}\n\n${readFileSync(promptFile, "utf8")}`);
351
435
  const adapter = getAdapter(assignment.adapter, adapters);
352
436
  // VIS-04: workers share one role tab. T2: `owned` names the pane canonically (ownership contract);
353
437
  // the legacy name stays the fallback for drivers without owned handling (subprocess spies).
@@ -401,6 +485,7 @@ export async function runDaemon(repoRoot, opts = {}) {
401
485
  let finished;
402
486
  let output;
403
487
  let exitCode;
488
+ let timedOut = false;
404
489
  if (interactive) {
405
490
  // v1.2 interactive: the TUI doesn't exit on completion — the trailer is the finish line.
406
491
  // The exit wrapper still fires if the TUI dies (crash/quit): fast-fail instead of burning the timeout.
@@ -411,11 +496,16 @@ export async function runDaemon(repoRoot, opts = {}) {
411
496
  let trustAnswered = false;
412
497
  finished = false;
413
498
  exitCode = null;
414
- output = "";
415
- const deadline = Date.now() + taskTimeoutMinutes * 60_000;
416
- while (Date.now() < deadline) {
499
+ output = await driver.read(slot, 1000);
500
+ // OBS-54: reaping keys on new pane output, not dispatch wall clock. Poll at least twice per
501
+ // stall window (and at the existing 30s cadence for normal windows) so an active worker resets it.
502
+ const stallWindowMs = taskTimeoutMinutes * 60_000;
503
+ let lastOutput = output;
504
+ let lastOutputAt = Date.now();
505
+ while (Date.now() - lastOutputAt < stallWindowMs) {
417
506
  const sliceStart = Date.now();
418
- const slice = Math.min(BLOCKED_POLL_MS, deadline - sliceStart);
507
+ const remaining = stallWindowMs - (sliceStart - lastOutputAt);
508
+ const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
419
509
  if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
420
510
  // verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
421
511
  // own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
@@ -429,6 +519,11 @@ export async function runDaemon(repoRoot, opts = {}) {
429
519
  break;
430
520
  }
431
521
  }
522
+ const currentOutput = await driver.read(slot, 1000);
523
+ if (currentOutput !== lastOutput) {
524
+ lastOutput = currentOutput;
525
+ lastOutputAt = Date.now();
526
+ }
432
527
  // v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
433
528
  await sampleContext();
434
529
  // page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
@@ -468,6 +563,7 @@ export async function runDaemon(repoRoot, opts = {}) {
468
563
  }
469
564
  if (!finished && exitCode === null) {
470
565
  // timed out (or only ever saw false positives): harvest whatever the pane holds now
566
+ timedOut = Date.now() - lastOutputAt >= stallWindowMs;
471
567
  output = await driver.read(slot, 1000);
472
568
  finished = new RegExp(trailerPattern(nonce)).test(output);
473
569
  const exit = exitRe.exec(output);
@@ -481,9 +577,27 @@ export async function runDaemon(repoRoot, opts = {}) {
481
577
  else {
482
578
  const inv = adapter.invoke(t, wt, assignment, { promptFile });
483
579
  await driver.run(slot, `${inv.command}; ${exitMarkerCmd}`);
484
- finished = await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, taskTimeoutMinutes * 60_000, { regex: true });
580
+ // OBS-54: headless workers have the same output-inactivity budget as visible panes.
581
+ const stallWindowMs = taskTimeoutMinutes * 60_000;
582
+ let lastOutput = await driver.read(slot, 500);
583
+ let lastOutputAt = Date.now();
584
+ finished = false;
585
+ while (Date.now() - lastOutputAt < stallWindowMs) {
586
+ const remaining = stallWindowMs - (Date.now() - lastOutputAt);
587
+ const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
588
+ if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
589
+ finished = true;
590
+ break;
591
+ }
592
+ const currentOutput = await driver.read(slot, 500);
593
+ if (currentOutput !== lastOutput) {
594
+ lastOutput = currentOutput;
595
+ lastOutputAt = Date.now();
596
+ }
597
+ }
485
598
  output = await driver.read(slot, 500);
486
599
  exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
600
+ timedOut = !finished && Date.now() - lastOutputAt >= stallWindowMs;
487
601
  }
488
602
  // SPEND-01 interactive metering race: the harvest loop breaks on the trailer, but the worker
489
603
  // shell may still be running post-trailer bookkeeping (session-store flush, fake usage stamp,
@@ -511,7 +625,31 @@ export async function runDaemon(repoRoot, opts = {}) {
511
625
  metered++;
512
626
  }
513
627
  const result = adapter.parse(output, nonce);
514
- journal.append("worker-result", t.id, { ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode, mode: interactive ? "interactive" : "print" });
628
+ const cause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut });
629
+ journal.append("worker-result", t.id, {
630
+ ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode,
631
+ mode: interactive ? "interactive" : "print", ...(cause ? { cause } : {}),
632
+ });
633
+ if (result.ok && finished)
634
+ noTrailerStreak.set(channelKey(assignment), 0);
635
+ else if (!finished && cause !== "provider-death") {
636
+ const ck = channelKey(assignment);
637
+ const streak = (noTrailerStreak.get(ck) ?? 0) + 1;
638
+ noTrailerStreak.set(ck, streak);
639
+ // OBS-57: two consecutive no-trailer windows in one run demote the channel for later attempts.
640
+ if (streak >= NO_TRAILER_DEMOTION_STREAK && !demotedChannels.has(ck)) {
641
+ demotedChannels.add(ck);
642
+ journal.append("channel-demotion", t.id, { channel: ck, streak });
643
+ }
644
+ }
645
+ // v1.46 T1: provider-outage requeue — same assignment, no attempt burn, no consult, capped.
646
+ if (cause === "provider-death" && providerDeathRequeues < PROVIDER_DEATH_REQUEUE_CAP) {
647
+ providerDeathRequeues++;
648
+ journal.append("provider-death-requeue", t.id, { attempt, requeue: providerDeathRequeues, assignment });
649
+ await new Promise((r) => setTimeout(r, PROVIDER_DEATH_BACKOFF_MS));
650
+ attempt--;
651
+ continue;
652
+ }
515
653
  // quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
516
654
  // print: guarded on exit code — exit-0 output that merely MENTIONS "rate limit" must not failover
517
655
  // interactive: a harvested trailer beats quota mentions; without one, quota text fails over (spec v1.2 §2)
@@ -632,7 +770,7 @@ export async function runDaemon(repoRoot, opts = {}) {
632
770
  journal.append("tip-moved", t.id, m.tipMoved);
633
771
  if (tipMoves++ === 0)
634
772
  continue gateLoop;
635
- await park(t, "task branch tip moved twice after gating", null, assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
773
+ await park(t, "task branch tip moved twice after gating", "tip-moved", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
636
774
  return;
637
775
  }
638
776
  if (!m.ok) {
@@ -777,7 +915,7 @@ export async function runDaemon(repoRoot, opts = {}) {
777
915
  const tipFail = summary.tipVerify === "failed"
778
916
  ? ` — TIP VERIFY FAILED on ${summary.lastMergedTask ? `last merge ${summary.lastMergedTask}` : "integration tip"}`
779
917
  : "";
780
- await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "attention" });
918
+ await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
781
919
  return summary;
782
920
  }
783
921
  finally {
package/dist/run/git.d.ts CHANGED
@@ -5,7 +5,9 @@ export interface ShResult {
5
5
  timedOut?: boolean;
6
6
  }
7
7
  export declare function sh(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
8
+ export declare function shGit(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
8
9
  export declare function shOk(cmd: string, cwd: string): Promise<string>;
10
+ export declare function shGitOk(cmd: string, cwd: string): Promise<string>;
9
11
  export declare function gitHead(cwd: string): Promise<string>;
10
12
  export declare const sanitizeBranch: (branch: string) => string;
11
13
  export declare const WORKTREES_DIR = "worktrees.noindex";
package/dist/run/git.js CHANGED
@@ -5,12 +5,12 @@ import { shq } from "../adapters/types.js";
5
5
  import { tickmarkrDir } from "../graph/graph.js";
6
6
  // stdin "ignore": same class as HARD-05 / SubprocessDriver — never leave an open pipe a child can block on
7
7
  // (pi -p / codex exec wait for stdin EOF). timedOut distinguishes SIGKILL-timeout from a real nonzero exit.
8
- export function sh(cmd, cwd, timeoutMs = 600000) {
8
+ function shell(cmd, cwd, timeoutMs, login) {
9
9
  return new Promise((resolve) => {
10
10
  // detached: bash gets its own process group so a timeout can kill the whole tree —
11
11
  // SIGKILLing bash alone orphans grandchildren (codex/pi) that hold the stdio pipes
12
12
  // open, so "close" never fires and the promise wedges forever (v1.33.1 init hang).
13
- const p = spawn("bash", ["-lc", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
13
+ const p = spawn("bash", [login ? "-lc" : "-c", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
14
14
  let stdout = "", stderr = "";
15
15
  let timedOut = false, done = false;
16
16
  const finish = (code, err) => {
@@ -38,14 +38,27 @@ export function sh(cmd, cwd, timeoutMs = 600000) {
38
38
  finish(code ?? 1); });
39
39
  });
40
40
  }
41
+ export function sh(cmd, cwd, timeoutMs = 600000) {
42
+ return shell(cmd, cwd, timeoutMs, true);
43
+ }
44
+ // Git plumbing never needs an operator profile; skip login-shell startup and its side effects.
45
+ export function shGit(cmd, cwd, timeoutMs = 600000) {
46
+ return shell(cmd, cwd, timeoutMs, false);
47
+ }
41
48
  export async function shOk(cmd, cwd) {
42
49
  const r = await sh(cmd, cwd);
43
50
  if (r.code !== 0)
44
51
  throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
45
52
  return r.stdout;
46
53
  }
54
+ export async function shGitOk(cmd, cwd) {
55
+ const r = await shGit(cmd, cwd);
56
+ if (r.code !== 0)
57
+ throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
58
+ return r.stdout;
59
+ }
47
60
  export async function gitHead(cwd) {
48
- return (await shOk("git rev-parse HEAD", cwd)).trim();
61
+ return (await shGitOk("git rev-parse HEAD", cwd)).trim();
49
62
  }
50
63
  export const sanitizeBranch = (branch) => branch.replace(/[^\w.-]+/g, "-");
51
64
  const sanitize = sanitizeBranch;
@@ -77,7 +90,7 @@ export async function createWorktree(repo, branch, baseRef) {
77
90
  const dir = join(tickmarkrDir(repo), WORKTREES_DIR, sanitize(branch));
78
91
  if (existsSync(dir))
79
92
  await removeWorktree(repo, dir);
80
- await shOk(`git worktree add -B ${shq(branch)} ${shq(dir)} ${shq(baseRef)}`, repo);
93
+ await shGitOk(`git worktree add -B ${shq(branch)} ${shq(dir)} ${shq(baseRef)}`, repo);
81
94
  linkNodeModules(repo, dir);
82
95
  return dir;
83
96
  }
@@ -116,7 +129,7 @@ export function linkNodeModules(repo, dir, { force = false } = {}) {
116
129
  export const WORKTREE_LAYOUT_CONTRACT = `## Worktree layout contract (harness-provisioned — do not modify)
117
130
  - node_modules is a symlink into the main repo's node_modules, provisioned by tickmarkr. Never commit, delete, or replace it — the harness re-asserts this link before gates run, so modifying it cannot help and may fail your attempt.`;
118
131
  export async function removeWorktree(repo, dir) {
119
- await sh(`git worktree remove --force ${shq(dir)}`, repo); // best-effort; stale dirs are re-added with -B
120
- await sh(`rm -rf ${shq(dir)}`, repo);
121
- await sh("git worktree prune", repo);
132
+ await shGit(`git worktree remove --force ${shq(dir)}`, repo); // best-effort; stale dirs are re-added with -B
133
+ await shGit(`rm -rf ${shq(dir)}`, repo);
134
+ await shGit("git worktree prune", repo);
122
135
  }