tickmarkr 1.45.0 → 1.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,27 @@
1
1
  import { channelKey, channelsFromConfig } from "../adapters/types.js";
2
2
  import { TIER_RANK } from "../config/config.js";
3
3
  import { disallowedBy } from "./preference.js";
4
- import { cellOf, EXPLORE_CAP, explorationBonus, learnedScore } from "./profile.js";
4
+ import { cellOf, EXPLORE_CAP, explorationBonus, learnedScore, MIN_SAMPLES } from "./profile.js";
5
+ export const NO_EXPLORE_ENV = "TICKMARKR_NO_EXPLORE";
6
+ export const QUALITY_ENV = "TICKMARKR_QUALITY";
7
+ const exploreCap = (cfg) => cfg.routing.explore?.cap ?? EXPLORE_CAP;
8
+ const qualityOn = (exploreCtx) => !!exploreCtx?.quality || process.env[QUALITY_ENV] === "1";
9
+ export const raiseTier = (tier) => tier === "cheap" ? "mid" : tier === "mid" ? "frontier" : "frontier";
10
+ const exploreOff = (task, cfg, exploreCtx) => {
11
+ if (qualityOn(exploreCtx) || exploreCtx?.noExplore || process.env[NO_EXPLORE_ENV] === "1")
12
+ return true;
13
+ const e = cfg.routing.explore;
14
+ if (!e)
15
+ return false;
16
+ if (e.mode === "off")
17
+ return true;
18
+ if (e.excludeShapes?.includes(task.shape))
19
+ return true;
20
+ const thr = e.excludeComplexityAtOrAbove;
21
+ if (thr != null && task.complexity >= thr)
22
+ return true;
23
+ return false;
24
+ };
5
25
  const autoPreferList = (doc, shape) => {
6
26
  const v = doc?.[shape];
7
27
  return Array.isArray(v) ? v : undefined;
@@ -44,9 +64,42 @@ function ladderFor(task, entry) {
44
64
  const escalate = task.routingHints?.escalate ?? entry?.escalate ?? true;
45
65
  return escalate ? ["retry", "escalate", "consult", "human"] : ["retry", "consult", "human"];
46
66
  }
47
- export function route(task, cfg, channels, profile, preferCtx) {
67
+ function withoutExcluded(channels, exclude) {
68
+ if (!exclude?.size)
69
+ return channels;
70
+ // OBS-57: in-run demotion after consecutive no-trailer windows — route around poisoned channels for the rest of the run.
71
+ return channels.filter((c) => !exclude.has(channelKey(c)));
72
+ }
73
+ const qualityBound = (quality, configFloor, taskFloor) => {
74
+ if (!quality)
75
+ return undefined;
76
+ if (configFloor)
77
+ return `floor ${configFloor}→${raiseTier(configFloor)} (--quality)`;
78
+ if (taskFloor)
79
+ return `floor ${taskFloor}→${raiseTier(taskFloor)} (--quality)`;
80
+ return undefined;
81
+ };
82
+ const maybeSlaLint = (lints, task, profile, slaMinutes, c) => {
83
+ if (slaMinutes === undefined || !profile)
84
+ return;
85
+ const cell = cellOf(profile, task.shape, channelKey(c), c.channel);
86
+ if (!cell?.doneMedianMs || cell.doneCount < MIN_SAMPLES)
87
+ return;
88
+ const slaMs = slaMinutes * 60_000;
89
+ if (cell.doneMedianMs <= slaMs)
90
+ return;
91
+ const medianMin = Math.round(cell.doneMedianMs / 60_000);
92
+ lints.push(`${task.id} (${task.shape}): median ${medianMin}m exceeds sla ${slaMinutes}m — learned perf term references sla ${slaMinutes}m ref`);
93
+ };
94
+ export function route(task, cfg, channels, profile, preferCtx, exclude, exploreCtx) {
95
+ channels = withoutExcluded(channels, exclude);
48
96
  const lints = [];
49
- const floor = cfg.routing.floors[task.shape];
97
+ const advisoryFloor = cfg.routing.floors[task.shape];
98
+ const quality = qualityOn(exploreCtx);
99
+ const floor = advisoryFloor;
100
+ const slaMinutes = cfg.routing.sla?.[task.shape];
101
+ // ponytail: sla is plan-time advisory only — never thread into learnedScore (would reroute warm rivals).
102
+ const scoreOpts = { availWeight: cfg.routing.learnedTuning?.availWeight };
50
103
  const entry = cfg.routing.map[task.shape];
51
104
  const prefer = effectivePrefer(task.shape, entry, preferCtx);
52
105
  const prefActive = !!(cfg.routing.allow || cfg.routing.deny);
@@ -76,11 +129,12 @@ export function route(task, cfg, channels, profile, preferCtx) {
76
129
  }
77
130
  };
78
131
  const lintFloor = (tier, what) => {
79
- if (floor && TIER_RANK[tier] < TIER_RANK[floor]) {
80
- lints.push(`${task.id} (${task.shape}): ${what} routes ${tier}, below advisory floor ${floor}`);
132
+ if (advisoryFloor && TIER_RANK[tier] < TIER_RANK[advisoryFloor]) {
133
+ lints.push(`${task.id} (${task.shape}): ${what} routes ${tier}, below advisory floor ${advisoryFloor}`);
81
134
  }
82
135
  };
83
- const taskFloor = task.routingHints?.floor;
136
+ const taskFloorRaw = task.routingHints?.floor;
137
+ const taskFloor = taskFloorRaw && quality ? raiseTier(taskFloorRaw) : taskFloorRaw;
84
138
  const source = task.routingHints?.source;
85
139
  const src = source ? `, ${source}` : ""; // never interpolate a possibly-undefined source
86
140
  // task pin: planner-authored, try-first — degrades on miss or below-floor (D-05, research A3), never throws
@@ -91,6 +145,7 @@ export function route(task, cfg, channels, profile, preferCtx) {
91
145
  const c = channels.find((c) => c.adapter === taskPin.via && c.model === taskPin.model);
92
146
  if (c && (!taskFloor || TIER_RANK[c.tier] >= TIER_RANK[taskFloor])) {
93
147
  lintFloor(c.tier, "task pin");
148
+ maybeSlaLint(lints, task, profile, slaMinutes, c);
94
149
  return { assignment: toAssignment(c), ladder: ladderFor(task, entry), lints, provenance: `pin ${taskPin.via}:${taskPin.model} (task hint${src})` };
95
150
  }
96
151
  const why = c ? `below task floor ${taskFloor}` : "unavailable";
@@ -102,10 +157,16 @@ export function route(task, cfg, channels, profile, preferCtx) {
102
157
  disallowedPin(entry.pin.via, entry.pin.model, "map pin (config routing.map)");
103
158
  const c = resolvePin(entry.pin, channels);
104
159
  lintFloor(c.tier, "map pin");
160
+ maybeSlaLint(lints, task, profile, slaMinutes, c);
105
161
  return { assignment: toAssignment(c), ladder: ladderFor(task, entry), lints, provenance: `${degraded}pin ${entry.pin.via}:${entry.pin.model} (config routing.map)` };
106
162
  }
107
163
  const baseTier = entry?.tier ?? floor ?? "cheap";
108
- const minTier = taskFloor && TIER_RANK[taskFloor] > TIER_RANK[baseTier] ? taskFloor : baseTier; // task floor is a hard >= constraint in all paths (D-04)
164
+ let minTier = taskFloor && TIER_RANK[taskFloor] > TIER_RANK[baseTier] ? taskFloor : baseTier; // task floor is a hard >= constraint in all paths (D-04)
165
+ if (quality && advisoryFloor) {
166
+ const raised = raiseTier(advisoryFloor);
167
+ if (TIER_RANK[raised] > TIER_RANK[minTier])
168
+ minTier = raised;
169
+ }
109
170
  if (entry?.tier)
110
171
  lintFloor(entry.tier, "map tier");
111
172
  if (prefActive)
@@ -138,13 +199,15 @@ export function route(task, cfg, channels, profile, preferCtx) {
138
199
  // is keyed by channelKey alone, which would collide only if a fleet exposed the same adapter:model on
139
200
  // both classes at once — impossible under today's scalar TierEntrySchema.channel.
140
201
  // ROUTE-15: availWeight threads from config as a pure param; undefined ⇒ module default (byte-identical).
141
- const scores = new Map(eligibleRaw.map((c) => [channelKey(c), learnedScore(profile, task.shape, channelKey(c), c.channel, { availWeight: cfg.routing.learnedTuning?.availWeight })]));
202
+ const scores = new Map(eligibleRaw.map((c) => [channelKey(c), learnedScore(profile, task.shape, channelKey(c), c.channel, scoreOpts)]));
142
203
  const scoreOf = (c) => scores.get(channelKey(c));
143
204
  // v1.6 Phase 14: exploration bonus precomputed ONCE (Pitfall 5), a new key ABOVE the score. EXP-01 needs
144
205
  // it above the score (a warm-good incumbent can't permanently outrank an under-cap rival in a static tie);
145
206
  // magnitude-free as its own lexicographic key. EXP-02 holds structurally: pins returned at :67/:78 and the
146
207
  // floor filtered eligibleRaw at :89, all upstream of this block, so the bonus decides only within a static tie.
147
- const bonuses = new Map(eligibleRaw.map((c) => [channelKey(c), explorationBonus(cellOf(profile, task.shape, channelKey(c), c.channel))]));
208
+ const cap = exploreCap(cfg);
209
+ const off = exploreOff(task, cfg, exploreCtx);
210
+ const bonuses = new Map(eligibleRaw.map((c) => [channelKey(c), off ? 0 : explorationBonus(cellOf(profile, task.shape, channelKey(c), c.channel), cap)]));
148
211
  const bonusOf = (c) => bonuses.get(channelKey(c));
149
212
  const staticWinner = [...eligibleRaw].sort(staticCmp)[0];
150
213
  // Phase 34 ROUTE-17: prefer becomes a BAND + group-rep keys so exploration fires ACROSS prefer
@@ -175,7 +238,7 @@ export function route(task, cfg, channels, profile, preferCtx) {
175
238
  const diffKey = ru ? firstDiff(w, ru) : -1;
176
239
  const probe = diffKey === 1 || diffKey === 6;
177
240
  if (probe) {
178
- learnedChosen = `via exploration probe (dispatches=${cellOf(profile, task.shape, channelKey(w), w.channel)?.dispatches ?? 0} < ${EXPLORE_CAP})`;
241
+ learnedChosen = `via exploration probe (dispatches=${cellOf(profile, task.shape, channelKey(w), w.channel)?.dispatches ?? 0} < ${cap})`;
179
242
  }
180
243
  else if (diffKey === 2 || diffKey === 7) {
181
244
  const n = cellOf(profile, task.shape, channelKey(w), w.channel)?.n ?? 0;
@@ -188,10 +251,11 @@ export function route(task, cfg, channels, profile, preferCtx) {
188
251
  else {
189
252
  eligible = eligibleRaw.sort(staticCmp); // ROUTE-07/09: literally the v1.5 code path — dead code cannot deviate
190
253
  }
191
- const bound = taskFloor && TIER_RANK[taskFloor] >= TIER_RANK[baseTier] ? `floor ${taskFloor} (task hint${src})` :
192
- entry?.tier ? `tier ${entry.tier} (config routing.map)` :
193
- floor ? `floor ${floor} (config floors)` :
194
- "tier cheap (default)";
254
+ const bound = qualityBound(quality, advisoryFloor, taskFloorRaw) ??
255
+ (taskFloor && TIER_RANK[taskFloor] >= TIER_RANK[baseTier] ? `floor ${taskFloor} (task hint${src})` :
256
+ entry?.tier ? `tier ${entry.tier} (config routing.map)` :
257
+ floor ? `floor ${floor} (config floors)` :
258
+ "tier cheap (default)");
195
259
  // name the key that actually broke the tie: prefer outranks the marginal-cost/tier keys, so if the
196
260
  // winner matched a prefer entry, prefer decided it — not "cheapest sufficient tier" (ROUTE-03, WR-01)
197
261
  const preferVia = preferFromAuto(task.shape, preferCtx)
@@ -199,9 +263,11 @@ export function route(task, cfg, channels, profile, preferCtx) {
199
263
  : "via prefer";
200
264
  const chosenBy = learnedChosen || (prefer && preferIndex(eligible[0], prefer) < prefer.length
201
265
  ? preferVia : "cheapest sufficient tier");
266
+ maybeSlaLint(lints, task, profile, slaMinutes, eligible[0]);
202
267
  return { assignment: toAssignment(eligible[0]), ladder: ladderFor(task, entry), lints, provenance: `${degraded}${bound}, marginal-cost auto (${chosenBy})`, ...(deviation ? { deviation } : {}) };
203
268
  }
204
- export function nextChannel(current, task, cfg, channels, tried, profile) {
269
+ export function nextChannel(current, task, cfg, channels, tried, profile, exclude) {
270
+ channels = withoutExcluded(channels, exclude);
205
271
  // already cheapest-sufficient: TIER_RANK asc is the PRIMARY key so escalation climbs one band at a time.
206
272
  // Do NOT "unify" this onto route()'s key order (marginal-cost first) — that reverses climb-one-band on mixed fleets (ROUTE-02, D2).
207
273
  // ROUTE-13: learnedScore is the STRICTLY-LAST key — within-band tiebreak only. Precomputed
@@ -10,6 +10,11 @@ export interface ConsultVerdict {
10
10
  excludeAdapter?: string;
11
11
  }
12
12
  export declare function renderRetryGuidance(v: ConsultVerdict): string;
13
+ export declare function augmentRetryBrief(feedback: string, opts: {
14
+ attempted: string[];
15
+ carried: string[];
16
+ present: Set<string>;
17
+ }): string;
13
18
  export interface Dossier {
14
19
  taskId: string;
15
20
  trigger: string;
@@ -19,6 +19,41 @@ export function renderRetryGuidance(v) {
19
19
  lines.push(part);
20
20
  return lines.slice(0, MAX_RETRY_GUIDANCE_LINES).map((l) => `- ${l}`).join("\n");
21
21
  }
22
+ const LANDED_COMMIT_PREMISE_RE = /\b(?:already\s+committed|is\s+already\s+committed|landed\s+commit|implementation\s+is\s+already)\b/i;
23
+ const COMMIT_HASH_RE = /\b[0-9a-f]{7,40}\b/gi;
24
+ // OBS-58: name prior attempt commits by hash and drop consult premises about landed work that the
25
+ // fresh worktree does not satisfy — regenerated against post-recreation state, not inherited verbatim.
26
+ export function augmentRetryBrief(feedback, opts) {
27
+ const { attempted, carried, present } = opts;
28
+ const named = [...new Set([...attempted, ...carried])];
29
+ const presentLower = new Set([...present].map((h) => h.toLowerCase()));
30
+ const parts = [];
31
+ if (named.length > 0) {
32
+ parts.push("## Prior attempt commits (by hash)\n"
33
+ + named.map((h) => `- ${h}${presentLower.has(h.toLowerCase()) ? " — present in this worktree" : " — not in this worktree"}`).join("\n"));
34
+ }
35
+ let body = feedback.trim();
36
+ if (body && LANDED_COMMIT_PREMISE_RE.test(body)) {
37
+ const falseHash = [...body.matchAll(COMMIT_HASH_RE)].some((m) => !presentLower.has(m[0].toLowerCase()));
38
+ const falseLanded = carried.length === 0 && attempted.length > 0;
39
+ if (falseHash || falseLanded) {
40
+ body = body
41
+ .split("\n")
42
+ .filter((line) => !LANDED_COMMIT_PREMISE_RE.test(line))
43
+ .join("\n")
44
+ .trim();
45
+ const correction = carried.length > 0
46
+ ? `Prior attempt work is present at: ${carried.join(", ")}.`
47
+ : attempted.length > 0
48
+ ? "Prior attempt commits could not be carried forward onto this worktree — re-land the work or continue from the integration tip."
49
+ : "No prior attempt commits are present in this worktree — implement from the integration tip.";
50
+ body = body ? `${correction}\n\n${body}` : correction;
51
+ }
52
+ }
53
+ if (body)
54
+ parts.push(body);
55
+ return parts.join("\n\n");
56
+ }
22
57
  const ACTIONS = ["retry", "reroute", "decompose", "human"];
23
58
  export function buildDossierPrompt(d, nonce) {
24
59
  return `TICKMARKR-CONSULT
@@ -1,5 +1,6 @@
1
1
  import { randomBytes } from "node:crypto";
2
- import { readFileSync, writeFileSync } from "node:fs";
2
+ import { shq } from "../adapters/types.js";
3
+ import { existsSync, readFileSync, writeFileSync } from "node:fs";
3
4
  import { join } from "node:path";
4
5
  import { trailerPattern, writePrompt } from "../adapters/prompt.js";
5
6
  import { allAdapters, discoverChannels, getAdapter, probeAll, readDoctor } from "../adapters/registry.js";
@@ -10,9 +11,9 @@ import { formatOwnedName } from "../drivers/types.js";
10
11
  import { captureBaseline, detectGateCommands } from "../gates/baseline.js";
11
12
  import { runGates } from "../gates/run-gates.js";
12
13
  import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
13
- import { consult, renderRetryGuidance } from "./consult.js";
14
- import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, WORKTREE_LAYOUT_CONTRACT } from "./git.js";
15
- import { engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
14
+ import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
15
+ import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, shGit, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
16
+ import { classifyWorkerResultCause, engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
16
17
  import { acquireRunLock, releaseRunLock } from "./lock.js";
17
18
  import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
18
19
  import { nextChannel, route } from "../route/router.js";
@@ -27,6 +28,30 @@ export function formatSummary(s) {
27
28
  }
28
29
  const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
29
30
  const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
31
+ const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
32
+ const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
33
+ const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
34
+ async function commitsAheadOf(base, wt) {
35
+ const head = await gitHead(wt);
36
+ if (head === base)
37
+ return [];
38
+ const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, wt);
39
+ if (r.code !== 0)
40
+ return [];
41
+ return r.stdout.trim().split("\n").filter(Boolean);
42
+ }
43
+ async function cherryPickCommits(wt, commits) {
44
+ const carried = [];
45
+ for (const hash of commits) {
46
+ const r = await shGit(`git cherry-pick --no-gpg-sign ${shq(hash)}`, wt);
47
+ if (r.code !== 0) {
48
+ await shGit("git cherry-pick --abort", wt);
49
+ break;
50
+ }
51
+ carried.push(hash);
52
+ }
53
+ return carried;
54
+ }
30
55
  export async function runDaemon(repoRoot, opts = {}) {
31
56
  const cfg = loadConfig(repoRoot, { globalDir: opts.globalDir });
32
57
  const adapters = opts.adapters ?? allAdapters();
@@ -128,6 +153,9 @@ export async function runDaemon(repoRoot, opts = {}) {
128
153
  const keepLlm = keepForever;
129
154
  const keptSlots = [];
130
155
  const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
156
+ // OBS-57: per-run in-run demotion — channels that burn consecutive no-trailer windows route around for later attempts.
157
+ const demotedChannels = new Set();
158
+ const noTrailerStreak = new Map();
131
159
  // OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
132
160
  // terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
133
161
  // owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
@@ -163,14 +191,14 @@ export async function runDaemon(repoRoot, opts = {}) {
163
191
  mergeChain = next.catch(() => undefined);
164
192
  return next;
165
193
  };
166
- // kind is null only at the humanGate call (assignment null no row anyway); gateFails/consults are
167
- // execTask-scoped counters passed in so a park row is a rich verified-failure observation (e.g. ladder-exhausted + gateFails:4).
194
+ // gateFails/consults are execTask-scoped counters passed in so a park row is a rich verified-failure
195
+ // observation (e.g. ladder-exhausted + gateFails:4); every task-human row has a closed kind, never prose alone.
168
196
  const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
169
197
  graph = setStatus(graph, t.id, "human");
170
198
  saveGraph(repoRoot, graph);
171
- journal.append("task-human", t.id, { reason });
199
+ journal.append("task-human", t.id, { reason, kind });
172
200
  if (assignment) {
173
- journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind ?? undefined, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
201
+ journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
174
202
  }
175
203
  await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
176
204
  await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}`, { tier: "attention" });
@@ -183,10 +211,10 @@ export async function runDaemon(repoRoot, opts = {}) {
183
211
  // term) would silently dispatch every unapproved gate that becomes ready during a resume — pinned
184
212
  // by the resume-path guard pin in tests/run/daemon.test.ts (the only test that reaches this guard
185
213
  // on the resume path; a park-then-resume task is filtered out by readyTasks() and never gets here).
186
- await park(t, `humanGate: "${t.title}" requires approval before dispatch`, null, null, 0, startMs);
214
+ await park(t, `humanGate: "${t.title}" requires approval before dispatch`, "human-gate", null, 0, startMs);
187
215
  return;
188
216
  }
189
- const r = route(t, cfg, channels, profile);
217
+ const r = route(t, cfg, channels, profile, undefined, demotedChannels);
190
218
  for (const lint of r.lints)
191
219
  journal.append("routing-lint", t.id, { lint });
192
220
  // VIS-02: journal a deviation from the static choice ONLY when one occurred (greppable absence = no deviation)
@@ -211,7 +239,7 @@ export async function runDaemon(repoRoot, opts = {}) {
211
239
  // trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
212
240
  // release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
213
241
  // EXISTING nextChannel `tried` parameter — zero router changes (D-03).
214
- const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile);
242
+ const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
215
243
  if (next)
216
244
  assignment = next;
217
245
  // ponytail: nextChannel null (every channel already tried / none available) — keep the static
@@ -245,9 +273,9 @@ export async function runDaemon(repoRoot, opts = {}) {
245
273
  // ROUTE-13: learned within-band failover + deviation audit. nextChannel stays pure (route/ never
246
274
  // journals); the daemon compares the learned pick against the static pick and owns the journal write.
247
275
  const failover = (site) => {
248
- const next = nextChannel(assignment, t, cfg, channels, tried, profile);
276
+ const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
249
277
  if (profile && next) {
250
- const staticNext = nextChannel(assignment, t, cfg, channels, tried);
278
+ const staticNext = nextChannel(assignment, t, cfg, channels, tried, undefined, demotedChannels);
251
279
  if (staticNext && channelKey(next) !== channelKey(staticNext)) {
252
280
  journal.append("failover-deviation", t.id, { site, static: channelKey(staticNext), chosen: channelKey(next) });
253
281
  }
@@ -311,7 +339,13 @@ export async function runDaemon(repoRoot, opts = {}) {
311
339
  // the existing attempt-cap check below with zero new code. Fresh path: rs is undefined ⇒ 0.
312
340
  // v1.24 OBS-18: after task-approved{release:attempt-cap}, replay zeros attempts so this loop
313
341
  // starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
342
+ let providerDeathRequeues = 0;
343
+ let providerDeathAttempt = -1;
314
344
  attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
345
+ if (attempt !== providerDeathAttempt) {
346
+ providerDeathRequeues = 0;
347
+ providerDeathAttempt = attempt;
348
+ }
315
349
  // v1.13 (VIS-09 safety): one FRESH nonce per attempt — see the run-scope comment above. A retained
316
350
  // prior-attempt trailer (herdr scrollback / subprocess buffer) must never satisfy this attempt.
317
351
  const nonce = randomBytes(4).toString("hex");
@@ -321,6 +355,16 @@ export async function runDaemon(repoRoot, opts = {}) {
321
355
  await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
322
356
  return;
323
357
  }
358
+ // OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
359
+ if (demotedChannels.has(channelKey(assignment))) {
360
+ const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
361
+ if (next) {
362
+ assignment = next;
363
+ const k = channelKey(next);
364
+ if (!tried.includes(k))
365
+ tried.push(k);
366
+ }
367
+ }
324
368
  // v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
325
369
  // + adapter capability resumes; every other path is today's fresh dispatch.
326
370
  const priorSession = retrySession;
@@ -347,7 +391,27 @@ export async function runDaemon(repoRoot, opts = {}) {
347
391
  journal.append("task-dispatch", t.id, { assignment, attempt, provenance: r.provenance, retryMode });
348
392
  const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
349
393
  const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
394
+ const priorWt = worktreePath(repoRoot, taskBranch);
395
+ const commitsToCarry = existsSync(priorWt) ? await commitsAheadOf(taskBase, priorWt) : [];
350
396
  const wt = await driver.worktree(repoRoot, taskBranch, taskBase);
397
+ // OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
398
+ // cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
399
+ // orphan work a consult already verified as landed.
400
+ let carriedCommits = [];
401
+ if (commitsToCarry.length > 0) {
402
+ carriedCommits = await cherryPickCommits(wt, commitsToCarry);
403
+ journal.append("worktree-recreation", t.id, { attempted: commitsToCarry, carried: carriedCommits });
404
+ }
405
+ const priorNamed = [...new Set([...commitsToCarry, ...carriedCommits])];
406
+ const presentCommits = new Set(carriedCommits);
407
+ for (const h of commitsToCarry) {
408
+ if (!presentCommits.has(h) && (await shGit(`git merge-base --is-ancestor ${shq(h)} HEAD`, wt)).code === 0) {
409
+ presentCommits.add(h);
410
+ }
411
+ }
412
+ if (feedback || priorNamed.length > 0) {
413
+ feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
414
+ }
351
415
  if (cfg.setup) {
352
416
  // v1.22 T3: setup runs inside the task worktree — seal herdr control vars so a setup script
353
417
  // cannot mutate the operator's panes. Worker/judge/review/consult are sealed at the driver
@@ -361,11 +425,13 @@ export async function runDaemon(repoRoot, opts = {}) {
361
425
  }
362
426
  }
363
427
  const promptFile = writePrompt(journal.dir, t, attempt, feedback, nonce);
428
+ // OBS-56: state the non-interactive, one-pass finish contract and the OBS-54 stall budget in every
429
+ // worker prompt, not only consult retry guidance. Prepended so prompt.ts's completion trailer stays last.
430
+ const workerContract = `## Harness contract\n- This harness is non-interactive: make one continuous pass; do not stop for questions or follow-up input.\n- You have a ${taskTimeoutMinutes} minute stall window. Budget the full suite once, then commit and emit the completion trailer before it expires.\n- Each test: acceptance criterion must exist as a vitest test whose title matches the criterion string verbatim.`; // OBS-64
364
431
  // OBS-47: state the worktree layout contract in the worker prompt (cheap-tier workers were
365
- // committing/deleting node_modules and tripping the scope gate). Prepended, not appended, so the
366
- // completion trailer stays the structural last line (prompt.ts's design). The harness re-asserts
367
- // the link itself before gates regardless of what the worker does with it.
368
- writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${readFileSync(promptFile, "utf8")}`);
432
+ // committing/deleting node_modules and tripping the scope gate). The harness re-asserts the link
433
+ // itself before gates regardless of what the worker does with it.
434
+ writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${workerContract}\n\n${readFileSync(promptFile, "utf8")}`);
369
435
  const adapter = getAdapter(assignment.adapter, adapters);
370
436
  // VIS-04: workers share one role tab. T2: `owned` names the pane canonically (ownership contract);
371
437
  // the legacy name stays the fallback for drivers without owned handling (subprocess spies).
@@ -419,6 +485,7 @@ export async function runDaemon(repoRoot, opts = {}) {
419
485
  let finished;
420
486
  let output;
421
487
  let exitCode;
488
+ let timedOut = false;
422
489
  if (interactive) {
423
490
  // v1.2 interactive: the TUI doesn't exit on completion — the trailer is the finish line.
424
491
  // The exit wrapper still fires if the TUI dies (crash/quit): fast-fail instead of burning the timeout.
@@ -429,11 +496,16 @@ export async function runDaemon(repoRoot, opts = {}) {
429
496
  let trustAnswered = false;
430
497
  finished = false;
431
498
  exitCode = null;
432
- output = "";
433
- const deadline = Date.now() + taskTimeoutMinutes * 60_000;
434
- while (Date.now() < deadline) {
499
+ output = await driver.read(slot, 1000);
500
+ // OBS-54: reaping keys on new pane output, not dispatch wall clock. Poll at least twice per
501
+ // stall window (and at the existing 30s cadence for normal windows) so an active worker resets it.
502
+ const stallWindowMs = taskTimeoutMinutes * 60_000;
503
+ let lastOutput = output;
504
+ let lastOutputAt = Date.now();
505
+ while (Date.now() - lastOutputAt < stallWindowMs) {
435
506
  const sliceStart = Date.now();
436
- const slice = Math.min(BLOCKED_POLL_MS, deadline - sliceStart);
507
+ const remaining = stallWindowMs - (sliceStart - lastOutputAt);
508
+ const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
437
509
  if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
438
510
  // verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
439
511
  // own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
@@ -447,6 +519,11 @@ export async function runDaemon(repoRoot, opts = {}) {
447
519
  break;
448
520
  }
449
521
  }
522
+ const currentOutput = await driver.read(slot, 1000);
523
+ if (currentOutput !== lastOutput) {
524
+ lastOutput = currentOutput;
525
+ lastOutputAt = Date.now();
526
+ }
450
527
  // v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
451
528
  await sampleContext();
452
529
  // page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
@@ -486,6 +563,7 @@ export async function runDaemon(repoRoot, opts = {}) {
486
563
  }
487
564
  if (!finished && exitCode === null) {
488
565
  // timed out (or only ever saw false positives): harvest whatever the pane holds now
566
+ timedOut = Date.now() - lastOutputAt >= stallWindowMs;
489
567
  output = await driver.read(slot, 1000);
490
568
  finished = new RegExp(trailerPattern(nonce)).test(output);
491
569
  const exit = exitRe.exec(output);
@@ -499,9 +577,27 @@ export async function runDaemon(repoRoot, opts = {}) {
499
577
  else {
500
578
  const inv = adapter.invoke(t, wt, assignment, { promptFile });
501
579
  await driver.run(slot, `${inv.command}; ${exitMarkerCmd}`);
502
- finished = await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, taskTimeoutMinutes * 60_000, { regex: true });
580
+ // OBS-54: headless workers have the same output-inactivity budget as visible panes.
581
+ const stallWindowMs = taskTimeoutMinutes * 60_000;
582
+ let lastOutput = await driver.read(slot, 500);
583
+ let lastOutputAt = Date.now();
584
+ finished = false;
585
+ while (Date.now() - lastOutputAt < stallWindowMs) {
586
+ const remaining = stallWindowMs - (Date.now() - lastOutputAt);
587
+ const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
588
+ if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
589
+ finished = true;
590
+ break;
591
+ }
592
+ const currentOutput = await driver.read(slot, 500);
593
+ if (currentOutput !== lastOutput) {
594
+ lastOutput = currentOutput;
595
+ lastOutputAt = Date.now();
596
+ }
597
+ }
503
598
  output = await driver.read(slot, 500);
504
599
  exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
600
+ timedOut = !finished && Date.now() - lastOutputAt >= stallWindowMs;
505
601
  }
506
602
  // SPEND-01 interactive metering race: the harvest loop breaks on the trailer, but the worker
507
603
  // shell may still be running post-trailer bookkeeping (session-store flush, fake usage stamp,
@@ -529,7 +625,31 @@ export async function runDaemon(repoRoot, opts = {}) {
529
625
  metered++;
530
626
  }
531
627
  const result = adapter.parse(output, nonce);
532
- journal.append("worker-result", t.id, { ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode, mode: interactive ? "interactive" : "print" });
628
+ const cause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut });
629
+ journal.append("worker-result", t.id, {
630
+ ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode,
631
+ mode: interactive ? "interactive" : "print", ...(cause ? { cause } : {}),
632
+ });
633
+ if (result.ok && finished)
634
+ noTrailerStreak.set(channelKey(assignment), 0);
635
+ else if (!finished && cause !== "provider-death") {
636
+ const ck = channelKey(assignment);
637
+ const streak = (noTrailerStreak.get(ck) ?? 0) + 1;
638
+ noTrailerStreak.set(ck, streak);
639
+ // OBS-57: two consecutive no-trailer windows in one run demote the channel for later attempts.
640
+ if (streak >= NO_TRAILER_DEMOTION_STREAK && !demotedChannels.has(ck)) {
641
+ demotedChannels.add(ck);
642
+ journal.append("channel-demotion", t.id, { channel: ck, streak });
643
+ }
644
+ }
645
+ // v1.46 T1: provider-outage requeue — same assignment, no attempt burn, no consult, capped.
646
+ if (cause === "provider-death" && providerDeathRequeues < PROVIDER_DEATH_REQUEUE_CAP) {
647
+ providerDeathRequeues++;
648
+ journal.append("provider-death-requeue", t.id, { attempt, requeue: providerDeathRequeues, assignment });
649
+ await new Promise((r) => setTimeout(r, PROVIDER_DEATH_BACKOFF_MS));
650
+ attempt--;
651
+ continue;
652
+ }
533
653
  // quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
534
654
  // print: guarded on exit code — exit-0 output that merely MENTIONS "rate limit" must not failover
535
655
  // interactive: a harvested trailer beats quota mentions; without one, quota text fails over (spec v1.2 §2)
@@ -650,7 +770,7 @@ export async function runDaemon(repoRoot, opts = {}) {
650
770
  journal.append("tip-moved", t.id, m.tipMoved);
651
771
  if (tipMoves++ === 0)
652
772
  continue gateLoop;
653
- await park(t, "task branch tip moved twice after gating", null, assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
773
+ await park(t, "task branch tip moved twice after gating", "tip-moved", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
654
774
  return;
655
775
  }
656
776
  if (!m.ok) {
@@ -795,7 +915,7 @@ export async function runDaemon(repoRoot, opts = {}) {
795
915
  const tipFail = summary.tipVerify === "failed"
796
916
  ? ` — TIP VERIFY FAILED on ${summary.lastMergedTask ? `last merge ${summary.lastMergedTask}` : "integration tip"}`
797
917
  : "";
798
- await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "attention" });
918
+ await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
799
919
  return summary;
800
920
  }
801
921
  finally {
package/dist/run/git.d.ts CHANGED
@@ -5,7 +5,9 @@ export interface ShResult {
5
5
  timedOut?: boolean;
6
6
  }
7
7
  export declare function sh(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
8
+ export declare function shGit(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
8
9
  export declare function shOk(cmd: string, cwd: string): Promise<string>;
10
+ export declare function shGitOk(cmd: string, cwd: string): Promise<string>;
9
11
  export declare function gitHead(cwd: string): Promise<string>;
10
12
  export declare const sanitizeBranch: (branch: string) => string;
11
13
  export declare const WORKTREES_DIR = "worktrees.noindex";
package/dist/run/git.js CHANGED
@@ -5,12 +5,12 @@ import { shq } from "../adapters/types.js";
5
5
  import { tickmarkrDir } from "../graph/graph.js";
6
6
  // stdin "ignore": same class as HARD-05 / SubprocessDriver — never leave an open pipe a child can block on
7
7
  // (pi -p / codex exec wait for stdin EOF). timedOut distinguishes SIGKILL-timeout from a real nonzero exit.
8
- export function sh(cmd, cwd, timeoutMs = 600000) {
8
+ function shell(cmd, cwd, timeoutMs, login) {
9
9
  return new Promise((resolve) => {
10
10
  // detached: bash gets its own process group so a timeout can kill the whole tree —
11
11
  // SIGKILLing bash alone orphans grandchildren (codex/pi) that hold the stdio pipes
12
12
  // open, so "close" never fires and the promise wedges forever (v1.33.1 init hang).
13
- const p = spawn("bash", ["-lc", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
13
+ const p = spawn("bash", [login ? "-lc" : "-c", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
14
14
  let stdout = "", stderr = "";
15
15
  let timedOut = false, done = false;
16
16
  const finish = (code, err) => {
@@ -38,14 +38,27 @@ export function sh(cmd, cwd, timeoutMs = 600000) {
38
38
  finish(code ?? 1); });
39
39
  });
40
40
  }
41
+ export function sh(cmd, cwd, timeoutMs = 600000) {
42
+ return shell(cmd, cwd, timeoutMs, true);
43
+ }
44
+ // Git plumbing never needs an operator profile; skip login-shell startup and its side effects.
45
+ export function shGit(cmd, cwd, timeoutMs = 600000) {
46
+ return shell(cmd, cwd, timeoutMs, false);
47
+ }
41
48
  export async function shOk(cmd, cwd) {
42
49
  const r = await sh(cmd, cwd);
43
50
  if (r.code !== 0)
44
51
  throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
45
52
  return r.stdout;
46
53
  }
54
+ export async function shGitOk(cmd, cwd) {
55
+ const r = await shGit(cmd, cwd);
56
+ if (r.code !== 0)
57
+ throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
58
+ return r.stdout;
59
+ }
47
60
  export async function gitHead(cwd) {
48
- return (await shOk("git rev-parse HEAD", cwd)).trim();
61
+ return (await shGitOk("git rev-parse HEAD", cwd)).trim();
49
62
  }
50
63
  export const sanitizeBranch = (branch) => branch.replace(/[^\w.-]+/g, "-");
51
64
  const sanitize = sanitizeBranch;
@@ -77,7 +90,7 @@ export async function createWorktree(repo, branch, baseRef) {
77
90
  const dir = join(tickmarkrDir(repo), WORKTREES_DIR, sanitize(branch));
78
91
  if (existsSync(dir))
79
92
  await removeWorktree(repo, dir);
80
- await shOk(`git worktree add -B ${shq(branch)} ${shq(dir)} ${shq(baseRef)}`, repo);
93
+ await shGitOk(`git worktree add -B ${shq(branch)} ${shq(dir)} ${shq(baseRef)}`, repo);
81
94
  linkNodeModules(repo, dir);
82
95
  return dir;
83
96
  }
@@ -116,7 +129,7 @@ export function linkNodeModules(repo, dir, { force = false } = {}) {
116
129
  export const WORKTREE_LAYOUT_CONTRACT = `## Worktree layout contract (harness-provisioned — do not modify)
117
130
  - node_modules is a symlink into the main repo's node_modules, provisioned by tickmarkr. Never commit, delete, or replace it — the harness re-asserts this link before gates run, so modifying it cannot help and may fail your attempt.`;
118
131
  export async function removeWorktree(repo, dir) {
119
- await sh(`git worktree remove --force ${shq(dir)}`, repo); // best-effort; stale dirs are re-added with -B
120
- await sh(`rm -rf ${shq(dir)}`, repo);
121
- await sh("git worktree prune", repo);
132
+ await shGit(`git worktree remove --force ${shq(dir)}`, repo); // best-effort; stale dirs are re-added with -B
133
+ await shGit(`rm -rf ${shq(dir)}`, repo);
134
+ await shGit("git worktree prune", repo);
122
135
  }