tickmarkr 1.44.0 → 1.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/fake.js +2 -2
- package/dist/adapters/registry.js +12 -3
- package/dist/brand.js +2 -1
- package/dist/cli/commands/approve.js +6 -7
- package/dist/cli/commands/compile.js +10 -6
- package/dist/cli/commands/init.js +55 -3
- package/dist/cli/commands/profile.js +129 -7
- package/dist/cli/commands/resume.js +5 -1
- package/dist/cli/commands/status.js +6 -4
- package/dist/compile/collateral.js +4 -2
- package/dist/drivers/herdr.d.ts +1 -0
- package/dist/drivers/herdr.js +12 -2
- package/dist/gates/acceptance.d.ts +2 -0
- package/dist/gates/acceptance.js +98 -27
- package/dist/gates/llm.d.ts +8 -0
- package/dist/gates/llm.js +92 -8
- package/dist/gates/review.js +6 -3
- package/dist/gates/scope.js +2 -2
- package/dist/graph/graph.d.ts +1 -0
- package/dist/graph/graph.js +12 -0
- package/dist/plan/prompt.js +1 -1
- package/dist/route/profile.d.ts +22 -0
- package/dist/route/profile.js +42 -10
- package/dist/route/router.d.ts +2 -2
- package/dist/route/router.js +10 -2
- package/dist/run/consult.d.ts +6 -1
- package/dist/run/consult.js +54 -10
- package/dist/run/daemon.d.ts +1 -0
- package/dist/run/daemon.js +165 -27
- package/dist/run/git.d.ts +2 -0
- package/dist/run/git.js +20 -7
- package/dist/run/journal.d.ts +53 -4
- package/dist/run/journal.js +137 -16
- package/dist/run/lock.js +3 -3
- package/dist/run/merge.js +8 -8
- package/package.json +1 -1
package/dist/run/consult.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { getAdapter } from "../adapters/registry.js";
|
|
4
|
-
import { bannerShell, paneDispatchCommand
|
|
5
|
-
import {
|
|
4
|
+
import { bannerShell, paneDispatchCommand } from "../brand.js";
|
|
5
|
+
import { augmentFakeVerdictOutput, extractVerdictJson, gateExitTrailer, gatePaneName, generateVerdictNonce, verdictNonceLine } from "../gates/llm.js";
|
|
6
6
|
import { sh } from "./git.js";
|
|
7
7
|
const MAX_RETRY_GUIDANCE_LINES = 10;
|
|
8
8
|
function guidanceParts(text) {
|
|
@@ -19,8 +19,43 @@ export function renderRetryGuidance(v) {
|
|
|
19
19
|
lines.push(part);
|
|
20
20
|
return lines.slice(0, MAX_RETRY_GUIDANCE_LINES).map((l) => `- ${l}`).join("\n");
|
|
21
21
|
}
|
|
22
|
+
const LANDED_COMMIT_PREMISE_RE = /\b(?:already\s+committed|is\s+already\s+committed|landed\s+commit|implementation\s+is\s+already)\b/i;
|
|
23
|
+
const COMMIT_HASH_RE = /\b[0-9a-f]{7,40}\b/gi;
|
|
24
|
+
// OBS-58: name prior attempt commits by hash and drop consult premises about landed work that the
|
|
25
|
+
// fresh worktree does not satisfy — regenerated against post-recreation state, not inherited verbatim.
|
|
26
|
+
export function augmentRetryBrief(feedback, opts) {
|
|
27
|
+
const { attempted, carried, present } = opts;
|
|
28
|
+
const named = [...new Set([...attempted, ...carried])];
|
|
29
|
+
const presentLower = new Set([...present].map((h) => h.toLowerCase()));
|
|
30
|
+
const parts = [];
|
|
31
|
+
if (named.length > 0) {
|
|
32
|
+
parts.push("## Prior attempt commits (by hash)\n"
|
|
33
|
+
+ named.map((h) => `- ${h}${presentLower.has(h.toLowerCase()) ? " — present in this worktree" : " — not in this worktree"}`).join("\n"));
|
|
34
|
+
}
|
|
35
|
+
let body = feedback.trim();
|
|
36
|
+
if (body && LANDED_COMMIT_PREMISE_RE.test(body)) {
|
|
37
|
+
const falseHash = [...body.matchAll(COMMIT_HASH_RE)].some((m) => !presentLower.has(m[0].toLowerCase()));
|
|
38
|
+
const falseLanded = carried.length === 0 && attempted.length > 0;
|
|
39
|
+
if (falseHash || falseLanded) {
|
|
40
|
+
body = body
|
|
41
|
+
.split("\n")
|
|
42
|
+
.filter((line) => !LANDED_COMMIT_PREMISE_RE.test(line))
|
|
43
|
+
.join("\n")
|
|
44
|
+
.trim();
|
|
45
|
+
const correction = carried.length > 0
|
|
46
|
+
? `Prior attempt work is present at: ${carried.join(", ")}.`
|
|
47
|
+
: attempted.length > 0
|
|
48
|
+
? "Prior attempt commits could not be carried forward onto this worktree — re-land the work or continue from the integration tip."
|
|
49
|
+
: "No prior attempt commits are present in this worktree — implement from the integration tip.";
|
|
50
|
+
body = body ? `${correction}\n\n${body}` : correction;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (body)
|
|
54
|
+
parts.push(body);
|
|
55
|
+
return parts.join("\n\n");
|
|
56
|
+
}
|
|
22
57
|
const ACTIONS = ["retry", "reroute", "decompose", "human"];
|
|
23
|
-
export function buildDossierPrompt(d) {
|
|
58
|
+
export function buildDossierPrompt(d, nonce) {
|
|
24
59
|
return `TICKMARKR-CONSULT
|
|
25
60
|
You are a senior engineering consult for the tickmarkr orchestrator. A worker task hit trouble.
|
|
26
61
|
Read the dossier and return a verdict. Be decisive; cost is the lowest priority, quality the highest.
|
|
@@ -47,8 +82,10 @@ channel of that adapter for this task. Use it for environmental failures ("the C
|
|
|
47
82
|
trust dialog, broken install) — not when a single model produced bad code. Omit for model-level
|
|
48
83
|
reroutes so other models of the same adapter remain eligible.
|
|
49
84
|
|
|
85
|
+
${verdictNonceLine(nonce)}
|
|
86
|
+
|
|
50
87
|
Respond with ONLY this JSON:
|
|
51
|
-
{"action": "retry" | "reroute" | "decompose" | "human", "reason": "why this action", "guidance": "imperative steps for the worker (newline-separated ok)", "excludeAdapter"?: "<adapter-id>"}
|
|
88
|
+
{"nonce": "${nonce}", "action": "retry" | "reroute" | "decompose" | "human", "reason": "why this action", "guidance": "imperative steps for the worker (newline-separated ok)", "excludeAdapter"?: "<adapter-id>"}
|
|
52
89
|
`;
|
|
53
90
|
}
|
|
54
91
|
let consultSeq = 0;
|
|
@@ -57,15 +94,17 @@ export async function consult(d, cfg, adapters, driver, cwd, runDir,
|
|
|
57
94
|
// via SlotOpts.owned; without it the legacy gatePaneName shape survives (non-daemon callers/tests).
|
|
58
95
|
opts = {}) {
|
|
59
96
|
const n = ++consultSeq;
|
|
97
|
+
const nonce = generateVerdictNonce();
|
|
60
98
|
const dir = join(runDir, "consults");
|
|
61
99
|
mkdirSync(dir, { recursive: true });
|
|
62
100
|
const promptFile = join(dir, `${d.taskId}-${n}.md`);
|
|
63
|
-
writeFileSync(promptFile, buildDossierPrompt(d));
|
|
101
|
+
writeFileSync(promptFile, buildDossierPrompt(d, nonce));
|
|
64
102
|
const adapter = getAdapter(cfg.consult.adapter, adapters);
|
|
65
103
|
let out;
|
|
66
104
|
if (cfg.visibility.llm === "headless") {
|
|
67
105
|
const r = await sh(adapter.headlessCommand(promptFile, cfg.consult.model), cwd, cfg.consult.stallMinutes * 60_000);
|
|
68
106
|
out = r.stdout + r.stderr;
|
|
107
|
+
out = augmentFakeVerdictOutput(adapter, out, nonce);
|
|
69
108
|
}
|
|
70
109
|
else {
|
|
71
110
|
// T8: role-first pane name for fleet visibility (consult · T2); consultSeq stays on the dossier artifact only
|
|
@@ -76,16 +115,21 @@ opts = {}) {
|
|
|
76
115
|
opts.onSlot?.(slot);
|
|
77
116
|
const scriptPath = join(dir, `${d.taskId}-${n}.sh`);
|
|
78
117
|
// OBS-50: visible consult panes get the brand banner; headless path above stays banner-free (machine-parsed stdout)
|
|
79
|
-
writeFileSync(scriptPath,
|
|
118
|
+
writeFileSync(scriptPath, [
|
|
119
|
+
"export BASH_SILENCE_DEPRECATION_WARNING=1",
|
|
120
|
+
bannerShell(),
|
|
121
|
+
adapter.headlessCommand(promptFile, cfg.consult.model),
|
|
122
|
+
gateExitTrailer(nonce),
|
|
123
|
+
].join("\n"));
|
|
80
124
|
await driver.run(slot, paneDispatchCommand(scriptPath));
|
|
81
|
-
//
|
|
82
|
-
|
|
83
|
-
await driver.waitOutput(slot, "TICKMARKR_EXIT:\\d", cfg.consult.stallMinutes * 60_000, { regex: true });
|
|
125
|
+
// nonce-suffixed exit only: a dossier quoting tickmarkr's own exit literal must not false-complete.
|
|
126
|
+
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, cfg.consult.stallMinutes * 60_000, { regex: true });
|
|
84
127
|
out = await driver.read(slot, 300);
|
|
85
128
|
if (!opts.keep)
|
|
86
129
|
await driver.close(slot);
|
|
130
|
+
out = augmentFakeVerdictOutput(adapter, out, nonce);
|
|
87
131
|
}
|
|
88
|
-
const v =
|
|
132
|
+
const v = extractVerdictJson(out, nonce);
|
|
89
133
|
if (!v || !ACTIONS.includes(v.action)) {
|
|
90
134
|
return { action: "human", notes: "consult verdict unparseable — failing safe to human" };
|
|
91
135
|
}
|
package/dist/run/daemon.d.ts
CHANGED
package/dist/run/daemon.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { shq } from "../adapters/types.js";
|
|
3
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { join } from "node:path";
|
|
4
5
|
import { trailerPattern, writePrompt } from "../adapters/prompt.js";
|
|
5
6
|
import { allAdapters, discoverChannels, getAdapter, probeAll, readDoctor } from "../adapters/registry.js";
|
|
@@ -9,10 +10,10 @@ import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js
|
|
|
9
10
|
import { formatOwnedName } from "../drivers/types.js";
|
|
10
11
|
import { captureBaseline, detectGateCommands } from "../gates/baseline.js";
|
|
11
12
|
import { runGates } from "../gates/run-gates.js";
|
|
12
|
-
import { addEvidence, attributeBlocked, blockedTasks, getTask, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
|
|
13
|
-
import { consult, renderRetryGuidance } from "./consult.js";
|
|
14
|
-
import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, WORKTREE_LAYOUT_CONTRACT } from "./git.js";
|
|
15
|
-
import { Journal, loadRoutingProfile, newRunId } from "./journal.js";
|
|
13
|
+
import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
|
|
14
|
+
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
15
|
+
import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, shGit, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
16
|
+
import { classifyWorkerResultCause, engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
|
|
16
17
|
import { acquireRunLock, releaseRunLock } from "./lock.js";
|
|
17
18
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
18
19
|
import { nextChannel, route } from "../route/router.js";
|
|
@@ -27,6 +28,30 @@ export function formatSummary(s) {
|
|
|
27
28
|
}
|
|
28
29
|
const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
|
|
29
30
|
const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
|
|
31
|
+
const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
|
|
32
|
+
const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
|
|
33
|
+
const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
|
|
34
|
+
async function commitsAheadOf(base, wt) {
|
|
35
|
+
const head = await gitHead(wt);
|
|
36
|
+
if (head === base)
|
|
37
|
+
return [];
|
|
38
|
+
const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, wt);
|
|
39
|
+
if (r.code !== 0)
|
|
40
|
+
return [];
|
|
41
|
+
return r.stdout.trim().split("\n").filter(Boolean);
|
|
42
|
+
}
|
|
43
|
+
async function cherryPickCommits(wt, commits) {
|
|
44
|
+
const carried = [];
|
|
45
|
+
for (const hash of commits) {
|
|
46
|
+
const r = await shGit(`git cherry-pick --no-gpg-sign ${shq(hash)}`, wt);
|
|
47
|
+
if (r.code !== 0) {
|
|
48
|
+
await shGit("git cherry-pick --abort", wt);
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
carried.push(hash);
|
|
52
|
+
}
|
|
53
|
+
return carried;
|
|
54
|
+
}
|
|
30
55
|
export async function runDaemon(repoRoot, opts = {}) {
|
|
31
56
|
const cfg = loadConfig(repoRoot, { globalDir: opts.globalDir });
|
|
32
57
|
const adapters = opts.adapters ?? allAdapters();
|
|
@@ -74,6 +99,24 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
74
99
|
if (!start)
|
|
75
100
|
throw new Error(`journal for ${runId} has no run-start event`);
|
|
76
101
|
baseRef = start.data.baseRef;
|
|
102
|
+
// T3 (Sol #2 / Fable F2): refuse to replay this journal's task states onto a graph it does not
|
|
103
|
+
// belong to — overlapping ids would inherit foreign done/human/approval state, missing ids throw.
|
|
104
|
+
// The SAME comparator status uses (engagementComparable); one decision, two consumers. Fail closed:
|
|
105
|
+
// no resume path silently accepts a mismatched or unbound journal. --graph-changed is the operator's
|
|
106
|
+
// audited release for the stop-amend-resume workflow, journaling a graph-rehash naming both hashes.
|
|
107
|
+
const loadedHash = graphDefinitionHash(graph);
|
|
108
|
+
const cmp = engagementComparable(journal.read(), loadedHash);
|
|
109
|
+
if (!cmp.comparable) {
|
|
110
|
+
if (!opts.graphChanged) {
|
|
111
|
+
throw new Error(cmp.reason === "unbound"
|
|
112
|
+
? `refusing to resume ${runId}: journal has no recorded graph definition hash (older tickmarkr) — pass --graph-changed to override`
|
|
113
|
+
: `refusing to resume ${runId}: graph changed since this run (recorded ${cmp.recorded} ≠ loaded ${loadedHash}) — pass --graph-changed to override`);
|
|
114
|
+
}
|
|
115
|
+
journal.append("graph-rehash", undefined, {
|
|
116
|
+
from: cmp.reason === "mismatch" ? cmp.recorded : null,
|
|
117
|
+
to: loadedHash,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
77
120
|
baseline = JSON.parse(readFileSync(join(journal.dir, "baseline.json"), "utf8"));
|
|
78
121
|
for (const [id, st] of journal.replayStatuses()) {
|
|
79
122
|
// operator release: a graph.json edit back to "pending" beats a replayed human/failed park (locked decision 12)
|
|
@@ -87,7 +130,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
87
130
|
baseRef = await gitHead(repoRoot);
|
|
88
131
|
baseline = await captureBaseline(repoRoot, commands);
|
|
89
132
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(baseline, null, 2));
|
|
90
|
-
journal.append("run-start", undefined, { pid: process.pid, baseRef, commands, channels: channels.map(channelKey), branch,
|
|
133
|
+
journal.append("run-start", undefined, { pid: process.pid, baseRef, commands, channels: channels.map(channelKey), branch, graphDefinitionHash: graphDefinitionHash(graph) }); // graphDefinitionHash: T3 engagement identity (status+resume share it); pid: v1.13 (VIS-11) liveness
|
|
91
134
|
}
|
|
92
135
|
// T6: open the narrator AFTER run-start/run-resume is journaled so the watch surface has a run to
|
|
93
136
|
// show. driver.narrator is undefined on subprocess → no-op (subprocess spawns nothing). Swallowed:
|
|
@@ -110,6 +153,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
110
153
|
const keepLlm = keepForever;
|
|
111
154
|
const keptSlots = [];
|
|
112
155
|
const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
|
|
156
|
+
// OBS-57: per-run in-run demotion — channels that burn consecutive no-trailer windows route around for later attempts.
|
|
157
|
+
const demotedChannels = new Set();
|
|
158
|
+
const noTrailerStreak = new Map();
|
|
113
159
|
// OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
|
|
114
160
|
// terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
|
|
115
161
|
// owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
|
|
@@ -145,14 +191,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
145
191
|
mergeChain = next.catch(() => undefined);
|
|
146
192
|
return next;
|
|
147
193
|
};
|
|
148
|
-
//
|
|
149
|
-
//
|
|
194
|
+
// gateFails/consults are execTask-scoped counters passed in so a park row is a rich verified-failure
|
|
195
|
+
// observation (e.g. ladder-exhausted + gateFails:4); every task-human row has a closed kind, never prose alone.
|
|
150
196
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
|
|
151
197
|
graph = setStatus(graph, t.id, "human");
|
|
152
198
|
saveGraph(repoRoot, graph);
|
|
153
|
-
journal.append("task-human", t.id, { reason });
|
|
199
|
+
journal.append("task-human", t.id, { reason, kind });
|
|
154
200
|
if (assignment) {
|
|
155
|
-
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind
|
|
201
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
|
|
156
202
|
}
|
|
157
203
|
await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
|
|
158
204
|
await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}`, { tier: "attention" });
|
|
@@ -165,10 +211,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
165
211
|
// term) would silently dispatch every unapproved gate that becomes ready during a resume — pinned
|
|
166
212
|
// by the resume-path guard pin in tests/run/daemon.test.ts (the only test that reaches this guard
|
|
167
213
|
// on the resume path; a park-then-resume task is filtered out by readyTasks() and never gets here).
|
|
168
|
-
await park(t, `humanGate: "${t.title}" requires approval before dispatch`,
|
|
214
|
+
await park(t, `humanGate: "${t.title}" requires approval before dispatch`, "human-gate", null, 0, startMs);
|
|
169
215
|
return;
|
|
170
216
|
}
|
|
171
|
-
const r = route(t, cfg, channels, profile);
|
|
217
|
+
const r = route(t, cfg, channels, profile, undefined, demotedChannels);
|
|
172
218
|
for (const lint of r.lints)
|
|
173
219
|
journal.append("routing-lint", t.id, { lint });
|
|
174
220
|
// VIS-02: journal a deviation from the static choice ONLY when one occurred (greppable absence = no deviation)
|
|
@@ -193,7 +239,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
193
239
|
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
194
240
|
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
195
241
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
196
|
-
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile);
|
|
242
|
+
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
|
|
197
243
|
if (next)
|
|
198
244
|
assignment = next;
|
|
199
245
|
// ponytail: nextChannel null (every channel already tried / none available) — keep the static
|
|
@@ -227,9 +273,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
227
273
|
// ROUTE-13: learned within-band failover + deviation audit. nextChannel stays pure (route/ never
|
|
228
274
|
// journals); the daemon compares the learned pick against the static pick and owns the journal write.
|
|
229
275
|
const failover = (site) => {
|
|
230
|
-
const next = nextChannel(assignment, t, cfg, channels, tried, profile);
|
|
276
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
|
|
231
277
|
if (profile && next) {
|
|
232
|
-
const staticNext = nextChannel(assignment, t, cfg, channels, tried);
|
|
278
|
+
const staticNext = nextChannel(assignment, t, cfg, channels, tried, undefined, demotedChannels);
|
|
233
279
|
if (staticNext && channelKey(next) !== channelKey(staticNext)) {
|
|
234
280
|
journal.append("failover-deviation", t.id, { site, static: channelKey(staticNext), chosen: channelKey(next) });
|
|
235
281
|
}
|
|
@@ -293,7 +339,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
293
339
|
// the existing attempt-cap check below with zero new code. Fresh path: rs is undefined ⇒ 0.
|
|
294
340
|
// v1.24 OBS-18: after task-approved{release:attempt-cap}, replay zeros attempts so this loop
|
|
295
341
|
// starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
|
|
342
|
+
let providerDeathRequeues = 0;
|
|
343
|
+
let providerDeathAttempt = -1;
|
|
296
344
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
345
|
+
if (attempt !== providerDeathAttempt) {
|
|
346
|
+
providerDeathRequeues = 0;
|
|
347
|
+
providerDeathAttempt = attempt;
|
|
348
|
+
}
|
|
297
349
|
// v1.13 (VIS-09 safety): one FRESH nonce per attempt — see the run-scope comment above. A retained
|
|
298
350
|
// prior-attempt trailer (herdr scrollback / subprocess buffer) must never satisfy this attempt.
|
|
299
351
|
const nonce = randomBytes(4).toString("hex");
|
|
@@ -303,6 +355,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
303
355
|
await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
304
356
|
return;
|
|
305
357
|
}
|
|
358
|
+
// OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
|
|
359
|
+
if (demotedChannels.has(channelKey(assignment))) {
|
|
360
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
|
|
361
|
+
if (next) {
|
|
362
|
+
assignment = next;
|
|
363
|
+
const k = channelKey(next);
|
|
364
|
+
if (!tried.includes(k))
|
|
365
|
+
tried.push(k);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
306
368
|
// v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
|
|
307
369
|
// + adapter capability resumes; every other path is today's fresh dispatch.
|
|
308
370
|
const priorSession = retrySession;
|
|
@@ -329,7 +391,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
329
391
|
journal.append("task-dispatch", t.id, { assignment, attempt, provenance: r.provenance, retryMode });
|
|
330
392
|
const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
|
|
331
393
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
394
|
+
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
395
|
+
const commitsToCarry = existsSync(priorWt) ? await commitsAheadOf(taskBase, priorWt) : [];
|
|
332
396
|
const wt = await driver.worktree(repoRoot, taskBranch, taskBase);
|
|
397
|
+
// OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
|
|
398
|
+
// cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
|
|
399
|
+
// orphan work a consult already verified as landed.
|
|
400
|
+
let carriedCommits = [];
|
|
401
|
+
if (commitsToCarry.length > 0) {
|
|
402
|
+
carriedCommits = await cherryPickCommits(wt, commitsToCarry);
|
|
403
|
+
journal.append("worktree-recreation", t.id, { attempted: commitsToCarry, carried: carriedCommits });
|
|
404
|
+
}
|
|
405
|
+
const priorNamed = [...new Set([...commitsToCarry, ...carriedCommits])];
|
|
406
|
+
const presentCommits = new Set(carriedCommits);
|
|
407
|
+
for (const h of commitsToCarry) {
|
|
408
|
+
if (!presentCommits.has(h) && (await shGit(`git merge-base --is-ancestor ${shq(h)} HEAD`, wt)).code === 0) {
|
|
409
|
+
presentCommits.add(h);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
if (feedback || priorNamed.length > 0) {
|
|
413
|
+
feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
|
|
414
|
+
}
|
|
333
415
|
if (cfg.setup) {
|
|
334
416
|
// v1.22 T3: setup runs inside the task worktree — seal herdr control vars so a setup script
|
|
335
417
|
// cannot mutate the operator's panes. Worker/judge/review/consult are sealed at the driver
|
|
@@ -343,11 +425,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
343
425
|
}
|
|
344
426
|
}
|
|
345
427
|
const promptFile = writePrompt(journal.dir, t, attempt, feedback, nonce);
|
|
428
|
+
// OBS-56: state the non-interactive, one-pass finish contract and the OBS-54 stall budget in every
|
|
429
|
+
// worker prompt, not only consult retry guidance. Prepended so prompt.ts's completion trailer stays last.
|
|
430
|
+
const workerContract = `## Harness contract\n- This harness is non-interactive: make one continuous pass; do not stop for questions or follow-up input.\n- You have a ${taskTimeoutMinutes} minute stall window. Budget the full suite once, then commit and emit the completion trailer before it expires.`;
|
|
346
431
|
// OBS-47: state the worktree layout contract in the worker prompt (cheap-tier workers were
|
|
347
|
-
// committing/deleting node_modules and tripping the scope gate).
|
|
348
|
-
//
|
|
349
|
-
|
|
350
|
-
writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${readFileSync(promptFile, "utf8")}`);
|
|
432
|
+
// committing/deleting node_modules and tripping the scope gate). The harness re-asserts the link
|
|
433
|
+
// itself before gates regardless of what the worker does with it.
|
|
434
|
+
writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${workerContract}\n\n${readFileSync(promptFile, "utf8")}`);
|
|
351
435
|
const adapter = getAdapter(assignment.adapter, adapters);
|
|
352
436
|
// VIS-04: workers share one role tab. T2: `owned` names the pane canonically (ownership contract);
|
|
353
437
|
// the legacy name stays the fallback for drivers without owned handling (subprocess spies).
|
|
@@ -401,6 +485,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
401
485
|
let finished;
|
|
402
486
|
let output;
|
|
403
487
|
let exitCode;
|
|
488
|
+
let timedOut = false;
|
|
404
489
|
if (interactive) {
|
|
405
490
|
// v1.2 interactive: the TUI doesn't exit on completion — the trailer is the finish line.
|
|
406
491
|
// The exit wrapper still fires if the TUI dies (crash/quit): fast-fail instead of burning the timeout.
|
|
@@ -411,11 +496,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
411
496
|
let trustAnswered = false;
|
|
412
497
|
finished = false;
|
|
413
498
|
exitCode = null;
|
|
414
|
-
output =
|
|
415
|
-
|
|
416
|
-
|
|
499
|
+
output = await driver.read(slot, 1000);
|
|
500
|
+
// OBS-54: reaping keys on new pane output, not dispatch wall clock. Poll at least twice per
|
|
501
|
+
// stall window (and at the existing 30s cadence for normal windows) so an active worker resets it.
|
|
502
|
+
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
503
|
+
let lastOutput = output;
|
|
504
|
+
let lastOutputAt = Date.now();
|
|
505
|
+
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
417
506
|
const sliceStart = Date.now();
|
|
418
|
-
const
|
|
507
|
+
const remaining = stallWindowMs - (sliceStart - lastOutputAt);
|
|
508
|
+
const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
419
509
|
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
420
510
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
421
511
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
@@ -429,6 +519,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
429
519
|
break;
|
|
430
520
|
}
|
|
431
521
|
}
|
|
522
|
+
const currentOutput = await driver.read(slot, 1000);
|
|
523
|
+
if (currentOutput !== lastOutput) {
|
|
524
|
+
lastOutput = currentOutput;
|
|
525
|
+
lastOutputAt = Date.now();
|
|
526
|
+
}
|
|
432
527
|
// v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
|
|
433
528
|
await sampleContext();
|
|
434
529
|
// page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
|
|
@@ -468,6 +563,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
468
563
|
}
|
|
469
564
|
if (!finished && exitCode === null) {
|
|
470
565
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
566
|
+
timedOut = Date.now() - lastOutputAt >= stallWindowMs;
|
|
471
567
|
output = await driver.read(slot, 1000);
|
|
472
568
|
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
473
569
|
const exit = exitRe.exec(output);
|
|
@@ -481,9 +577,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
481
577
|
else {
|
|
482
578
|
const inv = adapter.invoke(t, wt, assignment, { promptFile });
|
|
483
579
|
await driver.run(slot, `${inv.command}; ${exitMarkerCmd}`);
|
|
484
|
-
|
|
580
|
+
// OBS-54: headless workers have the same output-inactivity budget as visible panes.
|
|
581
|
+
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
582
|
+
let lastOutput = await driver.read(slot, 500);
|
|
583
|
+
let lastOutputAt = Date.now();
|
|
584
|
+
finished = false;
|
|
585
|
+
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
586
|
+
const remaining = stallWindowMs - (Date.now() - lastOutputAt);
|
|
587
|
+
const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
588
|
+
if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
589
|
+
finished = true;
|
|
590
|
+
break;
|
|
591
|
+
}
|
|
592
|
+
const currentOutput = await driver.read(slot, 500);
|
|
593
|
+
if (currentOutput !== lastOutput) {
|
|
594
|
+
lastOutput = currentOutput;
|
|
595
|
+
lastOutputAt = Date.now();
|
|
596
|
+
}
|
|
597
|
+
}
|
|
485
598
|
output = await driver.read(slot, 500);
|
|
486
599
|
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
600
|
+
timedOut = !finished && Date.now() - lastOutputAt >= stallWindowMs;
|
|
487
601
|
}
|
|
488
602
|
// SPEND-01 interactive metering race: the harvest loop breaks on the trailer, but the worker
|
|
489
603
|
// shell may still be running post-trailer bookkeeping (session-store flush, fake usage stamp,
|
|
@@ -511,7 +625,31 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
511
625
|
metered++;
|
|
512
626
|
}
|
|
513
627
|
const result = adapter.parse(output, nonce);
|
|
514
|
-
|
|
628
|
+
const cause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut });
|
|
629
|
+
journal.append("worker-result", t.id, {
|
|
630
|
+
ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode,
|
|
631
|
+
mode: interactive ? "interactive" : "print", ...(cause ? { cause } : {}),
|
|
632
|
+
});
|
|
633
|
+
if (result.ok && finished)
|
|
634
|
+
noTrailerStreak.set(channelKey(assignment), 0);
|
|
635
|
+
else if (!finished && cause !== "provider-death") {
|
|
636
|
+
const ck = channelKey(assignment);
|
|
637
|
+
const streak = (noTrailerStreak.get(ck) ?? 0) + 1;
|
|
638
|
+
noTrailerStreak.set(ck, streak);
|
|
639
|
+
// OBS-57: two consecutive no-trailer windows in one run demote the channel for later attempts.
|
|
640
|
+
if (streak >= NO_TRAILER_DEMOTION_STREAK && !demotedChannels.has(ck)) {
|
|
641
|
+
demotedChannels.add(ck);
|
|
642
|
+
journal.append("channel-demotion", t.id, { channel: ck, streak });
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
// v1.46 T1: provider-outage requeue — same assignment, no attempt burn, no consult, capped.
|
|
646
|
+
if (cause === "provider-death" && providerDeathRequeues < PROVIDER_DEATH_REQUEUE_CAP) {
|
|
647
|
+
providerDeathRequeues++;
|
|
648
|
+
journal.append("provider-death-requeue", t.id, { attempt, requeue: providerDeathRequeues, assignment });
|
|
649
|
+
await new Promise((r) => setTimeout(r, PROVIDER_DEATH_BACKOFF_MS));
|
|
650
|
+
attempt--;
|
|
651
|
+
continue;
|
|
652
|
+
}
|
|
515
653
|
// quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
|
|
516
654
|
// print: guarded on exit code — exit-0 output that merely MENTIONS "rate limit" must not failover
|
|
517
655
|
// interactive: a harvested trailer beats quota mentions; without one, quota text fails over (spec v1.2 §2)
|
|
@@ -632,7 +770,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
632
770
|
journal.append("tip-moved", t.id, m.tipMoved);
|
|
633
771
|
if (tipMoves++ === 0)
|
|
634
772
|
continue gateLoop;
|
|
635
|
-
await park(t, "task branch tip moved twice after gating",
|
|
773
|
+
await park(t, "task branch tip moved twice after gating", "tip-moved", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
636
774
|
return;
|
|
637
775
|
}
|
|
638
776
|
if (!m.ok) {
|
|
@@ -777,7 +915,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
777
915
|
const tipFail = summary.tipVerify === "failed"
|
|
778
916
|
? ` — TIP VERIFY FAILED on ${summary.lastMergedTask ? `last merge ${summary.lastMergedTask}` : "integration tip"}`
|
|
779
917
|
: "";
|
|
780
|
-
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "
|
|
918
|
+
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
|
|
781
919
|
return summary;
|
|
782
920
|
}
|
|
783
921
|
finally {
|
package/dist/run/git.d.ts
CHANGED
|
@@ -5,7 +5,9 @@ export interface ShResult {
|
|
|
5
5
|
timedOut?: boolean;
|
|
6
6
|
}
|
|
7
7
|
export declare function sh(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
|
|
8
|
+
export declare function shGit(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
|
|
8
9
|
export declare function shOk(cmd: string, cwd: string): Promise<string>;
|
|
10
|
+
export declare function shGitOk(cmd: string, cwd: string): Promise<string>;
|
|
9
11
|
export declare function gitHead(cwd: string): Promise<string>;
|
|
10
12
|
export declare const sanitizeBranch: (branch: string) => string;
|
|
11
13
|
export declare const WORKTREES_DIR = "worktrees.noindex";
|
package/dist/run/git.js
CHANGED
|
@@ -5,12 +5,12 @@ import { shq } from "../adapters/types.js";
|
|
|
5
5
|
import { tickmarkrDir } from "../graph/graph.js";
|
|
6
6
|
// stdin "ignore": same class as HARD-05 / SubprocessDriver — never leave an open pipe a child can block on
|
|
7
7
|
// (pi -p / codex exec wait for stdin EOF). timedOut distinguishes SIGKILL-timeout from a real nonzero exit.
|
|
8
|
-
|
|
8
|
+
function shell(cmd, cwd, timeoutMs, login) {
|
|
9
9
|
return new Promise((resolve) => {
|
|
10
10
|
// detached: bash gets its own process group so a timeout can kill the whole tree —
|
|
11
11
|
// SIGKILLing bash alone orphans grandchildren (codex/pi) that hold the stdio pipes
|
|
12
12
|
// open, so "close" never fires and the promise wedges forever (v1.33.1 init hang).
|
|
13
|
-
const p = spawn("bash", ["-lc", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
|
|
13
|
+
const p = spawn("bash", [login ? "-lc" : "-c", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
|
|
14
14
|
let stdout = "", stderr = "";
|
|
15
15
|
let timedOut = false, done = false;
|
|
16
16
|
const finish = (code, err) => {
|
|
@@ -38,14 +38,27 @@ export function sh(cmd, cwd, timeoutMs = 600000) {
|
|
|
38
38
|
finish(code ?? 1); });
|
|
39
39
|
});
|
|
40
40
|
}
|
|
41
|
+
export function sh(cmd, cwd, timeoutMs = 600000) {
|
|
42
|
+
return shell(cmd, cwd, timeoutMs, true);
|
|
43
|
+
}
|
|
44
|
+
// Git plumbing never needs an operator profile; skip login-shell startup and its side effects.
|
|
45
|
+
export function shGit(cmd, cwd, timeoutMs = 600000) {
|
|
46
|
+
return shell(cmd, cwd, timeoutMs, false);
|
|
47
|
+
}
|
|
41
48
|
export async function shOk(cmd, cwd) {
|
|
42
49
|
const r = await sh(cmd, cwd);
|
|
43
50
|
if (r.code !== 0)
|
|
44
51
|
throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
|
|
45
52
|
return r.stdout;
|
|
46
53
|
}
|
|
54
|
+
export async function shGitOk(cmd, cwd) {
|
|
55
|
+
const r = await shGit(cmd, cwd);
|
|
56
|
+
if (r.code !== 0)
|
|
57
|
+
throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
|
|
58
|
+
return r.stdout;
|
|
59
|
+
}
|
|
47
60
|
export async function gitHead(cwd) {
|
|
48
|
-
return (await
|
|
61
|
+
return (await shGitOk("git rev-parse HEAD", cwd)).trim();
|
|
49
62
|
}
|
|
50
63
|
export const sanitizeBranch = (branch) => branch.replace(/[^\w.-]+/g, "-");
|
|
51
64
|
const sanitize = sanitizeBranch;
|
|
@@ -77,7 +90,7 @@ export async function createWorktree(repo, branch, baseRef) {
|
|
|
77
90
|
const dir = join(tickmarkrDir(repo), WORKTREES_DIR, sanitize(branch));
|
|
78
91
|
if (existsSync(dir))
|
|
79
92
|
await removeWorktree(repo, dir);
|
|
80
|
-
await
|
|
93
|
+
await shGitOk(`git worktree add -B ${shq(branch)} ${shq(dir)} ${shq(baseRef)}`, repo);
|
|
81
94
|
linkNodeModules(repo, dir);
|
|
82
95
|
return dir;
|
|
83
96
|
}
|
|
@@ -116,7 +129,7 @@ export function linkNodeModules(repo, dir, { force = false } = {}) {
|
|
|
116
129
|
export const WORKTREE_LAYOUT_CONTRACT = `## Worktree layout contract (harness-provisioned — do not modify)
|
|
117
130
|
- node_modules is a symlink into the main repo's node_modules, provisioned by tickmarkr. Never commit, delete, or replace it — the harness re-asserts this link before gates run, so modifying it cannot help and may fail your attempt.`;
|
|
118
131
|
export async function removeWorktree(repo, dir) {
|
|
119
|
-
await
|
|
120
|
-
await
|
|
121
|
-
await
|
|
132
|
+
await shGit(`git worktree remove --force ${shq(dir)}`, repo); // best-effort; stale dirs are re-added with -B
|
|
133
|
+
await shGit(`rm -rf ${shq(dir)}`, repo);
|
|
134
|
+
await shGit("git worktree prune", repo);
|
|
122
135
|
}
|