tickmarkr 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +347 -0
- package/dist/adapters/claude-code.d.ts +4 -0
- package/dist/adapters/claude-code.js +191 -0
- package/dist/adapters/codex.d.ts +8 -0
- package/dist/adapters/codex.js +257 -0
- package/dist/adapters/cursor-agent.d.ts +4 -0
- package/dist/adapters/cursor-agent.js +62 -0
- package/dist/adapters/fake.d.ts +45 -0
- package/dist/adapters/fake.js +115 -0
- package/dist/adapters/grok.d.ts +4 -0
- package/dist/adapters/grok.js +122 -0
- package/dist/adapters/model-lints.d.ts +6 -0
- package/dist/adapters/model-lints.js +130 -0
- package/dist/adapters/opencode.d.ts +3 -0
- package/dist/adapters/opencode.js +108 -0
- package/dist/adapters/pi.d.ts +3 -0
- package/dist/adapters/pi.js +191 -0
- package/dist/adapters/prompt.d.ts +6 -0
- package/dist/adapters/prompt.js +84 -0
- package/dist/adapters/registry.d.ts +31 -0
- package/dist/adapters/registry.js +165 -0
- package/dist/adapters/types.d.ts +98 -0
- package/dist/adapters/types.js +56 -0
- package/dist/cli/commands/approve.d.ts +1 -0
- package/dist/cli/commands/approve.js +72 -0
- package/dist/cli/commands/compile.d.ts +1 -0
- package/dist/cli/commands/compile.js +23 -0
- package/dist/cli/commands/doctor.d.ts +2 -0
- package/dist/cli/commands/doctor.js +103 -0
- package/dist/cli/commands/init.d.ts +1 -0
- package/dist/cli/commands/init.js +32 -0
- package/dist/cli/commands/plan.d.ts +2 -0
- package/dist/cli/commands/plan.js +101 -0
- package/dist/cli/commands/profile.d.ts +1 -0
- package/dist/cli/commands/profile.js +56 -0
- package/dist/cli/commands/report.d.ts +4 -0
- package/dist/cli/commands/report.js +306 -0
- package/dist/cli/commands/resume.d.ts +1 -0
- package/dist/cli/commands/resume.js +11 -0
- package/dist/cli/commands/run.d.ts +1 -0
- package/dist/cli/commands/run.js +35 -0
- package/dist/cli/commands/scope.d.ts +3 -0
- package/dist/cli/commands/scope.js +22 -0
- package/dist/cli/commands/status.d.ts +9 -0
- package/dist/cli/commands/status.js +162 -0
- package/dist/cli/commands/unlock.d.ts +1 -0
- package/dist/cli/commands/unlock.js +11 -0
- package/dist/cli/index.d.ts +8 -0
- package/dist/cli/index.js +65 -0
- package/dist/compile/collateral.d.ts +6 -0
- package/dist/compile/collateral.js +128 -0
- package/dist/compile/common.d.ts +12 -0
- package/dist/compile/common.js +56 -0
- package/dist/compile/gsd.d.ts +3 -0
- package/dist/compile/gsd.js +222 -0
- package/dist/compile/index.d.ts +4 -0
- package/dist/compile/index.js +33 -0
- package/dist/compile/native.d.ts +4 -0
- package/dist/compile/native.js +196 -0
- package/dist/compile/prd.d.ts +2 -0
- package/dist/compile/prd.js +64 -0
- package/dist/compile/speckit.d.ts +2 -0
- package/dist/compile/speckit.js +66 -0
- package/dist/config/config.d.ts +184 -0
- package/dist/config/config.js +329 -0
- package/dist/drivers/herdr.d.ts +44 -0
- package/dist/drivers/herdr.js +394 -0
- package/dist/drivers/index.d.ts +3 -0
- package/dist/drivers/index.js +13 -0
- package/dist/drivers/subprocess.d.ts +25 -0
- package/dist/drivers/subprocess.js +106 -0
- package/dist/drivers/types.d.ts +61 -0
- package/dist/drivers/types.js +77 -0
- package/dist/gates/acceptance.d.ts +19 -0
- package/dist/gates/acceptance.js +92 -0
- package/dist/gates/baseline.d.ts +12 -0
- package/dist/gates/baseline.js +97 -0
- package/dist/gates/evidence.d.ts +4 -0
- package/dist/gates/evidence.js +12 -0
- package/dist/gates/llm.d.ts +25 -0
- package/dist/gates/llm.js +101 -0
- package/dist/gates/review.d.ts +12 -0
- package/dist/gates/review.js +80 -0
- package/dist/gates/run-gates.d.ts +34 -0
- package/dist/gates/run-gates.js +105 -0
- package/dist/gates/scope.d.ts +8 -0
- package/dist/gates/scope.js +32 -0
- package/dist/gates/types.d.ts +6 -0
- package/dist/gates/types.js +1 -0
- package/dist/graph/graph.d.ts +20 -0
- package/dist/graph/graph.js +148 -0
- package/dist/graph/schema.d.ts +185 -0
- package/dist/graph/schema.js +141 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/plan/prompt.d.ts +4 -0
- package/dist/plan/prompt.js +31 -0
- package/dist/plan/scope.d.ts +17 -0
- package/dist/plan/scope.js +141 -0
- package/dist/report/cost.d.ts +24 -0
- package/dist/report/cost.js +89 -0
- package/dist/route/preference.d.ts +27 -0
- package/dist/route/preference.js +49 -0
- package/dist/route/profile.d.ts +65 -0
- package/dist/route/profile.js +197 -0
- package/dist/route/router.d.ts +26 -0
- package/dist/route/router.js +200 -0
- package/dist/run/consult.d.ts +23 -0
- package/dist/run/consult.js +80 -0
- package/dist/run/daemon.d.ts +23 -0
- package/dist/run/daemon.js +728 -0
- package/dist/run/git.d.ts +12 -0
- package/dist/run/git.js +56 -0
- package/dist/run/journal.d.ts +91 -0
- package/dist/run/journal.js +287 -0
- package/dist/run/lock.d.ts +29 -0
- package/dist/run/lock.js +182 -0
- package/dist/run/merge.d.ts +12 -0
- package/dist/run/merge.js +44 -0
- package/dist/run/reconcile.d.ts +4 -0
- package/dist/run/reconcile.js +95 -0
- package/package.json +39 -0
- package/schema/rungraph.schema.json +306 -0
|
@@ -0,0 +1,728 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { trailerPattern, writePrompt } from "../adapters/prompt.js";
|
|
5
|
+
import { allAdapters, discoverChannels, getAdapter, probeAll, readDoctor } from "../adapters/registry.js";
|
|
6
|
+
import { addUsage, channelKey, matchesTrustDialog, QUOTA_RE } from "../adapters/types.js";
|
|
7
|
+
import { loadConfig } from "../config/config.js";
|
|
8
|
+
import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js";
|
|
9
|
+
import { formatOwnedName } from "../drivers/types.js";
|
|
10
|
+
import { captureBaseline, detectGateCommands } from "../gates/baseline.js";
|
|
11
|
+
import { runGates } from "../gates/run-gates.js";
|
|
12
|
+
import { addEvidence, attributeBlocked, blockedTasks, getTask, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
|
|
13
|
+
import { consult } from "./consult.js";
|
|
14
|
+
import { gitHead, sh } from "./git.js";
|
|
15
|
+
import { Journal, loadRoutingProfile, newRunId } from "./journal.js";
|
|
16
|
+
import { acquireRunLock, releaseRunLock } from "./lock.js";
|
|
17
|
+
import { ensureIntegration, integrationBranch, integrationHead, mergeTask } from "./merge.js";
|
|
18
|
+
import { nextChannel, route } from "../route/router.js";
|
|
19
|
+
import { desiredPanes } from "./reconcile.js";
|
|
20
|
+
// VIS-01: one formatter, four readers (run-end journal event, run/resume CLI, run-end notify).
|
|
21
|
+
// Parity by construction — every caller renders the same complete bucket line.
|
|
22
|
+
export function formatSummary(s) {
|
|
23
|
+
return `done: ${s.done.length}, failed: ${s.failed.length}, human: ${s.human.length}, blocked: ${s.blocked.length}, pending: ${s.pending.length}\nintegration branch: ${s.branch}`;
|
|
24
|
+
}
|
|
25
|
+
const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
|
|
26
|
+
const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
|
|
27
|
+
export async function runDaemon(repoRoot, opts = {}) {
|
|
28
|
+
const cfg = loadConfig(repoRoot, { globalDir: opts.globalDir });
|
|
29
|
+
const adapters = opts.adapters ?? allAdapters();
|
|
30
|
+
const health = readDoctor(repoRoot) ?? (await probeAll(adapters));
|
|
31
|
+
const channels = discoverChannels(cfg, adapters, health);
|
|
32
|
+
// v1.6 ROUTE-06: build the learned profile ONCE at startup (never per task, never in the comparator).
|
|
33
|
+
// No preview — the daemon honors routing.learned:off and gets undefined; this snapshot is immutable
|
|
34
|
+
// for the run, so this run's own telemetry never feeds back into its own routing.
|
|
35
|
+
const profile = loadRoutingProfile(repoRoot, cfg);
|
|
36
|
+
const driver = opts.driver ?? new SubprocessDriver();
|
|
37
|
+
// HARD-01/02: hold the run lock across the whole read-modify-write of graph.json. Acquire
|
|
38
|
+
// BEFORE loadGraph; release in the finally below (every exit path, incl. throws).
|
|
39
|
+
const runId = opts.runId ?? newRunId();
|
|
40
|
+
// T6 narrator: one live status surface per run (herdr only — driver.narrator is undefined on
|
|
41
|
+
// subprocess, so the optional-chain open below is a no-op there). Declared out here so the finally
|
|
42
|
+
// can close it on every exit path. Cosmetic-only: any failure is swallowed (never affects the run).
|
|
43
|
+
let narrator = null;
|
|
44
|
+
const lock = acquireRunLock(repoRoot, runId);
|
|
45
|
+
try {
|
|
46
|
+
let graph = loadGraph(repoRoot);
|
|
47
|
+
const journal = opts.resume ? Journal.open(repoRoot, runId, opts.narrate) : Journal.create(repoRoot, runId, opts.narrate);
|
|
48
|
+
const branchEvent = opts.resume
|
|
49
|
+
? [...journal.read()].reverse().find((e) => (e.event === "run-start" || e.event === "run-end" || e.event === "merge") && typeof e.data.branch === "string")
|
|
50
|
+
: undefined;
|
|
51
|
+
const recordedBranch = typeof branchEvent?.data.branch === "string" ? branchEvent.data.branch : undefined;
|
|
52
|
+
const branch = recordedBranch
|
|
53
|
+
? branchEvent.event === "merge" ? recordedBranch.slice(0, recordedBranch.lastIndexOf("--")) : recordedBranch
|
|
54
|
+
: opts.resume ? `drovr/${runId}` : integrationBranch(cfg, runId);
|
|
55
|
+
if (lock.reclaimed)
|
|
56
|
+
journal.append("lock-reclaimed", undefined, lock.reclaimed); // HARD-02 audit trail
|
|
57
|
+
// GATE-08 (v1.12): the humanGate guard consults this run's journaled approvals, not just the compiled
|
|
58
|
+
// flag. Built ONCE at startup from the journal; a fresh run's journal is empty ⇒ empty set ⇒ unapproved
|
|
59
|
+
// gates park exactly as today. (D-02 step 3 — the load-bearing change: a command + event WITHOUT this
|
|
60
|
+
// guard change ships a no-op; the task replays to pending, re-enters execTask, and re-parks.)
|
|
61
|
+
const approved = new Set(journal.read().filter((e) => e.event === "task-approved" && e.taskId).map((e) => e.taskId));
|
|
62
|
+
const commands = detectGateCommands(repoRoot, cfg);
|
|
63
|
+
let baseRef;
|
|
64
|
+
let baseline;
|
|
65
|
+
// Phase 46 (RES-01/RES-02): the resume-state map is built ONCE here so execTask closes over it.
|
|
66
|
+
// Empty Map on fresh runs — every seed below conditions on resume.get(t.id), never on opts.resume (the
|
|
67
|
+
// GATE-08 lesson at the humanGate guard: condition on the data, not the code path). Dead-code
|
|
68
|
+
// equivalence to the router.ts:194 profile⇒undefined pattern: no map entry ⇒ today's literal.
|
|
69
|
+
const resume = opts.resume ? journal.replayResumeState() : new Map();
|
|
70
|
+
if (opts.resume) {
|
|
71
|
+
const start = journal.read().find((e) => e.event === "run-start");
|
|
72
|
+
if (!start)
|
|
73
|
+
throw new Error(`journal for ${runId} has no run-start event`);
|
|
74
|
+
baseRef = start.data.baseRef;
|
|
75
|
+
baseline = JSON.parse(readFileSync(join(journal.dir, "baseline.json"), "utf8"));
|
|
76
|
+
for (const [id, st] of journal.replayStatuses()) {
|
|
77
|
+
// operator release: a graph.json edit back to "pending" beats a replayed human/failed park (locked decision 12)
|
|
78
|
+
if ((st === "human" || st === "failed") && getTask(graph, id).status === "pending")
|
|
79
|
+
continue;
|
|
80
|
+
graph = setStatus(graph, id, st);
|
|
81
|
+
}
|
|
82
|
+
journal.append("run-resume", undefined, { pid: process.pid }); // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
baseRef = await gitHead(repoRoot);
|
|
86
|
+
baseline = await captureBaseline(repoRoot, commands);
|
|
87
|
+
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(baseline, null, 2));
|
|
88
|
+
journal.append("run-start", undefined, { pid: process.pid, baseRef, commands, channels: channels.map(channelKey), branch }); // pid: v1.13 (VIS-11) liveness
|
|
89
|
+
}
|
|
90
|
+
// T6: open the narrator AFTER run-start/run-resume is journaled so the watch surface has a run to
|
|
91
|
+
// show. driver.narrator is undefined on subprocess → no-op (subprocess spawns nothing). Swallowed:
|
|
92
|
+
// a failed-to-open or later-dead watch pane never affects the run.
|
|
93
|
+
try {
|
|
94
|
+
narrator = await driver.narrator?.(repoRoot, "drovr status --watch", runId) ?? null;
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
narrator = null; // cosmetic-only — the run proceeds without a live surface
|
|
98
|
+
}
|
|
99
|
+
const intWt = await ensureIntegration(repoRoot, branch, baseRef);
|
|
100
|
+
const concurrency = opts.concurrency ?? cfg.concurrency;
|
|
101
|
+
// v1.1 visibility: role-named slots; panes persist per keepPanes (attempt = v1 close-after-harvest)
|
|
102
|
+
const keepOpen = cfg.visibility.keepPanes !== "attempt";
|
|
103
|
+
// D-07 (HYG-09) fleet hygiene — ephemeral panes self-clean + done means gone. keepLlm decouples the
|
|
104
|
+
// judge/review/consult panes from the worker-oriented keepOpen: they close when their result is read
|
|
105
|
+
// unless the operator opts into "forever" (the keep-everything debug override). Reproduces the
|
|
106
|
+
// leftover-judge-pane incident (under keepPanes:"run" the run-end sweep left them open until the end).
|
|
107
|
+
const keepForever = cfg.visibility.keepPanes === "forever";
|
|
108
|
+
const keepLlm = keepForever;
|
|
109
|
+
const keptSlots = [];
|
|
110
|
+
const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
|
|
111
|
+
// OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
|
|
112
|
+
// terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
|
|
113
|
+
// owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
|
|
114
|
+
// reconcile (optional chain → no-op), so gates and the oracle suite never feel this. keepPanes
|
|
115
|
+
// "forever" is the keep-everything debug override — it disables the sweep entirely.
|
|
116
|
+
const reconcile = async (opts) => {
|
|
117
|
+
if (keepForever)
|
|
118
|
+
return;
|
|
119
|
+
try {
|
|
120
|
+
await driver.reconcile?.(desiredPanes(journal.read(), runId), runId, opts);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
/* cosmetic — visibility is never a gate */
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
await reconcile(); // run start/resume boundary: nothing in flight — full sweep, incl. older runs' leftovers
|
|
127
|
+
// v1.4 self-reference guard: a random nonce on the worker trailer AND exit marker. Displayed
|
|
128
|
+
// source/diffs (e.g. a worker editing drovr's own prompt.ts/daemon.ts) can't know it, so an echoed
|
|
129
|
+
// DROVR_RESULT/DROVR_EXIT literal can never premature-harvest the worker. Quote-split keeps the
|
|
130
|
+
// echoed command line itself from matching the marker it prints.
|
|
131
|
+
//
|
|
132
|
+
// v1.13 (VIS-09 safety, 43-02): the nonce is per-ATTEMPT, declared at the top of the attempt loop
|
|
133
|
+
// below — NOT here at run scope. A run-scoped nonce is a latent hazard: HerdrDriver.read() is
|
|
134
|
+
// `pane read --lines 1000` over scrollback and SubprocessDriver never clears s.buf, so any transcript
|
|
135
|
+
// retention across attempts would let attempt N harvest attempt N-1's DROVR_RESULT out of scrollback
|
|
136
|
+
// as its OWN completion — silently LYING about a worker's outcome. Pinned by the stale-trailer oracle
|
|
137
|
+
// in tests/run/daemon.test.ts ("a retained prior-attempt trailer cannot complete a retry"); a future
|
|
138
|
+
// hoist back to run scope reddens it.
|
|
139
|
+
// merges are serialized — two concurrent `git merge`s in one worktree would corrupt each other
|
|
140
|
+
let mergeChain = Promise.resolve();
|
|
141
|
+
const mergeSerial = (taskBranch, t, gated) => {
|
|
142
|
+
const next = mergeChain.then(() => mergeTask(intWt, taskBranch, `drovr: merge ${t.id} ${t.title}`, gated));
|
|
143
|
+
mergeChain = next.catch(() => undefined);
|
|
144
|
+
return next;
|
|
145
|
+
};
|
|
146
|
+
// kind is null only at the humanGate call (assignment null ⇒ no row anyway); gateFails/consults are
|
|
147
|
+
// execTask-scoped counters passed in so a park row is a rich verified-failure observation (e.g. ladder-exhausted + gateFails:4).
|
|
148
|
+
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
|
|
149
|
+
graph = setStatus(graph, t.id, "human");
|
|
150
|
+
saveGraph(repoRoot, graph);
|
|
151
|
+
journal.append("task-human", t.id, { reason });
|
|
152
|
+
if (assignment) {
|
|
153
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind ?? undefined, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
|
|
154
|
+
}
|
|
155
|
+
await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
|
|
156
|
+
await driver.notify(`drovr ${runId}: ${t.id} needs a human — ${reason}`, { tier: "attention" });
|
|
157
|
+
};
|
|
158
|
+
const execTask = async (t) => {
|
|
159
|
+
const startMs = Date.now();
|
|
160
|
+
if (t.humanGate && !approved.has(t.id)) {
|
|
161
|
+
// GATE-08: the condition is the APPROVAL, never the code path. `!opts.resume` (or any run-phase
|
|
162
|
+
// term) would silently dispatch every unapproved gate that becomes ready during a resume — pinned
|
|
163
|
+
// by the resume-path guard pin in tests/run/daemon.test.ts (the only test that reaches this guard
|
|
164
|
+
// on the resume path; a park-then-resume task is filtered out by readyTasks() and never gets here).
|
|
165
|
+
await park(t, `humanGate: "${t.title}" requires approval before dispatch`, null, null, 0, startMs);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
const r = route(t, cfg, channels, profile);
|
|
169
|
+
for (const lint of r.lints)
|
|
170
|
+
journal.append("routing-lint", t.id, { lint });
|
|
171
|
+
// VIS-02: journal a deviation from the static choice ONLY when one occurred (greppable absence = no deviation)
|
|
172
|
+
if (r.deviation)
|
|
173
|
+
journal.append("route-deviation", t.id, { ...r.deviation, provenance: r.provenance });
|
|
174
|
+
let assignment = r.assignment;
|
|
175
|
+
// Phase 46 (RES-01/RES-02, incident run-20260711-185020 L57-58): resume continues the escalation
|
|
176
|
+
// ladder. Replaying `tried` alone does NOT fix the incident — the first post-resume assignment comes
|
|
177
|
+
// from route() above, which is history-free by design and statically re-picked the consult-banned
|
|
178
|
+
// channel 2 ms after run-resume (daemon.ts:160, the incident's exact mechanism). The override below
|
|
179
|
+
// IS the fix; the tried seed and the attempt-loop start close RES-01/RES-02 alongside it.
|
|
180
|
+
//
|
|
181
|
+
// v1.24 OBS-18: a task-approved{release:attempt-cap} zeros rs.attempts (fresh budget) and clears
|
|
182
|
+
// lastAssignment while keeping tried. Only restore lastAssignment when attempts > 0 — after a
|
|
183
|
+
// fresh-budget release, prefer nextChannel over the surviving tried-list so burned channels are
|
|
184
|
+
// not re-tried first (consult bans / prior failovers survive the release).
|
|
185
|
+
const rs = resume.get(t.id);
|
|
186
|
+
if (rs?.lastAssignment && rs.attempts > 0 && channels.some((c) => channelKey(c) === channelKey(rs.lastAssignment))) {
|
|
187
|
+
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
188
|
+
}
|
|
189
|
+
else if (rs && rs.tried.length) {
|
|
190
|
+
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
191
|
+
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
192
|
+
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
193
|
+
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile);
|
|
194
|
+
if (next)
|
|
195
|
+
assignment = next;
|
|
196
|
+
// ponytail: nextChannel null (every channel already tried / none available) — keep the static
|
|
197
|
+
// assignment and proceed. Dispatching on a previously-tried channel beats deadlocking a resumed
|
|
198
|
+
// run; a park-instead policy can come later if it ever bites.
|
|
199
|
+
}
|
|
200
|
+
// pre-kill invariant: tried always contains the current assignment. Spread, never alias the
|
|
201
|
+
// journal-derived array (no hidden mutation of replayed state).
|
|
202
|
+
const tried = rs?.tried.length ? [...rs.tried] : [channelKey(assignment)];
|
|
203
|
+
if (!tried.includes(channelKey(assignment)))
|
|
204
|
+
tried.push(channelKey(assignment));
|
|
205
|
+
// VIS-02 convention: absence = no seeding happened. The observable surface for criterion 2's
|
|
206
|
+
// exclusion-list-equality oracle. Daemon-side append only — no journal.ts write-path change (Phase 48
|
|
207
|
+
// stays unblocked); inert to replayStatuses (unknown events ignored, pinned at journal.test.ts:70-80).
|
|
208
|
+
if (rs)
|
|
209
|
+
journal.append("resume-restore", t.id, { attempts: rs.attempts, tried: [...tried], assignment });
|
|
210
|
+
const badReviewers = []; // v1.1: reviewer channels that produced unparseable output for this task
|
|
211
|
+
let feedback = "";
|
|
212
|
+
let ladderIdx = 0;
|
|
213
|
+
let modeFallbackNoted = false; // v1.2: journal the interactive→print fallback once per task, not per attempt
|
|
214
|
+
let gateFails = 0; // TEL-02: incremented ONLY where feedback is built from failing gates — never derived from attempts (quota failovers bump attempts too, Pitfall 6)
|
|
215
|
+
let consults = 0; // TEL-02: bumped in the runConsult wrapper so one counter covers all three trigger sites, across the attempt loop
|
|
216
|
+
let tokens; // SPEND-02: accumulated across attempts — parked spend is still spend
|
|
217
|
+
let metered = 0; // SPEND-02: attempts that returned a usage record; distinguishes unmetered from measured-zero
|
|
218
|
+
let tipMoves = 0; // OBS-15: one re-gate allowance per task, never reset by a worker retry
|
|
219
|
+
let retryMode = "fresh";
|
|
220
|
+
let lastContextTokens; // v1.23 reset signal, including stalled/quota attempts
|
|
221
|
+
// v1.29: only a gate-failed attempt can seed same-session retry. The next attempt consumes this
|
|
222
|
+
// once; a changed channel, unknown context, or missing resumeCommand falls back to fresh.
|
|
223
|
+
let retrySession;
|
|
224
|
+
// ROUTE-13: learned within-band failover + deviation audit. nextChannel stays pure (route/ never
|
|
225
|
+
// journals); the daemon compares the learned pick against the static pick and owns the journal write.
|
|
226
|
+
const failover = (site) => {
|
|
227
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile);
|
|
228
|
+
if (profile && next) {
|
|
229
|
+
const staticNext = nextChannel(assignment, t, cfg, channels, tried);
|
|
230
|
+
if (staticNext && channelKey(next) !== channelKey(staticNext)) {
|
|
231
|
+
journal.append("failover-deviation", t.id, { site, static: channelKey(staticNext), chosen: channelKey(next) });
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return next;
|
|
235
|
+
};
|
|
236
|
+
const runConsult = (trigger, transcript, diffOrFeedback, gates) => {
|
|
237
|
+
consults++;
|
|
238
|
+
return consult({
|
|
239
|
+
taskId: t.id, trigger,
|
|
240
|
+
journalTail: JSON.stringify(journal.read().slice(-20)),
|
|
241
|
+
transcript: transcript.slice(-8000),
|
|
242
|
+
diff: diffOrFeedback, gates,
|
|
243
|
+
}, cfg, adapters, driver, repoRoot, journal.dir,
|
|
244
|
+
// D-07: consult panes self-clean when the verdict is read (keepLlm) — only "forever" keeps them.
|
|
245
|
+
{ keep: keepLlm, onSlot: keepLlm ? (s) => keptSlots.push(s) : undefined, runId });
|
|
246
|
+
};
|
|
247
|
+
// returns true → continue attempting, false → task is terminal (parked)
|
|
248
|
+
// trigger (why the consult ran) is threaded in so the decompose/human park keeps its cause —
|
|
249
|
+
// rows 6/10/11 of the mapping (gate-fail vs stall vs merge-conflict) would otherwise conflate.
|
|
250
|
+
const applyVerdict = async (v, attempts, trigger) => {
|
|
251
|
+
journal.append("consult-verdict", t.id, {
|
|
252
|
+
action: v.action, notes: v.notes,
|
|
253
|
+
...(v.excludeAdapter ? { excludeAdapter: v.excludeAdapter } : {}),
|
|
254
|
+
});
|
|
255
|
+
await driver.notify(`drovr ${runId}: ${t.id} consult verdict: ${v.action}`, { tier: "attention" });
|
|
256
|
+
if (v.action === "retry") {
|
|
257
|
+
feedback = v.notes || feedback;
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
if (v.action === "reroute") {
|
|
261
|
+
// OBS-20 / v1.24 T1: adapter-scoped exclusion for environmental CLI failures. Expand the
|
|
262
|
+
// task-local tried list with every available channel of the named adapter, then reuse
|
|
263
|
+
// nextChannel's existing tried parameter — zero router changes (D-03). Unknown adapter
|
|
264
|
+
// (zero matches) is a no-op expansion ⇒ ordinary channel-level reroute. Task-scoped:
|
|
265
|
+
// `tried` lives inside execTask, so a sibling task is unaffected.
|
|
266
|
+
if (v.excludeAdapter) {
|
|
267
|
+
for (const c of channels) {
|
|
268
|
+
if (c.adapter === v.excludeAdapter) {
|
|
269
|
+
const k = channelKey(c);
|
|
270
|
+
if (!tried.includes(k))
|
|
271
|
+
tried.push(k);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
const next = failover("consult-reroute");
|
|
276
|
+
if (next) {
|
|
277
|
+
assignment = next;
|
|
278
|
+
tried.push(channelKey(next));
|
|
279
|
+
return true;
|
|
280
|
+
}
|
|
281
|
+
await park(t, "consult said reroute but every channel is exhausted", "reroute-exhausted", assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
await park(t, `consult verdict: ${v.action} — ${v.notes}`, trigger, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode); // decompose|human
|
|
285
|
+
return false;
|
|
286
|
+
};
|
|
287
|
+
// Phase 46 (RES-01): start at the replayed attempt count; a replayed count ≥ MAX_ATTEMPTS parks via
|
|
288
|
+
// the existing attempt-cap check below with zero new code. Fresh path: rs is undefined ⇒ 0.
|
|
289
|
+
// v1.24 OBS-18: after task-approved{release:attempt-cap}, replay zeros attempts so this loop
|
|
290
|
+
// starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
|
|
291
|
+
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
292
|
+
// v1.13 (VIS-09 safety): one FRESH nonce per attempt — see the run-scope comment above. A retained
|
|
293
|
+
// prior-attempt trailer (herdr scrollback / subprocess buffer) must never satisfy this attempt.
|
|
294
|
+
const nonce = randomBytes(4).toString("hex");
|
|
295
|
+
const exitMarkerCmd = `printf '\\nDROVR_''EXIT_${nonce}:%s\\n' $?`;
|
|
296
|
+
const exitRe = new RegExp(`DROVR_EXIT_${nonce}:(\\d+)`);
|
|
297
|
+
if (attempt >= MAX_ATTEMPTS) {
|
|
298
|
+
await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
// v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
|
|
302
|
+
// + adapter capability resumes; every other path is today's fresh dispatch.
|
|
303
|
+
const priorSession = retrySession;
|
|
304
|
+
retrySession = undefined;
|
|
305
|
+
retryMode = priorSession
|
|
306
|
+
&& priorSession.channel === channelKey(assignment)
|
|
307
|
+
&& priorSession.contextTokens !== undefined
|
|
308
|
+
&& priorSession.contextTokens < cfg.contextWarnTokens
|
|
309
|
+
&& adapters.find((a) => a.id === assignment.adapter)?.resumeCommand
|
|
310
|
+
? "resume"
|
|
311
|
+
: "fresh";
|
|
312
|
+
// v1.23 T3: over-threshold context still forces fresh at the retry boundary; never interrupt a
|
|
313
|
+
// running attempt. Unknown/below emits no reset event.
|
|
314
|
+
if (attempt > 0 && lastContextTokens !== undefined && lastContextTokens >= cfg.contextWarnTokens) {
|
|
315
|
+
journal.append("session-reset", t.id, {
|
|
316
|
+
tokens: lastContextTokens,
|
|
317
|
+
threshold: cfg.contextWarnTokens,
|
|
318
|
+
attempt, // the fresh attempt about to dispatch
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
lastContextTokens = undefined;
|
|
322
|
+
graph = setStatus(graph, t.id, "running");
|
|
323
|
+
saveGraph(repoRoot, graph);
|
|
324
|
+
journal.append("task-dispatch", t.id, { assignment, attempt, provenance: r.provenance, retryMode });
|
|
325
|
+
const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
|
|
326
|
+
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
327
|
+
const wt = await driver.worktree(repoRoot, taskBranch, taskBase);
|
|
328
|
+
if (cfg.setup) {
|
|
329
|
+
// v1.22 T3: setup runs inside the task worktree — seal herdr control vars so a setup script
|
|
330
|
+
// cannot mutate the operator's panes. Worker/judge/review/consult are sealed at the driver
|
|
331
|
+
// boundary (SubprocessDriver spawn env / HerdrDriver pane seed); this is the remaining
|
|
332
|
+
// daemon-owned child shell that is not a driver.slot.
|
|
333
|
+
const sr = await sh(`${herdrSealShellPrefix()}${cfg.setup}`, wt, 10 * 60_000);
|
|
334
|
+
journal.append("worktree-setup", t.id, { code: sr.code });
|
|
335
|
+
if (sr.code !== 0) {
|
|
336
|
+
await park(t, `worktree setup failed (exit ${sr.code}): ${cfg.setup}`, "setup", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
const promptFile = writePrompt(journal.dir, t, attempt, feedback, nonce);
|
|
341
|
+
const adapter = getAdapter(assignment.adapter, adapters);
|
|
342
|
+
// VIS-04: workers share one role tab. T2: `owned` names the pane canonically (ownership contract);
|
|
343
|
+
// the legacy name stays the fallback for drivers without owned handling (subprocess spies).
|
|
344
|
+
const slot = await driver.slot(wt, `${t.id}-worker-${assignment.adapter}-a${attempt}-${runTag}`, { group: "workers", owned: { role: "worker", taskId: t.id, attempt, runId } });
|
|
345
|
+
const sessionId = retryMode === "resume" ? priorSession.id : slot.name;
|
|
346
|
+
const icmd = retryMode === "resume"
|
|
347
|
+
? adapter.resumeCommand(sessionId, promptFile, assignment.model)
|
|
348
|
+
: cfg.visibility.worker === "interactive" && driver.interactive
|
|
349
|
+
? adapter.interactiveCommand(promptFile, assignment.model)
|
|
350
|
+
: null;
|
|
351
|
+
if (cfg.visibility.worker === "interactive" && icmd === null && !modeFallbackNoted) {
|
|
352
|
+
modeFallbackNoted = true;
|
|
353
|
+
journal.append("worker-mode-fallback", t.id, { reason: driver.interactive ? "adapter" : "driver" });
|
|
354
|
+
}
|
|
355
|
+
const interactive = icmd !== null;
|
|
356
|
+
// SPEND-01: this attempt's dispatch wall-clock — the usage collect cursor. Captured once here, the
|
|
357
|
+
// single site, so a test can reason about it; keep Date.now() out of profile.ts (still pure) and
|
|
358
|
+
// out of adapter module scope (the cursor is a parameter, threaded from the daemon).
|
|
359
|
+
const attemptStart = Date.now();
|
|
360
|
+
// v1.23 T2: once-per-attempt latch for context threshold crossing. Sample ONLY at existing poll
|
|
361
|
+
// seams (interactive wait slices) — never a new timer loop. null/unknown usage fails OPEN
|
|
362
|
+
// (never treated as over-threshold). Journal + notify fire at most once while the value stays high.
|
|
363
|
+
let contextWarned = false;
|
|
364
|
+
let contextTokens;
|
|
365
|
+
const sampleContext = async () => {
|
|
366
|
+
if (contextWarned || !adapter.contextUsage)
|
|
367
|
+
return;
|
|
368
|
+
let usage = null;
|
|
369
|
+
try {
|
|
370
|
+
// SessionRef id stays stable across resume attempts; adapters return null on a store miss.
|
|
371
|
+
usage = adapter.contextUsage({ cwd: wt, id: sessionId });
|
|
372
|
+
}
|
|
373
|
+
catch {
|
|
374
|
+
return; // fail-open: a broken reader never blocks the attempt
|
|
375
|
+
}
|
|
376
|
+
if (!usage || typeof usage.tokens !== "number" || !Number.isFinite(usage.tokens))
|
|
377
|
+
return;
|
|
378
|
+
contextTokens = usage.tokens; // last known valid sample, including under-threshold resume candidates
|
|
379
|
+
if (usage.tokens < cfg.contextWarnTokens)
|
|
380
|
+
return;
|
|
381
|
+
contextWarned = true;
|
|
382
|
+
lastContextTokens = usage.tokens;
|
|
383
|
+
journal.append("context-sample", t.id, {
|
|
384
|
+
tokens: usage.tokens,
|
|
385
|
+
...(usage.limit !== undefined ? { limit: usage.limit } : {}),
|
|
386
|
+
threshold: cfg.contextWarnTokens,
|
|
387
|
+
attempt,
|
|
388
|
+
});
|
|
389
|
+
await driver.notify(`drovr ${runId}: ${t.id} context ${usage.tokens} tokens ≥ ${cfg.contextWarnTokens}`, { tier: "attention" });
|
|
390
|
+
};
|
|
391
|
+
let finished;
|
|
392
|
+
let output;
|
|
393
|
+
let exitCode;
|
|
394
|
+
if (interactive) {
|
|
395
|
+
// v1.2 interactive: the TUI doesn't exit on completion — the trailer is the finish line.
|
|
396
|
+
// The exit wrapper still fires if the TUI dies (crash/quit): fast-fail instead of burning the timeout.
|
|
397
|
+
await driver.run(slot, `${icmd}; ${exitMarkerCmd}`);
|
|
398
|
+
let paged = false;
|
|
399
|
+
// v1.22 T5 / OBS-19: auto-answer a fingerprint-matched trust dialog exactly once per slot.
|
|
400
|
+
// Any other blocked/idle dialog still pages the operator (paged latch below).
|
|
401
|
+
let trustAnswered = false;
|
|
402
|
+
finished = false;
|
|
403
|
+
exitCode = null;
|
|
404
|
+
output = "";
|
|
405
|
+
const deadline = Date.now() + cfg.taskTimeoutMinutes * 60_000;
|
|
406
|
+
while (Date.now() < deadline) {
|
|
407
|
+
const sliceStart = Date.now();
|
|
408
|
+
const slice = Math.min(BLOCKED_POLL_MS, deadline - sliceStart);
|
|
409
|
+
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|DROVR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
410
|
+
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing drovr's
|
|
411
|
+
// own source, where "DROVR_EXIT:" is a string literal) must not end the wait. Only a
|
|
412
|
+
// parseable trailer or a digit-suffixed exit marker in the harvest is completion.
|
|
413
|
+
output = await driver.read(slot, 1000); // TUI transcripts carry chrome — read deeper than print's 500
|
|
414
|
+
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
415
|
+
const exit = exitRe.exec(output);
|
|
416
|
+
if (finished || exit) {
|
|
417
|
+
exitCode = exit ? Number(exit[1]) : null; // null ⇔ the TUI is still alive
|
|
418
|
+
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
419
|
+
break;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
// v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
|
|
423
|
+
await sampleContext();
|
|
424
|
+
// page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
|
|
425
|
+
// (live check: cursor's trust dialog scraped as idle). "unknown" never pages — that's just
|
|
426
|
+
// a pane the scraper can't read (subprocess, dead pane); the task timeout covers those.
|
|
427
|
+
const st = paged ? "" : await driver.status(slot);
|
|
428
|
+
if (!paged && (st === "blocked" || st === "idle")) {
|
|
429
|
+
// T5: once-per-slot auto-answer when the adapter declares a trust dialog and the pane
|
|
430
|
+
// text matches. drovr created the worktree from the operator's own repo — safe by construction.
|
|
431
|
+
if (!trustAnswered && adapter.trustDialog && driver.sendKey) {
|
|
432
|
+
try {
|
|
433
|
+
const paneText = await driver.read(slot, 80);
|
|
434
|
+
if (matchesTrustDialog(paneText, adapter.trustDialog)) {
|
|
435
|
+
trustAnswered = true;
|
|
436
|
+
// v1.25 T1: audit trail for live runs — prove the dialog appeared and was answered.
|
|
437
|
+
// Latch + sendKey + no-page continue stay byte-identical; this append is additive only.
|
|
438
|
+
journal.append("trust-auto-answer", t.id, { slot: slot.name, adapter: adapter.id });
|
|
439
|
+
await driver.sendKey(slot, adapter.trustDialog.key);
|
|
440
|
+
const spent = Date.now() - sliceStart;
|
|
441
|
+
if (spent < slice)
|
|
442
|
+
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
443
|
+
continue; // do not page — keep waiting for the trailer
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
catch {
|
|
447
|
+
/* read/send failed — fall through to page the operator */
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
paged = true; // page once — the visible pane is the operator's to unblock; task timeout is the backstop
|
|
451
|
+
const why = st === "blocked" ? "is blocked on a prompt — approve in its pane" : "looks idle without finishing — check its pane";
|
|
452
|
+
await driver.notify(`drovr ${runId}: ${slot.name} ${why}`, { tier: "attention" });
|
|
453
|
+
}
|
|
454
|
+
// a dead pane or a false-positive marker display returns fast — sleep the unspent slice, never hot-spin
|
|
455
|
+
const spent = Date.now() - sliceStart;
|
|
456
|
+
if (spent < slice)
|
|
457
|
+
await new Promise((r) => setTimeout(r, Math.min(slice - spent, 1_000)));
|
|
458
|
+
}
|
|
459
|
+
if (!finished && exitCode === null) {
|
|
460
|
+
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
461
|
+
output = await driver.read(slot, 1000);
|
|
462
|
+
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
463
|
+
const exit = exitRe.exec(output);
|
|
464
|
+
exitCode = exit ? Number(exit[1]) : null;
|
|
465
|
+
}
|
|
466
|
+
if (finished) {
|
|
467
|
+
await driver.waitAgentStatus(slot, "idle", 5_000); // settle, then re-harvest the final render
|
|
468
|
+
output = await driver.read(slot, 1000);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
else {
|
|
472
|
+
const inv = adapter.invoke(t, wt, assignment, { promptFile });
|
|
473
|
+
await driver.run(slot, `${inv.command}; ${exitMarkerCmd}`);
|
|
474
|
+
finished = await driver.waitOutput(slot, `DROVR_EXIT_${nonce}:\\d`, cfg.taskTimeoutMinutes * 60_000, { regex: true });
|
|
475
|
+
output = await driver.read(slot, 500);
|
|
476
|
+
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
477
|
+
}
|
|
478
|
+
if (keepOpen)
|
|
479
|
+
keptSlots.push(slot);
|
|
480
|
+
else
|
|
481
|
+
await driver.close(slot);
|
|
482
|
+
// SPEND-01: usage from the harness's own cwd-keyed structured store, read POST-HOC from disk —
|
|
483
|
+
// `wt` is this task's private worktree, so the path is unique; the read is sliced to records
|
|
484
|
+
// stamped at/after this attempt's dispatch instant. Never the harvested pane text, never the
|
|
485
|
+
// parsed worker trailer. No interactive branch: a TUI writes the same store. undefined ⇒ unmetered.
|
|
486
|
+
// SPEND-02: fold this attempt's slice into the task accumulator only when it's a real observation —
|
|
487
|
+
// an absent record leaves `tokens`/`metered` untouched (never a materialized zero).
|
|
488
|
+
const attemptUsage = adapter.collectUsage?.(wt, attemptStart);
|
|
489
|
+
if (attemptUsage) {
|
|
490
|
+
tokens = addUsage(tokens, attemptUsage);
|
|
491
|
+
metered++;
|
|
492
|
+
}
|
|
493
|
+
const result = adapter.parse(output, nonce);
|
|
494
|
+
journal.append("worker-result", t.id, { ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode, mode: interactive ? "interactive" : "print" });
|
|
495
|
+
// quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
|
|
496
|
+
// print: guarded on exit code — exit-0 output that merely MENTIONS "rate limit" must not failover
|
|
497
|
+
// interactive: a harvested trailer beats quota mentions; without one, quota text fails over (spec v1.2 §2)
|
|
498
|
+
const quotaHit = (interactive ? !finished : exitCode !== 0) && QUOTA_RE.test(output);
|
|
499
|
+
if (quotaHit) {
|
|
500
|
+
const next = failover("quota-failover");
|
|
501
|
+
journal.append("quota-failover", t.id, { from: channelKey(assignment), to: next ? channelKey(next) : null });
|
|
502
|
+
if (next) {
|
|
503
|
+
await driver.notify(`drovr ${runId}: ${t.id} quota failover`, { tier: "attention" });
|
|
504
|
+
// OBS-17 T2: the superseded slot's pane closes AT REROUTE TIME — it holds a throttled
|
|
505
|
+
// dead-end, not failure context; the next safe-point reconcile catches a missed close.
|
|
506
|
+
if (!keepForever) {
|
|
507
|
+
const idx = keptSlots.indexOf(slot);
|
|
508
|
+
if (idx >= 0) {
|
|
509
|
+
keptSlots.splice(idx, 1);
|
|
510
|
+
try {
|
|
511
|
+
await driver.close(slot);
|
|
512
|
+
}
|
|
513
|
+
catch { /* cosmetic — reconcile is the backstop */ }
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
// v1.8 TEL-05 — FROM-channel attribution for mid-task quota failover: `assignment` is still the
|
|
517
|
+
// throttled-away-FROM channel here (before the reassign below). durationMs:0 marks this as a
|
|
518
|
+
// failover FACT, not a timed attempt. The park branch is deliberately NOT written here — park()
|
|
519
|
+
// already records parkKind:"quota", so writing here too would double-count in Phase 26 ROUTE-12.
|
|
520
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts: attempt + 1, outcome: "failed", durationMs: 0, quotaFailover: true, retryMode });
|
|
521
|
+
assignment = next;
|
|
522
|
+
tried.push(channelKey(next));
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
await park(t, "quota exhausted on every eligible channel", "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
if (!finished) {
|
|
529
|
+
// ROUTE-18 (OBS-04): the channel burned a window without emitting a trailer (no-trailer timeout
|
|
530
|
+
// OR trailer-less crash-exit — both finished:false). durationMs:0 marks a FACT row, not a timed
|
|
531
|
+
// attempt; attributed to the still-current assignment (the TEL-05 quotaFailover:382 shape, field
|
|
532
|
+
// swapped). Strictly AFTER the quota check above — the quota branch returns/continues before
|
|
533
|
+
// reaching here, so a quota hit can never also carry overrun (no double-count). Read side: 48-01.
|
|
534
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts: attempt + 1, outcome: "failed", durationMs: 0, overrun: true, retryMode });
|
|
535
|
+
const v = await runConsult("stall", output, exitCode !== null && interactive
|
|
536
|
+
? `worker process exited (code ${exitCode}) without a trailer`
|
|
537
|
+
: `no completion marker within ${cfg.taskTimeoutMinutes}m`, []);
|
|
538
|
+
if (await applyVerdict(v, attempt + 1, "stall"))
|
|
539
|
+
continue;
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
graph = setStatus(graph, t.id, "gated");
|
|
543
|
+
saveGraph(repoRoot, graph);
|
|
544
|
+
const onGate = async (e) => {
|
|
545
|
+
if (e.phase === "start")
|
|
546
|
+
return;
|
|
547
|
+
const g = e.result;
|
|
548
|
+
// GATE-09 (ROADMAP SC-4): journal every judge retry as an attributable event — which gate flaked,
|
|
549
|
+
// which channel flaked, which channel retried — so `drovr journal`/report can distinguish "judge
|
|
550
|
+
// flaked, retried" from "worker failed" (run-20260711-185020 P43-03 L70-72 billed a judge flake as
|
|
551
|
+
// a worker attempt; 47-01 fixed WHO retries, this closes the audit-trail half). The condition is
|
|
552
|
+
// META-ONLY (D-03): gate === "acceptance" + typeof-shape guards on meta.judgeRetry — never a
|
|
553
|
+
// details-regex. The v1.1 review regex below is grandfathered, not precedent. Appended BEFORE the
|
|
554
|
+
// gate-result so attribution precedes the verdict in the stream. secondUnparseable is derived from
|
|
555
|
+
// the final result's meta.unparseable (set by run-gates when the retry ALSO flaked — double-garbage).
|
|
556
|
+
if (g.gate === "acceptance" && typeof g.meta?.judgeRetry === "object" && g.meta.judgeRetry !== null) {
|
|
557
|
+
const jr = g.meta.judgeRetry;
|
|
558
|
+
if (typeof jr.flaked === "string" && typeof jr.retried === "string") {
|
|
559
|
+
journal.append("judge-retry", t.id, {
|
|
560
|
+
gate: "acceptance", flaked: jr.flaked, retried: jr.retried,
|
|
561
|
+
...(g.meta.unparseable === true ? { secondUnparseable: true } : {}),
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
journal.append("gate-result", t.id, { gate: g.gate, pass: g.pass, details: g.details });
|
|
566
|
+
// v1.1 failover: never re-ask a reviewer channel that produced garbage for this task
|
|
567
|
+
if (g.gate === "review" && !g.pass && /unparseable/.test(g.details) && typeof g.meta?.reviewer === "string") {
|
|
568
|
+
badReviewers.push(g.meta.reviewer);
|
|
569
|
+
}
|
|
570
|
+
};
|
|
571
|
+
let results = [];
|
|
572
|
+
let commits = [];
|
|
573
|
+
gateLoop: while (true) {
|
|
574
|
+
const gated = await gitHead(wt);
|
|
575
|
+
({ results, commits } = await runGates(t, {
|
|
576
|
+
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
577
|
+
commands, baseline, channels, adapters, cfg,
|
|
578
|
+
via: cfg.visibility.llm === "pane"
|
|
579
|
+
? {
|
|
580
|
+
driver,
|
|
581
|
+
// D-07: judge/review panes self-clean when their verdict is read (keepLlm) — only "forever" keeps them.
|
|
582
|
+
keep: keepLlm,
|
|
583
|
+
onSlot: keepLlm ? (s) => keptSlots.push(s) : undefined,
|
|
584
|
+
// T2 ownership contract: canonical names (drovr:<role>:<task>:0:<runId>) so reconcile
|
|
585
|
+
// owns judge/review panes; run-gates' -r1 retry suffix becomes attempt 1 in llm.ts.
|
|
586
|
+
// Same-name reuse across worker attempts is safe: panes self-clean when read (keepLlm),
|
|
587
|
+
// and herdr's DEFECT-01 reclaim covers a kept holdover under keepPanes:forever.
|
|
588
|
+
nameFor: (role) => formatOwnedName({ role, taskId: t.id, attempt: 0, runId }),
|
|
589
|
+
// role-tab label (SUP-01): role-first + task id, unique per concurrent instance within a run.
|
|
590
|
+
// Duplicate labels from a resumed run or operator-made tabs are accepted (per-process state).
|
|
591
|
+
labelFor: (role) => `${role.toUpperCase()} ${t.id}`,
|
|
592
|
+
}
|
|
593
|
+
: undefined,
|
|
594
|
+
excludeReviewers: badReviewers,
|
|
595
|
+
onGate,
|
|
596
|
+
}));
|
|
597
|
+
graph = addEvidence(graph, t.id, { commits, gateResults: results, artifacts: [promptFile] });
|
|
598
|
+
saveGraph(repoRoot, graph);
|
|
599
|
+
if (results.every((g) => g.pass)) {
|
|
600
|
+
const m = await mergeSerial(taskBranch, t, gated);
|
|
601
|
+
if (m.tipMoved) {
|
|
602
|
+
journal.append("tip-moved", t.id, m.tipMoved);
|
|
603
|
+
if (tipMoves++ === 0)
|
|
604
|
+
continue gateLoop;
|
|
605
|
+
await park(t, "task branch tip moved twice after gating", null, assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
606
|
+
return;
|
|
607
|
+
}
|
|
608
|
+
if (!m.ok) {
|
|
609
|
+
journal.append("merge-conflict", t.id, { conflict: m.conflict });
|
|
610
|
+
const v = await runConsult("merge-conflict", output, m.conflict ?? "", results);
|
|
611
|
+
if (await applyVerdict(v, attempt + 1, "merge-conflict"))
|
|
612
|
+
continue attempts;
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
graph = setStatus(graph, t.id, "done");
|
|
616
|
+
saveGraph(repoRoot, graph);
|
|
617
|
+
journal.append("task-done", t.id, { attempts: attempt + 1, assignment });
|
|
618
|
+
journal.append("merge", t.id, { branch: taskBranch, commit: await integrationHead(intWt) });
|
|
619
|
+
// firstAttemptOk/gateFails/consults are recorded FACTS, not policy — a parkKind:"stall" row is
|
|
620
|
+
// recorded but NOT quality-negative in v1.6; Phase 12 owns reward policy, so flipping it later needs zero data migration.
|
|
621
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts: attempt + 1, outcome: "done", durationMs: Date.now() - startMs, firstAttemptOk: attempt === 0, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
|
|
622
|
+
// D-07 done means gone (merged-P42-01-worker incident): a merged task's worker pane closes on
|
|
623
|
+
// the task-done path, not at run end. Only THIS successful attempt's `slot` is in scope — prior
|
|
624
|
+
// failed attempts' slots stay in keptSlots governed by keepPanes (they hold failure context the
|
|
625
|
+
// operator may need). keepPanes:"forever" is the keep-everything debug override. Removing from
|
|
626
|
+
// keptSlots guarantees the run-end sweep cannot double-close; the indexOf guard also covers
|
|
627
|
+
// "attempt" (slot already closed per-attempt at the worker line above) — close only what you own (Pitfall 5).
|
|
628
|
+
if (!keepForever) {
|
|
629
|
+
const idx = keptSlots.indexOf(slot);
|
|
630
|
+
if (idx >= 0) {
|
|
631
|
+
keptSlots.splice(idx, 1);
|
|
632
|
+
await driver.close(slot);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
await reconcile({ spareLiveLlm: true }); // task-done is a terminal event — sweep this task's leftovers
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
638
|
+
break gateLoop;
|
|
639
|
+
}
|
|
640
|
+
gateFails++; // this attempt's gates failed — the one place quality degradation is verified (never inferred from attempts)
|
|
641
|
+
retrySession = { channel: channelKey(assignment), id: sessionId, contextTokens };
|
|
642
|
+
feedback = results.filter((g) => !g.pass).map((g) => `${g.gate}: ${g.details}`).join("\n\n");
|
|
643
|
+
const step = r.ladder[Math.min(ladderIdx++, r.ladder.length - 1)];
|
|
644
|
+
journal.append("escalation", t.id, { step, attempt: attempt + 1 });
|
|
645
|
+
await driver.notify(`drovr ${runId}: ${t.id} escalation: ${step}`, { tier: "attention" });
|
|
646
|
+
if (step === "retry")
|
|
647
|
+
continue;
|
|
648
|
+
if (step === "escalate") {
|
|
649
|
+
const next = failover("escalate");
|
|
650
|
+
if (next) {
|
|
651
|
+
assignment = next;
|
|
652
|
+
tried.push(channelKey(next));
|
|
653
|
+
continue;
|
|
654
|
+
}
|
|
655
|
+
// no channel left — fall through to a consult
|
|
656
|
+
}
|
|
657
|
+
if (step === "escalate" || step === "consult") {
|
|
658
|
+
const v = await runConsult("gate-fail", output, feedback, results);
|
|
659
|
+
if (await applyVerdict(v, attempt + 1, "gate-fail"))
|
|
660
|
+
continue;
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
await park(t, "escalation ladder exhausted", "ladder-exhausted", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
};
|
|
667
|
+
const inflight = new Map();
|
|
668
|
+
while (true) {
|
|
669
|
+
const ready = readyTasks(graph)
|
|
670
|
+
.filter((t) => !inflight.has(t.id))
|
|
671
|
+
.slice(0, Math.max(0, concurrency - inflight.size));
|
|
672
|
+
for (const t of ready) {
|
|
673
|
+
const p = execTask(t)
|
|
674
|
+
.catch(async (err) => {
|
|
675
|
+
graph = setStatus(graph, t.id, "failed");
|
|
676
|
+
saveGraph(repoRoot, graph);
|
|
677
|
+
journal.append("task-failed", t.id, { error: String(err) });
|
|
678
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
679
|
+
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
680
|
+
})
|
|
681
|
+
.finally(() => inflight.delete(t.id));
|
|
682
|
+
inflight.set(t.id, p);
|
|
683
|
+
}
|
|
684
|
+
if (inflight.size === 0)
|
|
685
|
+
break;
|
|
686
|
+
await Promise.race(inflight.values());
|
|
687
|
+
}
|
|
688
|
+
// D-07: the sweep now closes only what's LEFT in keptSlots — done-closed worker slots were removed
|
|
689
|
+
// (no double-close) and self-cleaned LLM/consult panes were never added under keepLlm:false. This
|
|
690
|
+
// leaves failed/parked attempts' worker slots, which keep their failure context until run end.
|
|
691
|
+
if (cfg.visibility.keepPanes === "run") {
|
|
692
|
+
for (const s of keptSlots)
|
|
693
|
+
await driver.close(s); // panes persist for the run's duration, then clean up
|
|
694
|
+
}
|
|
695
|
+
saveGraph(repoRoot, graph);
|
|
696
|
+
const byStatus = (s) => graph.tasks.filter((t) => t.status === s).map((t) => t.id);
|
|
697
|
+
// buckets derived from the graph at summary time (D-01/D-02); the loop has exited with
|
|
698
|
+
// inflight.size === 0, so the five buckets sum to graph.tasks.length by construction.
|
|
699
|
+
const summary = {
|
|
700
|
+
runId,
|
|
701
|
+
branch,
|
|
702
|
+
done: byStatus("done"),
|
|
703
|
+
failed: byStatus("failed"),
|
|
704
|
+
human: byStatus("human"),
|
|
705
|
+
blocked: blockedTasks(graph).map((t) => t.id),
|
|
706
|
+
pending: pendingTasks(graph).map((t) => t.id),
|
|
707
|
+
};
|
|
708
|
+
journal.append("run-end", undefined, { ...summary });
|
|
709
|
+
await reconcile(); // run-end boundary: nothing in flight — full sweep (empty desired set)
|
|
710
|
+
// VIS-02: name each blocked subtree by its nearest parked/failed root, e.g. "3 blocked behind P40-02".
|
|
711
|
+
const attribution = [...attributeBlocked(graph).entries()]
|
|
712
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
713
|
+
.map(([root, count]) => `${count} blocked behind ${root}`)
|
|
714
|
+
.join(", ");
|
|
715
|
+
await driver.notify(`drovr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""} — integration branch ${branch} (merge to main is yours)`, { tier: "attention" });
|
|
716
|
+
return summary;
|
|
717
|
+
}
|
|
718
|
+
finally {
|
|
719
|
+
// T6: close the narrator at run end — cosmetic, swallowed (never blocks lock release)
|
|
720
|
+
if (narrator) {
|
|
721
|
+
try {
|
|
722
|
+
await driver.close(narrator);
|
|
723
|
+
}
|
|
724
|
+
catch { /* cosmetic */ }
|
|
725
|
+
}
|
|
726
|
+
releaseRunLock(repoRoot);
|
|
727
|
+
}
|
|
728
|
+
}
|