tickmarkr 2.3.0 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/catalog-remote.d.ts +12 -4
- package/dist/adapters/catalog-remote.js +97 -45
- package/dist/adapters/catalog.js +5 -3
- package/dist/adapters/claude-code.d.ts +1 -1
- package/dist/adapters/claude-code.js +8 -5
- package/dist/adapters/codex.js +6 -7
- package/dist/adapters/model-lints.d.ts +9 -5
- package/dist/adapters/model-lints.js +56 -15
- package/dist/adapters/model-windows.js +11 -0
- package/dist/adapters/prompt.js +1 -0
- package/dist/adapters/qwen.d.ts +5 -0
- package/dist/adapters/qwen.js +153 -0
- package/dist/adapters/registry.js +13 -1
- package/dist/adapters/types.d.ts +1 -0
- package/dist/adapters/types.js +1 -0
- package/dist/cli/commands/compile.d.ts +3 -0
- package/dist/cli/commands/compile.js +91 -34
- package/dist/cli/commands/doctor.d.ts +4 -3
- package/dist/cli/commands/doctor.js +28 -9
- package/dist/cli/commands/fleet.d.ts +4 -0
- package/dist/cli/commands/fleet.js +60 -15
- package/dist/cli/commands/init.js +12 -13
- package/dist/cli/commands/plan.js +75 -11
- package/dist/cli/commands/run.js +20 -1
- package/dist/cli/commands/status.d.ts +1 -0
- package/dist/cli/commands/status.js +59 -16
- package/dist/cli/commands/verify.d.ts +1 -0
- package/dist/cli/commands/verify.js +5 -0
- package/dist/cli/commands/version.js +2 -2
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +2 -2
- package/dist/compile/collateral.d.ts +14 -12
- package/dist/compile/collateral.js +32 -33
- package/dist/compile/index.d.ts +4 -1
- package/dist/compile/index.js +53 -8
- package/dist/compile/native.d.ts +4 -2
- package/dist/compile/native.js +63 -6
- package/dist/compile/ownership.js +41 -10
- package/dist/config/config.d.ts +1 -0
- package/dist/config/config.js +51 -5
- package/dist/drivers/herdr.d.ts +2 -0
- package/dist/drivers/herdr.js +43 -4
- package/dist/drivers/orca.d.ts +18 -1
- package/dist/drivers/orca.js +163 -15
- package/dist/drivers/types.d.ts +10 -0
- package/dist/gates/baseline.d.ts +26 -2
- package/dist/gates/baseline.js +115 -13
- package/dist/gates/review.d.ts +6 -4
- package/dist/gates/review.js +26 -31
- package/dist/gates/run-gates.d.ts +5 -2
- package/dist/gates/run-gates.js +34 -17
- package/dist/graph/graph.d.ts +20 -0
- package/dist/graph/graph.js +66 -1
- package/dist/route/preference.d.ts +6 -0
- package/dist/route/preference.js +40 -0
- package/dist/route/router.js +15 -2
- package/dist/run/consult.d.ts +1 -0
- package/dist/run/consult.js +35 -7
- package/dist/run/daemon.d.ts +9 -0
- package/dist/run/daemon.js +266 -59
- package/dist/run/git.d.ts +4 -0
- package/dist/run/git.js +51 -6
- package/dist/run/journal.d.ts +1 -1
- package/dist/run/journal.js +5 -2
- package/dist/run/lock.d.ts +6 -0
- package/dist/run/lock.js +41 -1
- package/dist/tui/ink/fleet-app.d.ts +4 -0
- package/dist/tui/ink/fleet-app.js +45 -16
- package/package.json +59 -1
- package/skills/tickmarkr-overseer/SKILL.md +39 -4
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +79 -0
- package/skills/tickmarkr-overseer/scripts/watch-context.sh +90 -4
package/dist/run/daemon.js
CHANGED
|
@@ -4,6 +4,7 @@ import { shq } from "../adapters/types.js";
|
|
|
4
4
|
import { appendFileSync, closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, readlinkSync, readSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { tmpdir } from "node:os";
|
|
6
6
|
import { basename, dirname, isAbsolute, join, posix, relative, resolve, sep } from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
7
8
|
import { stringify } from "yaml";
|
|
8
9
|
import { classifyDeadChannel, NO_TRAILER_SUMMARY, trailerPattern, UNPARSEABLE_TRAILER_SUMMARY, writePrompt } from "../adapters/prompt.js";
|
|
9
10
|
import { allAdapters, getAdapter, probeAll, readDoctor, rolePools } from "../adapters/registry.js";
|
|
@@ -13,7 +14,7 @@ import { collateralHits } from "../compile/collateral.js";
|
|
|
13
14
|
import { DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
14
15
|
import { DeliveryReadinessError } from "../drivers/herdr.js";
|
|
15
16
|
import { driverEvidence } from "../drivers/index.js";
|
|
16
|
-
import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js";
|
|
17
|
+
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
17
18
|
import { formatOwnedName } from "../drivers/types.js";
|
|
18
19
|
import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../gates/baseline.js";
|
|
19
20
|
import { runGates } from "../gates/run-gates.js";
|
|
@@ -74,7 +75,7 @@ export function resolveRunMode(repoRoot, opts = {}) {
|
|
|
74
75
|
// task can fail in routing before task-dispatch, and the catch appends task-failed — treating that
|
|
75
76
|
// as enactment reports "complete" over an approval that never ran (the exact silent-completion this
|
|
76
77
|
// task exists to kill).
|
|
77
|
-
const DISPATCH_ENACTMENT = new Set(["task-dispatch", "repair-dispatch"]);
|
|
78
|
+
const DISPATCH_ENACTMENT = new Set(["task-dispatch", "repair-dispatch", "resume-restore"]);
|
|
78
79
|
const RECHECK_ENACTMENT = "recheck-battery";
|
|
79
80
|
// The ONE approval that enacts without buying a worker: GATE_SATISFIED_RELEASE resumes from the
|
|
80
81
|
// persisted task branch after the approved gate (execTask's satisfiedGate branch), whose first act
|
|
@@ -137,7 +138,8 @@ export function formatSummary(s) {
|
|
|
137
138
|
* run id (cli/commands/status.ts positionalRunId), so naming it here is what stops the board from
|
|
138
139
|
* following the newest journal in a repo that already carries a second, newer run — a board showing
|
|
139
140
|
* the wrong run is a recorded incident (skills/tickmarkr-overseer/SKILL.md). */
|
|
140
|
-
export const
|
|
141
|
+
export const daemonEntrypoint = fileURLToPath(new URL("../cli/index.js", import.meta.url));
|
|
142
|
+
export const watchCommand = (runId) => `${shq(process.execPath)} ${shq(daemonEntrypoint)} status --watch ${shq(runId)}`;
|
|
141
143
|
const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
|
|
142
144
|
// v1.85 T3 (retry economics): two repairs per engagement, then the fresh ladder. A repair re-uses the
|
|
143
145
|
// findings and the landed diff instead of re-buying onboarding; when two of them have not closed the
|
|
@@ -213,7 +215,7 @@ function narrowRepairBattery(failing) {
|
|
|
213
215
|
}
|
|
214
216
|
// v2.0 T2 (OBS-554): the measurement keys run-gates stamps on a GateResult, lifted verbatim onto the
|
|
215
217
|
// gate row. One list, one lift — both onGate sites record through the same helper.
|
|
216
|
-
const GATE_TELEMETRY_KEYS = ["durationMs", "load1Start", "load1End", "selectedDurationMs", "fullDurationMs", "invocations"];
|
|
218
|
+
const GATE_TELEMETRY_KEYS = ["durationMs", "load1Start", "load1End", "load1Max", "load1Mean", "selectedDurationMs", "fullDurationMs", "invocations"];
|
|
217
219
|
const gateMeasurement = (meta = {}) => Object.fromEntries(GATE_TELEMETRY_KEYS.filter((k) => meta[k] !== undefined).map((k) => [k, meta[k]]));
|
|
218
220
|
/**
|
|
219
221
|
* T4 (OBS-265): the journal with the review objections a round did NOT hinge on removed. Judge and
|
|
@@ -289,6 +291,11 @@ function approvedReviewRoundCeiling(events, taskId) {
|
|
|
289
291
|
}
|
|
290
292
|
const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
|
|
291
293
|
export const SUITE_POLL_MS = 250;
|
|
294
|
+
// ponytail: one fixed ceiling on the live-suite wait; the baseline's test ceilingMs is the upgrade path
|
|
295
|
+
export const SUITE_WAIT_CEILING_MS = 600_000;
|
|
296
|
+
let suiteWaitCeilingMs = SUITE_WAIT_CEILING_MS;
|
|
297
|
+
export const setSuiteWaitCeilingForTests = (ms) => { suiteWaitCeilingMs = ms; };
|
|
298
|
+
export const resetSuiteWaitCeilingForTests = () => { suiteWaitCeilingMs = SUITE_WAIT_CEILING_MS; };
|
|
292
299
|
export const APPROVAL_POLL_MS = 250;
|
|
293
300
|
const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
|
|
294
301
|
const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
|
|
@@ -339,6 +346,49 @@ export function resetNudgeTimingForTests() {
|
|
|
339
346
|
// diff merely quotes "rate limit" keeps working undisturbed).
|
|
340
347
|
const QUOTA_BANNER_SILENT_MS = 3 * 60_000;
|
|
341
348
|
let quotaBannerSilentMs = QUOTA_BANNER_SILENT_MS;
|
|
349
|
+
const WORKER_STARTUP_FAILURE_RE = /(?:not logged in|authentication(?:[ _-](?:required|failed|error))|401 unauthorized|model[_ -]?not[_ -]?found|(?:model|deployment)[^\n]{0,80}(?:not found|does not exist|is unavailable)|(?:404|not[_ -]?found)[^\n]{0,80}(?:model|deployment))/i;
|
|
350
|
+
// A startup banner is evidence only while the process is starting. Applying this matcher to every
|
|
351
|
+
// later poll let ordinary worker prose (including a diff that mentions model-not-found handling)
|
|
352
|
+
// condemn a live attempt. Bound both dimensions: time prevents a late line from becoming startup
|
|
353
|
+
// evidence, and bytes prevent a chatty launch from keeping an ever-growing classification surface.
|
|
354
|
+
export const WORKER_STARTUP_WINDOW_MS = EARLY_LAUNCH_LIVENESS_MS;
|
|
355
|
+
export const WORKER_STARTUP_WINDOW_BYTES = 64 * 1024;
|
|
356
|
+
let workerStartupWindowMs = WORKER_STARTUP_WINDOW_MS;
|
|
357
|
+
/** Test seam — exercises both sides of the startup boundary without a production-length fixture. */
|
|
358
|
+
export function setWorkerStartupWindowMsForTests(ms) {
|
|
359
|
+
workerStartupWindowMs = ms;
|
|
360
|
+
}
|
|
361
|
+
export function resetWorkerStartupWindowMsForTests() {
|
|
362
|
+
workerStartupWindowMs = WORKER_STARTUP_WINDOW_MS;
|
|
363
|
+
}
|
|
364
|
+
function startupFailureInWindow(output, launchedAt) {
|
|
365
|
+
if (launchedAt === undefined || Date.now() - launchedAt > workerStartupWindowMs)
|
|
366
|
+
return false;
|
|
367
|
+
const bytes = Buffer.from(output);
|
|
368
|
+
const bounded = bytes.length <= WORKER_STARTUP_WINDOW_BYTES
|
|
369
|
+
? output
|
|
370
|
+
: bytes.subarray(0, WORKER_STARTUP_WINDOW_BYTES).toString("utf8");
|
|
371
|
+
return WORKER_STARTUP_FAILURE_RE.test(stallSnapshotBannerRows(bounded));
|
|
372
|
+
}
|
|
373
|
+
// OBS-901/906: one daemon-owned writer for every execution surface. Drivers expose one retained
|
|
374
|
+
// stream through read(); the daemon persists that exact surface before any close/reconcile can make
|
|
375
|
+
// it disappear. MAX_BUF is the execution layer's existing two-MiB retention contract; truncate the
|
|
376
|
+
// encoded bytes from the head so the completion/error tail is always the part that survives.
|
|
377
|
+
async function captureWorkerStream(journalDir, taskId, attempt, driver, slot, fallback) {
|
|
378
|
+
let captured = fallback;
|
|
379
|
+
try {
|
|
380
|
+
captured = await driver.read(slot, PANE_READ_ROWS);
|
|
381
|
+
}
|
|
382
|
+
catch {
|
|
383
|
+
// Earlier successful reads are still a captured stream. Persist them rather than losing the
|
|
384
|
+
// only failure evidence because the pane vanished between its terminal read and this snapshot.
|
|
385
|
+
}
|
|
386
|
+
const bytes = Buffer.from(captured);
|
|
387
|
+
const tail = bytes.length > MAX_BUF ? bytes.subarray(bytes.length - MAX_BUF) : bytes;
|
|
388
|
+
const path = posix.join("prompts", `${taskId}-a${attempt}.out`);
|
|
389
|
+
writeFileSync(join(journalDir, path), tail);
|
|
390
|
+
return { path, output: captured };
|
|
391
|
+
}
|
|
342
392
|
/** Test seam — shrink the quota-banner silence gate without minute-long sleeps. */
|
|
343
393
|
export function setQuotaBannerSilentMsForTests(ms) {
|
|
344
394
|
quotaBannerSilentMs = ms;
|
|
@@ -642,7 +692,11 @@ export function countLiveSuites(snapshot, repoRoot, daemonPid = process.pid, cwd
|
|
|
642
692
|
}
|
|
643
693
|
}
|
|
644
694
|
const root = realpathSync(repoRoot);
|
|
645
|
-
|
|
695
|
+
// OBS-889 (run 3372): a bare-codex worker carries its whole prompt in argv and the prompt names the
|
|
696
|
+
// runner ("…as a vitest test whose…"), so a finished interactive worker counted as a live suite and
|
|
697
|
+
// held a task's gates for 9 min 46 s. A runner is named in a command's HEAD — `node <bin>`,
|
|
698
|
+
// `npm test`, `npx vitest`, `sh -c npm test` — never 140 KB into it: read the first four tokens only.
|
|
699
|
+
const candidates = rows.filter((row) => !ancestors.has(row.pid) && SUITE_COMMAND_RE.test(row.command.split(/\s+/, 4).join(" ")));
|
|
646
700
|
const attributable = new Set(candidates.filter((row) => {
|
|
647
701
|
if (descendants.has(row.pid))
|
|
648
702
|
return true;
|
|
@@ -1219,6 +1273,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1219
1273
|
const trackedDriver = {
|
|
1220
1274
|
id: driver.id,
|
|
1221
1275
|
interactive: driver.interactive,
|
|
1276
|
+
...(driver.readSource ? { readSource: driver.readSource } : {}),
|
|
1277
|
+
...(driver.describe ? { describe: driver.describe.bind(driver) } : {}),
|
|
1222
1278
|
slot: async (cwd, name, o) => { const s = await driver.slot(cwd, name, o); liveSlots.add(s); return s; },
|
|
1223
1279
|
run: (s, cmd) => driver.run(s, cmd),
|
|
1224
1280
|
waitOutput: (s, p, ms, o) => driver.waitOutput(s, p, ms, o),
|
|
@@ -1227,10 +1283,25 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1227
1283
|
read: (s, n) => driver.read(s, n),
|
|
1228
1284
|
...(driver.sendKey ? { sendKey: driver.sendKey.bind(driver) } : {}),
|
|
1229
1285
|
...(driver.nudge ? { nudge: driver.nudge.bind(driver) } : {}),
|
|
1286
|
+
...(driver.narrator ? { narrator: async (cwd, command, id) => {
|
|
1287
|
+
const s = await driver.narrator(cwd, command, id);
|
|
1288
|
+
liveSlots.add(s);
|
|
1289
|
+
return s;
|
|
1290
|
+
} } : {}),
|
|
1291
|
+
...(driver.project ? { project: driver.project.bind(driver) } : {}),
|
|
1292
|
+
...(driver.reconcile ? { reconcile: driver.reconcile.bind(driver) } : {}),
|
|
1230
1293
|
notify: (m, o) => driver.notify(m, o),
|
|
1231
1294
|
close: closeSlot,
|
|
1232
1295
|
worktree: (r, b, base) => driver.worktree(r, b, base),
|
|
1233
1296
|
};
|
|
1297
|
+
const absentCapabilities = new Set();
|
|
1298
|
+
const noteCapabilityAbsent = (capability) => {
|
|
1299
|
+
if (absentCapabilities.has(capability)
|
|
1300
|
+
|| journal.read().some((event) => event.event === "driver-capability-absent" && event.data.capability === capability))
|
|
1301
|
+
return;
|
|
1302
|
+
absentCapabilities.add(capability);
|
|
1303
|
+
journal.append("driver-capability-absent", undefined, { driver: driver.id, capability });
|
|
1304
|
+
};
|
|
1234
1305
|
// Termination (SIGINT/SIGTERM): record the daemon-controlled exit before closing every live slot,
|
|
1235
1306
|
// reconcile owned panes against an EMPTY
|
|
1236
1307
|
// desired set (herdr panes not in memory; panesToClose spares foreign names, watch panes, and
|
|
@@ -1255,6 +1326,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1255
1326
|
catch (error) {
|
|
1256
1327
|
console.error(`tickmarkr ${runId}: deliberate exit cause could not be journalled (${error instanceof Error ? error.message : String(error)})`);
|
|
1257
1328
|
}
|
|
1329
|
+
// Stop the scheduler before the first awaited retirement. Otherwise a freed slot can fund a
|
|
1330
|
+
// new attempt while this reaper is still closing the old ones.
|
|
1331
|
+
abortRun(new Error(`terminated by ${sig}`));
|
|
1258
1332
|
if (cfg.visibility.keepPanes !== "forever") {
|
|
1259
1333
|
for (const s of liveSlots) { // closeSlot only deletes the element being visited — safe during Set iteration
|
|
1260
1334
|
try {
|
|
@@ -1269,13 +1343,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1269
1343
|
}
|
|
1270
1344
|
releaseRunLock(repoRoot); // the process dies at exit() below — the finally never runs on this path
|
|
1271
1345
|
}
|
|
1272
|
-
abortRun(new Error(`terminated by ${sig}`));
|
|
1273
1346
|
exit(sig === "SIGINT" ? 130 : 143);
|
|
1274
1347
|
})();
|
|
1275
1348
|
};
|
|
1276
1349
|
process.on("SIGINT", onTermination);
|
|
1277
1350
|
process.on("SIGTERM", onTermination);
|
|
1278
1351
|
journal = opts.resume ? Journal.open(repoRoot, runId, opts.narrate) : Journal.create(repoRoot, runId, opts.narrate);
|
|
1352
|
+
const reviewHistory = journal.read()
|
|
1353
|
+
.filter((event) => event.event === "gate-result" && event.data.gate === "review" && typeof event.data.reviewer === "string")
|
|
1354
|
+
.map((event) => event.data.reviewer);
|
|
1279
1355
|
// Capture this before this observer appends run-resume. A prior run-end belongs to a completed
|
|
1280
1356
|
// lifecycle and must use the ordinary resume/redispatch rules; only an interrupted live attempt
|
|
1281
1357
|
// owns reusable measurements or unclean-death residue.
|
|
@@ -1305,6 +1381,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1305
1381
|
const approved = new Set(approvalStartupEvents.filter((e) => e.event === "task-approved" && e.taskId).map((e) => e.taskId));
|
|
1306
1382
|
let approvalSweepCursor = approvalStartupEvents.length;
|
|
1307
1383
|
const commands = detectGateCommands(repoRoot, cfg);
|
|
1384
|
+
const watchName = formatOwnedName({ role: "watch", taskId: "run", attempt: 0, runId });
|
|
1385
|
+
let watchSlot;
|
|
1386
|
+
const openBoard = async () => {
|
|
1387
|
+
if (!trackedDriver.narrator) {
|
|
1388
|
+
noteCapabilityAbsent("narrator");
|
|
1389
|
+
return;
|
|
1390
|
+
}
|
|
1391
|
+
try {
|
|
1392
|
+
watchSlot = await trackedDriver.narrator(repoRoot, watchCommand(runId), runId);
|
|
1393
|
+
}
|
|
1394
|
+
catch (error) {
|
|
1395
|
+
journal.append("watch-placement-failed", undefined, { error: error instanceof Error ? error.message : String(error) });
|
|
1396
|
+
}
|
|
1397
|
+
};
|
|
1308
1398
|
let baseRef;
|
|
1309
1399
|
let baseline;
|
|
1310
1400
|
// Phase 46 (RES-01/RES-02): the resume-state map is built ONCE here so execTask closes over it.
|
|
@@ -1372,9 +1462,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1372
1462
|
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
1373
1463
|
...(opts.retryFailed ? { retryFailed: true } : {}),
|
|
1374
1464
|
});
|
|
1465
|
+
await openBoard();
|
|
1375
1466
|
}
|
|
1376
1467
|
else {
|
|
1377
1468
|
baseRef = await gitHead(repoRoot);
|
|
1469
|
+
journal.append("baseline-start", undefined, { baseRef, commands });
|
|
1470
|
+
await openBoard();
|
|
1378
1471
|
baseline = await captureBaseline(repoRoot, commands);
|
|
1379
1472
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(baseline, null, 2));
|
|
1380
1473
|
writeFileSync(join(journal.dir, "graph.json"), readFileSync(join(tickmarkrDir(repoRoot), "graph.json")));
|
|
@@ -1420,21 +1513,6 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1420
1513
|
const collateral = pinnedCollateral ?? collateralHits(graph.tasks, repoRoot);
|
|
1421
1514
|
if (!pinnedCollateral)
|
|
1422
1515
|
writeFileSync(collateralPath, JSON.stringify(Object.fromEntries(collateral), null, 2));
|
|
1423
|
-
// T6: open the narrator AFTER run-start/run-resume is journaled so the watch surface has a run to
|
|
1424
|
-
// show. driver.narrator is undefined on subprocess → no-op (subprocess spawns nothing). Swallowed:
|
|
1425
|
-
// a failed-to-open or later-dead watch pane never affects the run.
|
|
1426
|
-
// OBS-103: hold the returned slot — narrator() returns the run's board under its canonical owned
|
|
1427
|
-
// name whichever way it got there (the herdr driver retires a survivor it finds and re-splits, so
|
|
1428
|
-
// the live command is run-bound), and the run-end sweep below retires it by that name regardless
|
|
1429
|
-
// of which daemon instance split the pane.
|
|
1430
|
-
const watchName = formatOwnedName({ role: "watch", taskId: "run", attempt: 0, runId });
|
|
1431
|
-
let watchSlot;
|
|
1432
|
-
try {
|
|
1433
|
-
watchSlot = await driver.narrator?.(repoRoot, watchCommand(runId), runId);
|
|
1434
|
-
}
|
|
1435
|
-
catch (error) {
|
|
1436
|
-
journal.append("watch-placement-failed", undefined, { error: error instanceof Error ? error.message : String(error) });
|
|
1437
|
-
}
|
|
1438
1516
|
const intWt = await ensureIntegration(repoRoot, branch, baseRef);
|
|
1439
1517
|
// v1.1 visibility: role-named slots; panes persist per keepPanes (attempt = v1 close-after-harvest)
|
|
1440
1518
|
const keepOpen = cfg.visibility.keepPanes !== "attempt";
|
|
@@ -1473,7 +1551,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1473
1551
|
if (watchSlot && watchSlot.name === watchName && !desired.has(watchName)) {
|
|
1474
1552
|
const w = watchSlot;
|
|
1475
1553
|
watchSlot = undefined;
|
|
1476
|
-
await
|
|
1554
|
+
await trackedDriver.close(w);
|
|
1477
1555
|
}
|
|
1478
1556
|
}
|
|
1479
1557
|
catch {
|
|
@@ -1525,10 +1603,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1525
1603
|
await previous;
|
|
1526
1604
|
try {
|
|
1527
1605
|
let lastCount = -1;
|
|
1606
|
+
const startedAt = Date.now();
|
|
1528
1607
|
for (;;) {
|
|
1529
1608
|
const count = await liveSuiteCount(repoRoot);
|
|
1530
1609
|
if (count === 0)
|
|
1531
1610
|
break;
|
|
1611
|
+
// OBS-889: a census that never reaches zero held T5's gates with no row and no end. Proceed at
|
|
1612
|
+
// the ceiling, flagged: the verdict that follows was produced beside whatever is still counted.
|
|
1613
|
+
if (Date.now() - startedAt >= suiteWaitCeilingMs) {
|
|
1614
|
+
journal.append("suite-wait-ceiling", taskId, { count, waitedMs: Date.now() - startedAt });
|
|
1615
|
+
break;
|
|
1616
|
+
}
|
|
1532
1617
|
if (count !== lastCount)
|
|
1533
1618
|
journal.append("suite-wait", taskId, { count });
|
|
1534
1619
|
lastCount = count;
|
|
@@ -1799,6 +1884,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1799
1884
|
// `test` row whose scope no consumer could recover.
|
|
1800
1885
|
...(Array.isArray(g.meta?.selectedTests) ? { selectedTests: g.meta.selectedTests } : {}),
|
|
1801
1886
|
...(g.meta?.fullSuite === true ? { fullSuite: true } : {}),
|
|
1887
|
+
...(g.meta?.reapedGroup === true ? { reapedGroup: true } : {}),
|
|
1888
|
+
...(g.gate === "review" && typeof g.meta?.reviewer === "string" ? {
|
|
1889
|
+
reviewer: g.meta.reviewer,
|
|
1890
|
+
...(typeof g.meta.vendor === "string" ? { vendor: g.meta.vendor } : {}),
|
|
1891
|
+
...(typeof g.meta.provider === "string" ? { provider: g.meta.provider } : {}),
|
|
1892
|
+
...(typeof g.meta.rotationSeat === "number" ? { rotationSeat: g.meta.rotationSeat } : {}),
|
|
1893
|
+
...(typeof g.meta.timeoutMs === "number" ? { timeoutMs: g.meta.timeoutMs } : {}),
|
|
1894
|
+
} : {}),
|
|
1802
1895
|
// A finding's path is its own evidence path. Do not pass task scope here: a declaration says
|
|
1803
1896
|
// where work is allowed, not where this verdict found the defect.
|
|
1804
1897
|
...(blocking ? {
|
|
@@ -1820,9 +1913,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1820
1913
|
// T7: the capacity the gate's own command child ran under, lifted verbatim from the result
|
|
1821
1914
|
// the battery produced — read where the shell built that child's environment, never
|
|
1822
1915
|
// re-derived from the run's own budget, which would answer a different number than the
|
|
1823
|
-
// operator's export did. It is this row's only COMPARABLE identity: the
|
|
1824
|
-
//
|
|
1825
|
-
//
|
|
1916
|
+
// operator's export did. It is this row's only COMPARABLE identity: the load samples describe
|
|
1917
|
+
// contention while capacity describes how the command divided the machine, and matching
|
|
1918
|
+
// capacity never claims the machine was calm. A gate that ran
|
|
1826
1919
|
// no command carries nothing — it divided nothing.
|
|
1827
1920
|
//
|
|
1828
1921
|
// `dirtiedBy` is the one hole in that lift: run-gates REPLACES a green battery verdict with
|
|
@@ -2167,6 +2260,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2167
2260
|
// deterministic-fingerprint occurrence/review round for budget accounting.
|
|
2168
2261
|
...(!satisfiedGate && !recheck ? { replayMeasurement: true } : {}),
|
|
2169
2262
|
};
|
|
2263
|
+
await trackedDriver.project?.(t.id, "in-review");
|
|
2170
2264
|
journal.phaseStart(t.id, "gates");
|
|
2171
2265
|
const { results } = await withSuiteWindow(t.id, resumedTask.gates.includes("test") && commands.test !== undefined, () => runGates(resumedTask, {
|
|
2172
2266
|
worktree: wt, baseRef: taskBase, result: priorResult, author: gateAuthor,
|
|
@@ -2184,6 +2278,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2184
2278
|
}
|
|
2185
2279
|
: undefined,
|
|
2186
2280
|
excludeReviewers: badReviewers,
|
|
2281
|
+
reviewHistory,
|
|
2187
2282
|
onGate: async (e) => {
|
|
2188
2283
|
if (e.phase === "start") {
|
|
2189
2284
|
notePhaseStart(e);
|
|
@@ -2260,6 +2355,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2260
2355
|
attempts: rs?.attempts ?? 0, assignment: gateAuthor, taskContentDigest: contentDigest,
|
|
2261
2356
|
});
|
|
2262
2357
|
journal.append("merge", t.id, { branch: taskBranch, commit: await integrationHead(intWt) });
|
|
2358
|
+
await trackedDriver.project?.(t.id, "completed");
|
|
2263
2359
|
journal.telemetry({
|
|
2264
2360
|
taskId: t.id, shape: t.shape, adapter: gateAuthor.adapter, model: gateAuthor.model,
|
|
2265
2361
|
channel: gateAuthor.channel, attempts: rs?.attempts ?? 0, outcome: "done",
|
|
@@ -2275,6 +2371,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2275
2371
|
// starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
|
|
2276
2372
|
let providerDeathRequeues = 0;
|
|
2277
2373
|
let providerDeathAttempt = -1;
|
|
2374
|
+
// A completed attempt may keep its pane for gate-failure context, but once another worker is
|
|
2375
|
+
// funded that context is superseded and — more importantly — must not remain a second writer on
|
|
2376
|
+
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
2377
|
+
let supersededWorkerSlot;
|
|
2278
2378
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
2279
2379
|
if (attempt !== providerDeathAttempt) {
|
|
2280
2380
|
providerDeathRequeues = 0;
|
|
@@ -2289,6 +2389,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2289
2389
|
await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2290
2390
|
return;
|
|
2291
2391
|
}
|
|
2392
|
+
if (supersededWorkerSlot && !keepForever) {
|
|
2393
|
+
const prior = supersededWorkerSlot;
|
|
2394
|
+
supersededWorkerSlot = undefined;
|
|
2395
|
+
const idx = keptSlots.indexOf(prior);
|
|
2396
|
+
if (idx >= 0)
|
|
2397
|
+
keptSlots.splice(idx, 1);
|
|
2398
|
+
try {
|
|
2399
|
+
await closeSlot(prior);
|
|
2400
|
+
}
|
|
2401
|
+
catch (error) {
|
|
2402
|
+
await park(t, `superseded worker could not be swept before dispatch: ${error instanceof Error ? error.message : String(error)}`, "stall", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2403
|
+
return;
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2292
2406
|
// OBS-419: read the approval-carried ceiling at the funding boundary, after every escalation
|
|
2293
2407
|
// decision but before the sole task-dispatch below. Both interactive and headless worker-launch
|
|
2294
2408
|
// branches are dominated by this check, so no route can fund another worker without consulting it.
|
|
@@ -2430,11 +2544,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2430
2544
|
lastContextTokens = undefined;
|
|
2431
2545
|
graph = setStatus(graph, t.id, "running");
|
|
2432
2546
|
saveGraph(repoRoot, graph);
|
|
2547
|
+
await trackedDriver.project?.(t.id, "in-progress");
|
|
2433
2548
|
// T6: a dispatch that carries an outstanding finding says so, and names it. Without this the
|
|
2434
2549
|
// ledger cannot tell a carried dispatch from an amnesiac one — the exact question a run that
|
|
2435
2550
|
// spends two frontier attempts re-deriving a known defect has to be able to answer afterwards.
|
|
2436
2551
|
journal.append("task-dispatch", t.id, {
|
|
2437
2552
|
assignment, attempt, provenance: dispatchProvenance(r.provenance), retryMode,
|
|
2553
|
+
excludedChannels: [...demotedChannels].sort(),
|
|
2438
2554
|
...(outstandingFindings.length > 0 ? { carriedFindings: outstandingFindings } : {}),
|
|
2439
2555
|
...(carriedConsultGuidance ? { carriedConsultGuidance } : {}),
|
|
2440
2556
|
});
|
|
@@ -2546,6 +2662,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2546
2662
|
// itself before gates regardless of what the worker does with it.
|
|
2547
2663
|
writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${workerContract}\n\n${readFileSync(promptFile, "utf8")}`);
|
|
2548
2664
|
const adapter = getAdapter(assignment.adapter, adapters);
|
|
2665
|
+
if (adapter.trustDialog && !trackedDriver.sendKey)
|
|
2666
|
+
noteCapabilityAbsent("sendKey");
|
|
2549
2667
|
// VIS-04: workers share one role tab. T2: `owned` names the pane canonically (ownership contract);
|
|
2550
2668
|
// the legacy name stays the fallback for drivers without owned handling (subprocess spies).
|
|
2551
2669
|
const workerSlotOpts = {
|
|
@@ -2599,7 +2717,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2599
2717
|
// between task-dispatch and this line — worktree recreation, setup, prompt write, slot allocation,
|
|
2600
2718
|
// the launch itself — can still die with no worker having seen the brief, and `--retry-failed`
|
|
2601
2719
|
// must then re-send that same brief rather than a fresh prompt on a possibly banned channel.
|
|
2602
|
-
|
|
2720
|
+
let workerLaunchedAt;
|
|
2721
|
+
const noteLaunched = async () => {
|
|
2722
|
+
// Startup evidence begins when the worker exists, not while prompt preparation, slot
|
|
2723
|
+
// allocation, or interactive readiness is still consuming the attempt wall clock.
|
|
2724
|
+
workerLaunchedAt ??= Date.now();
|
|
2725
|
+
journal.append("worker-launch", t.id, {
|
|
2726
|
+
attempt,
|
|
2727
|
+
retryMode,
|
|
2728
|
+
...(trackedDriver.describe ? await trackedDriver.describe(slot) : {}),
|
|
2729
|
+
});
|
|
2730
|
+
};
|
|
2731
|
+
const noteWorkerLiveness = (event, data) => journal.append(event, t.id, { ...data, source: trackedDriver.readSource ?? "driver.read" });
|
|
2603
2732
|
let cpuFlat;
|
|
2604
2733
|
let cpuAccountant;
|
|
2605
2734
|
let cpuGapCount = 0;
|
|
@@ -2731,6 +2860,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2731
2860
|
// subprocess tree that REACHED its exit marker, not about what the worker claimed.
|
|
2732
2861
|
let processExited = false;
|
|
2733
2862
|
let earlyLaunchDead = false;
|
|
2863
|
+
let startupFailure = false;
|
|
2734
2864
|
let deadWorkerPark;
|
|
2735
2865
|
let settleParsed;
|
|
2736
2866
|
let seedResult;
|
|
@@ -2813,7 +2943,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2813
2943
|
continue attempts;
|
|
2814
2944
|
return;
|
|
2815
2945
|
}
|
|
2816
|
-
noteLaunched();
|
|
2946
|
+
await noteLaunched();
|
|
2817
2947
|
output = seedResult.output;
|
|
2818
2948
|
}
|
|
2819
2949
|
else {
|
|
@@ -2827,14 +2957,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2827
2957
|
continue attempts;
|
|
2828
2958
|
return;
|
|
2829
2959
|
}
|
|
2830
|
-
noteLaunched();
|
|
2960
|
+
await noteLaunched();
|
|
2831
2961
|
output = await driver.read(slot, PANE_READ_ROWS);
|
|
2832
2962
|
}
|
|
2833
2963
|
// The returning paths report the same fact on the result; both callbacks land on the one
|
|
2834
2964
|
// latch, and the second is a no-op. A seed that answered is never re-answered by the loop.
|
|
2835
2965
|
if (seedResult?.trustAnswered)
|
|
2836
2966
|
noteSeedTrustAnswered();
|
|
2837
|
-
|
|
2967
|
+
startupFailure = startupFailureInWindow(output, workerLaunchedAt);
|
|
2968
|
+
if (seedResult?.seedFailed || startupFailure) {
|
|
2838
2969
|
finished = false;
|
|
2839
2970
|
}
|
|
2840
2971
|
else {
|
|
@@ -2877,6 +3008,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2877
3008
|
const sliceStart = Date.now();
|
|
2878
3009
|
const remaining = stallWindowMs - (sliceStart - lastProgressAt);
|
|
2879
3010
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3011
|
+
const startupLeft = workerLaunchedAt === undefined
|
|
3012
|
+
? 0
|
|
3013
|
+
: workerStartupWindowMs - (sliceStart - workerLaunchedAt);
|
|
3014
|
+
if (startupLeft > 0)
|
|
3015
|
+
slice = Math.min(slice, Math.max(1, startupLeft));
|
|
2880
3016
|
if (!everHadOutput) {
|
|
2881
3017
|
const earlyLeft = earlyLaunchLivenessMs - (sliceStart - attemptStart);
|
|
2882
3018
|
if (earlyLeft > 0)
|
|
@@ -2896,6 +3032,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2896
3032
|
const exit = exitRe.exec(output);
|
|
2897
3033
|
if (finished || exit) {
|
|
2898
3034
|
exitCode = exit ? Number(exit[1]) : null; // null ⇔ the TUI is still alive
|
|
3035
|
+
processExited = exit !== null;
|
|
2899
3036
|
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
2900
3037
|
break;
|
|
2901
3038
|
}
|
|
@@ -2922,7 +3059,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2922
3059
|
throw error;
|
|
2923
3060
|
if (!paneReadHeld) {
|
|
2924
3061
|
paneReadHeld = true;
|
|
2925
|
-
|
|
3062
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
2926
3063
|
slot: slot.name, attempt, reason: "pane-read-unreadable",
|
|
2927
3064
|
error: error instanceof Error ? error.message : String(error),
|
|
2928
3065
|
});
|
|
@@ -2935,6 +3072,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2935
3072
|
}
|
|
2936
3073
|
if (paneText.length > 0)
|
|
2937
3074
|
everHadOutput = true;
|
|
3075
|
+
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
3076
|
+
startupFailure = true;
|
|
3077
|
+
output = paneText;
|
|
3078
|
+
break;
|
|
3079
|
+
}
|
|
2938
3080
|
// OBS-117 (v1.71 T6): zero raw output by the early-launch deadline is a dead channel now.
|
|
2939
3081
|
if (!everHadOutput && Date.now() - attemptStart >= earlyLaunchLivenessMs) {
|
|
2940
3082
|
earlyLaunchDead = true;
|
|
@@ -3008,7 +3150,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3008
3150
|
catch (error) {
|
|
3009
3151
|
if (!paneStatusHeld) {
|
|
3010
3152
|
paneStatusHeld = true;
|
|
3011
|
-
|
|
3153
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
3012
3154
|
slot: slot.name, attempt, reason: "pane-status-unreadable",
|
|
3013
3155
|
error: error instanceof Error ? error.message : String(error),
|
|
3014
3156
|
});
|
|
@@ -3085,7 +3227,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3085
3227
|
paneAbsent = false;
|
|
3086
3228
|
if (!paneReadHeld) {
|
|
3087
3229
|
paneReadHeld = true;
|
|
3088
|
-
|
|
3230
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
3089
3231
|
slot: slot.name, attempt, reason: "pane-read-unreadable",
|
|
3090
3232
|
error: error instanceof Error ? error.message : String(error),
|
|
3091
3233
|
});
|
|
@@ -3109,7 +3251,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3109
3251
|
paneAbsent = false;
|
|
3110
3252
|
if (!paneReadHeld) {
|
|
3111
3253
|
paneReadHeld = true;
|
|
3112
|
-
|
|
3254
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
3113
3255
|
slot: slot.name, attempt, reason: "pane-read-unreadable",
|
|
3114
3256
|
error: error instanceof Error ? error.message : String(error),
|
|
3115
3257
|
});
|
|
@@ -3133,7 +3275,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3133
3275
|
if (deathCertain) {
|
|
3134
3276
|
const preservation = await preserveDeadWorker(wt, taskBase);
|
|
3135
3277
|
if (!preservation.ref) {
|
|
3136
|
-
|
|
3278
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
3137
3279
|
slot: slot.name, attempt, reason: `worktree-${preservation.state}`,
|
|
3138
3280
|
});
|
|
3139
3281
|
continue;
|
|
@@ -3142,7 +3284,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3142
3284
|
const reason = `worker is unambiguously dead: pane absent, process tree empty, and worktree unchanged; preserved at ${ref}`;
|
|
3143
3285
|
deadWorkerPark = { ref, reason };
|
|
3144
3286
|
journal.append("worktree-preserved", t.id, { ref });
|
|
3145
|
-
|
|
3287
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
3146
3288
|
slot: slot.name, attempt, reason: "unambiguous-worker-death", ref,
|
|
3147
3289
|
});
|
|
3148
3290
|
break;
|
|
@@ -3158,8 +3300,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3158
3300
|
// Everything the fast-kill can decide from the CHANNEL, evaluated before the CPU leg is
|
|
3159
3301
|
// asked for anything: the kill's own probe costs a `ps` every 100ms, so an ineligible
|
|
3160
3302
|
// slice must not pay for it. The CPU reading is consulted at the kill itself, below.
|
|
3161
|
-
const fastKillEligible = !
|
|
3162
|
-
&& !nudgePending && !nudgeFailed
|
|
3303
|
+
const fastKillEligible = !nudgePending && !nudgeFailed
|
|
3163
3304
|
&& sliceNow - lastOutputGrowthAt >= deadChannelFastKillMs
|
|
3164
3305
|
&& worktreeSinceLaunch === "unchanged";
|
|
3165
3306
|
const harvestCpuEligible = !nudgePending && !nudgeFailed
|
|
@@ -3200,14 +3341,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3200
3341
|
// (an elapsed "9s"→"10s" lengthens the read and would hide a frozen pane). But the
|
|
3201
3342
|
// tracker SHARES the read window's ceiling: its row signal saturates once a sample
|
|
3202
3343
|
// FILLS a PANE_READ_ROWS read on raw lines (blanks/chrome included — see stall.ts),
|
|
3203
|
-
// and past that point a flat tracker means "unmeasurable", not "dead".
|
|
3204
|
-
//
|
|
3205
|
-
//
|
|
3206
|
-
//
|
|
3207
|
-
// worker past the ceiling would be concluded dead mid-work. So the kill STANDS DOWN on
|
|
3208
|
-
// a saturated row signal (journaled once per attempt); the rolling window still owns
|
|
3209
|
-
// that pane, exactly as pre-T1. Token growth counts as life either way, so a metered
|
|
3210
|
-
// worker thinking through a long tool run survives.
|
|
3344
|
+
// and past that point a flat tracker means "unmeasurable", not "dead". Journal that
|
|
3345
|
+
// unreadability once, then let the independent CPU and worktree legs decide: accruing or
|
|
3346
|
+
// unreadable CPU preserves the worker; flat CPU beside an unchanged tree may conclude.
|
|
3347
|
+
// A saturated row alone is never a death verdict.
|
|
3211
3348
|
// The triad has NO status exemption: a pane that herdr reports as blocked, idle, working
|
|
3212
3349
|
// or unknown dies alike once it holds no trailer, an unchanged tree and no growth — waiting the
|
|
3213
3350
|
// rolling window out on a status reading is exactly the blindness T1 removes. A matched
|
|
@@ -3221,7 +3358,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3221
3358
|
// filesystem/git observation answer that clause.
|
|
3222
3359
|
if (stallProgress.rowSignalSaturated && !rowSaturationHeld) {
|
|
3223
3360
|
rowSaturationHeld = true;
|
|
3224
|
-
journal.append("
|
|
3361
|
+
journal.append("contact-unreadable", t.id, {
|
|
3362
|
+
slot: slot.name, attempt, source: "pane-rows", reason: "row-signal-saturated", concludes: false,
|
|
3363
|
+
});
|
|
3225
3364
|
}
|
|
3226
3365
|
// OBS-548: the FOURTH leg, and the one the daemon already measured. The three legs above
|
|
3227
3366
|
// are all channel-side: they say a pane stopped talking. The tree's CPU says whether
|
|
@@ -3238,12 +3377,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3238
3377
|
journal.append("worker-dead", t.id, {
|
|
3239
3378
|
slot: slot.name, attempt, silentMs: sliceNow - lastOutputGrowthAt,
|
|
3240
3379
|
cpuMs: leg.cpu.ms, cpuResolutionMs: leg.cpu.resolutionMs,
|
|
3380
|
+
source: trackedDriver.readSource ?? "driver.read",
|
|
3241
3381
|
});
|
|
3242
3382
|
break;
|
|
3243
3383
|
}
|
|
3244
3384
|
if (!cpuHeld) {
|
|
3245
3385
|
cpuHeld = true;
|
|
3246
|
-
|
|
3386
|
+
noteWorkerLiveness("worker-dead-held", {
|
|
3247
3387
|
slot: slot.name, attempt, reason: leg.state === "accruing" ? "cpu-accruing" : "cpu-unmeasurable",
|
|
3248
3388
|
});
|
|
3249
3389
|
}
|
|
@@ -3285,6 +3425,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3285
3425
|
const exit = exitRe.exec(output);
|
|
3286
3426
|
if (finished || exit) {
|
|
3287
3427
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3428
|
+
processExited = exit !== null;
|
|
3288
3429
|
await sampleContext();
|
|
3289
3430
|
break;
|
|
3290
3431
|
}
|
|
@@ -3330,6 +3471,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3330
3471
|
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
3331
3472
|
const exit = exitRe.exec(output);
|
|
3332
3473
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3474
|
+
processExited = exit !== null;
|
|
3333
3475
|
}
|
|
3334
3476
|
if (finished) {
|
|
3335
3477
|
await driver.waitAgentStatus(slot, "idle", 5_000); // settle, then re-harvest the final render
|
|
@@ -3371,11 +3513,12 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3371
3513
|
continue attempts;
|
|
3372
3514
|
return;
|
|
3373
3515
|
}
|
|
3374
|
-
noteLaunched();
|
|
3516
|
+
await noteLaunched();
|
|
3375
3517
|
// OBS-54: headless workers have the same output-inactivity budget as visible panes.
|
|
3376
3518
|
// v1.76: same monotonic-progress measure as the interactive site; harvest stays raw.
|
|
3377
3519
|
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
3378
3520
|
const initialPane = await driver.read(slot, 500);
|
|
3521
|
+
startupFailure = startupFailureInWindow(initialPane, workerLaunchedAt);
|
|
3379
3522
|
let everHadOutput = initialPane.length > 0;
|
|
3380
3523
|
const stallProgress = new StallProgressTracker();
|
|
3381
3524
|
stallProgress.observe({ paneText: initialPane, seedSubmitted: true, contextTokens });
|
|
@@ -3388,9 +3531,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3388
3531
|
// worker-result-harvested row. The interactive site has always kept them apart (`finished`
|
|
3389
3532
|
// there is the trailer regex; the exit marker only sets exitCode), and the cause taxonomy
|
|
3390
3533
|
// already names this shape "clean-exit-no-trailer" — unreachable in print mode until now.
|
|
3391
|
-
while (Date.now() - lastProgressAt < stallWindowMs) {
|
|
3534
|
+
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
3392
3535
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
3393
3536
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3537
|
+
const startupLeft = workerLaunchedAt === undefined
|
|
3538
|
+
? 0
|
|
3539
|
+
: workerStartupWindowMs - (Date.now() - workerLaunchedAt);
|
|
3540
|
+
if (startupLeft > 0)
|
|
3541
|
+
slice = Math.min(slice, Math.max(1, startupLeft));
|
|
3394
3542
|
if (!everHadOutput) {
|
|
3395
3543
|
const earlyLeft = earlyLaunchLivenessMs - (Date.now() - attemptStart);
|
|
3396
3544
|
if (earlyLeft > 0)
|
|
@@ -3405,6 +3553,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3405
3553
|
const paneText = await driver.read(slot, 500);
|
|
3406
3554
|
if (paneText.length > 0)
|
|
3407
3555
|
everHadOutput = true;
|
|
3556
|
+
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
3557
|
+
startupFailure = true;
|
|
3558
|
+
break;
|
|
3559
|
+
}
|
|
3408
3560
|
if (!everHadOutput && Date.now() - attemptStart >= earlyLaunchLivenessMs) {
|
|
3409
3561
|
earlyLaunchDead = true;
|
|
3410
3562
|
break;
|
|
@@ -3443,12 +3595,52 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3443
3595
|
if (interactive && finished && !exitRe.test(output)) {
|
|
3444
3596
|
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true });
|
|
3445
3597
|
}
|
|
3446
|
-
//
|
|
3447
|
-
//
|
|
3448
|
-
|
|
3598
|
+
// Free availability reroutes deliberately reuse the charged-attempt ordinal. Stream artifacts
|
|
3599
|
+
// cannot: every dispatch owns bytes that must remain independently recoverable, so derive the
|
|
3600
|
+
// append-only dispatch ordinal from the journal instead of overwriting an earlier a<n>.out.
|
|
3601
|
+
const streamAttempt = journal.read().filter((event) => event.event === "task-dispatch" && event.taskId === t.id).length - 1;
|
|
3602
|
+
const capturedStream = await captureWorkerStream(journal.dir, t.id, streamAttempt, driver, slot, output);
|
|
3603
|
+
// An interactive settle already selected the bounded parse it is allowed to use. Every other
|
|
3604
|
+
// path parses the common final stream snapshot — notably envelope adapters whose trailer is
|
|
3605
|
+
// encoded and therefore invisible to the raw wait regex. Keep `output` bounded: every text
|
|
3606
|
+
// classifier below retains its pre-artifact tail surface.
|
|
3607
|
+
let result = settleParsed ?? adapter.parse(capturedStream.output, nonce);
|
|
3608
|
+
const adapterCause = result.cause;
|
|
3609
|
+
// Law 46: there is one completion fact. A classified adapter result with no parse-failure cause
|
|
3610
|
+
// means its decoder found the nonce-bound trailer even when the raw JSON envelope escaped it.
|
|
3611
|
+
const workerFinished = finished || adapterCause === undefined;
|
|
3612
|
+
finished = workerFinished;
|
|
3613
|
+
const adapterStartupFailure = adapterCause === "startup-failure";
|
|
3614
|
+
const workerCause = adapterStartupFailure
|
|
3615
|
+
? "startup-failure"
|
|
3616
|
+
: startupFailure
|
|
3617
|
+
? "startup-failure"
|
|
3618
|
+
: classifyWorkerResultCause({ output, ok: result.ok, finished: workerFinished, exitCode, summary: result.summary, timedOut, deadChannel: deadChannelKilled });
|
|
3619
|
+
// OBS-897/889(4): neither a timeout/quiet-harvest nor a trailer is a verdict on a tree while
|
|
3620
|
+
// its writer remains alive. Reap through the same driver lifecycle every execution surface
|
|
3621
|
+
// already implements, then (and only then) permit commitsAheadOf/gates below to read the tree.
|
|
3622
|
+
// A failed reap is a stall disposition, never permission to gate a moving branch.
|
|
3623
|
+
let reapFailure;
|
|
3624
|
+
if (!processExited && !deadWorkerPark) {
|
|
3625
|
+
try {
|
|
3626
|
+
await closeSlot(slot);
|
|
3627
|
+
if (!workerFinished) {
|
|
3628
|
+
journal.append("worker-reaped-before-harvest", t.id, {
|
|
3629
|
+
slot: slot.name, attempt, cause: workerCause ?? "stall-timeout",
|
|
3630
|
+
});
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
catch (error) {
|
|
3634
|
+
reapFailure = error instanceof Error ? error.message : String(error);
|
|
3635
|
+
}
|
|
3636
|
+
}
|
|
3637
|
+
else if (keepOpen && (workerFinished || processExited || driver.id !== "subprocess")) {
|
|
3449
3638
|
keptSlots.push(slot);
|
|
3450
|
-
|
|
3639
|
+
supersededWorkerSlot = slot;
|
|
3640
|
+
}
|
|
3641
|
+
else {
|
|
3451
3642
|
await closeSlot(slot);
|
|
3643
|
+
}
|
|
3452
3644
|
// SPEND-01: usage from the harness's own cwd-keyed structured store, read POST-HOC from disk —
|
|
3453
3645
|
// `wt` is this task's private worktree, so the path is unique; the read is sliced to records
|
|
3454
3646
|
// stamped at/after this attempt's dispatch instant. Never the harvested pane text, never the
|
|
@@ -3464,13 +3656,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3464
3656
|
await park(t, deadWorkerPark.reason, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { ref: deadWorkerPark.ref });
|
|
3465
3657
|
return;
|
|
3466
3658
|
}
|
|
3467
|
-
let result = settleParsed ?? adapter.parse(output, nonce);
|
|
3468
|
-
const workerFinished = finished;
|
|
3469
|
-
const workerCause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut, deadChannel: deadChannelKilled });
|
|
3470
3659
|
journal.append("worker-result", t.id, {
|
|
3471
3660
|
ok: result.ok, summary: result.summary, deviations: result.deviations, finished: workerFinished, exitCode,
|
|
3472
|
-
mode: interactive ? "interactive" : "print",
|
|
3661
|
+
mode: interactive ? "interactive" : "print", stream: capturedStream.path,
|
|
3662
|
+
...(workerCause ? { cause: workerCause } : {}),
|
|
3473
3663
|
});
|
|
3664
|
+
if (reapFailure) {
|
|
3665
|
+
await park(t, `worker could not be reaped before harvest: ${reapFailure}`, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { reapFailure });
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3474
3668
|
// T2 review (routing precedence): provider-death, quota and dead-channel classification
|
|
3475
3669
|
// derive from ONE rule — the worker's OWN outcome, workerFinished and this PRE-HARVEST
|
|
3476
3670
|
// parse — never from the synthesized result below. A worker that committed and then walled
|
|
@@ -3557,6 +3751,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3557
3751
|
}
|
|
3558
3752
|
catch { /* cosmetic — reconcile is the backstop */ }
|
|
3559
3753
|
}
|
|
3754
|
+
if (supersededWorkerSlot === slot)
|
|
3755
|
+
supersededWorkerSlot = undefined;
|
|
3560
3756
|
}
|
|
3561
3757
|
// v1.8 TEL-05 — FROM-channel attribution for mid-task quota failover: `assignment` is still the
|
|
3562
3758
|
// throttled-away-FROM channel here (before the reassign below). durationMs:0 marks this as a
|
|
@@ -3583,7 +3779,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3583
3779
|
// T2 review: classify the PRE-HARVEST parse — classifyDeadChannel bails on any ok:true
|
|
3584
3780
|
// result, so reading the synthesized harvest result would swallow auth-required /
|
|
3585
3781
|
// setup-required / provider-outage for every committed-but-walled attempt (in both modes).
|
|
3586
|
-
const dead = classifyDeadChannel(preHarvestResult
|
|
3782
|
+
const dead = classifyDeadChannel({ ...preHarvestResult, raw: output })
|
|
3783
|
+
?? (earlyLaunchDead || startupFailure || adapterStartupFailure ? "setup-required" : undefined);
|
|
3587
3784
|
if (dead) {
|
|
3588
3785
|
const from = channelKey(assignment);
|
|
3589
3786
|
demotedChannels.add(from); // excluded for later attempts AND later tasks in this run
|
|
@@ -3602,12 +3799,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3602
3799
|
}
|
|
3603
3800
|
catch { /* cosmetic — reconcile is the backstop */ }
|
|
3604
3801
|
}
|
|
3802
|
+
if (supersededWorkerSlot === slot)
|
|
3803
|
+
supersededWorkerSlot = undefined;
|
|
3605
3804
|
}
|
|
3606
3805
|
assignment = next;
|
|
3607
3806
|
tried.push(channelKey(next));
|
|
3807
|
+
// Startup failures bought no worker turn: preserve the attempt ordinal/budget while the
|
|
3808
|
+
// verified-dead channel remains excluded for every later dispatch in this run.
|
|
3809
|
+
if (adapterStartupFailure || startupFailure)
|
|
3810
|
+
attempt--;
|
|
3608
3811
|
continue;
|
|
3609
3812
|
}
|
|
3610
|
-
|
|
3813
|
+
const chargedAttempts = adapterStartupFailure || startupFailure ? attempt : attempt + 1;
|
|
3814
|
+
await park(t, `dead channel (${dead}) and no eligible channel remains`, "reroute-exhausted", assignment, chargedAttempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3611
3815
|
return;
|
|
3612
3816
|
}
|
|
3613
3817
|
if (!finished) {
|
|
@@ -3690,6 +3894,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3690
3894
|
gateLoop: while (true) {
|
|
3691
3895
|
const gated = await gitHead(wt);
|
|
3692
3896
|
gateSubject = { commit: await gateCommitSubject(taskBase, gated, wt), attempt };
|
|
3897
|
+
await trackedDriver.project?.(t.id, "in-review");
|
|
3693
3898
|
journal.phaseStart(t.id, "gates");
|
|
3694
3899
|
({ results, commits } = await withSuiteWindow(t.id, t.gates.includes("test") && commands.test !== undefined, () => runGates(t, {
|
|
3695
3900
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
@@ -3713,6 +3918,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3713
3918
|
}
|
|
3714
3919
|
: undefined,
|
|
3715
3920
|
excludeReviewers: badReviewers,
|
|
3921
|
+
reviewHistory,
|
|
3716
3922
|
onGate,
|
|
3717
3923
|
})));
|
|
3718
3924
|
results.forEach(classifySignalOnlyTest);
|
|
@@ -3751,6 +3957,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3751
3957
|
attempts: attempt + 1, assignment, taskContentDigest: contentDigest,
|
|
3752
3958
|
});
|
|
3753
3959
|
journal.append("merge", t.id, { branch: taskBranch, commit: await integrationHead(intWt) });
|
|
3960
|
+
await trackedDriver.project?.(t.id, "completed");
|
|
3754
3961
|
// firstAttemptOk/gateFails/consults are recorded FACTS, not policy — a parkKind:"stall" row is
|
|
3755
3962
|
// recorded but NOT quality-negative in v1.6; Phase 12 owns reward policy, so flipping it later needs zero data migration.
|
|
3756
3963
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts: attempt + 1, outcome: "done", durationMs: Date.now() - startMs, firstAttemptOk: attempt === 0, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
|