tickmarkr 1.45.0 → 1.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/fake.js +2 -2
- package/dist/adapters/registry.js +12 -3
- package/dist/brand.js +2 -1
- package/dist/cli/commands/approve.js +6 -7
- package/dist/cli/commands/init.js +55 -3
- package/dist/cli/commands/profile.js +129 -7
- package/dist/compile/collateral.js +4 -2
- package/dist/drivers/herdr.d.ts +1 -0
- package/dist/drivers/herdr.js +12 -2
- package/dist/gates/scope.js +2 -2
- package/dist/plan/prompt.js +1 -1
- package/dist/route/profile.d.ts +22 -0
- package/dist/route/profile.js +42 -10
- package/dist/route/router.d.ts +2 -2
- package/dist/route/router.js +10 -2
- package/dist/run/consult.d.ts +5 -0
- package/dist/run/consult.js +35 -0
- package/dist/run/daemon.js +145 -25
- package/dist/run/git.d.ts +2 -0
- package/dist/run/git.js +20 -7
- package/dist/run/journal.d.ts +39 -2
- package/dist/run/journal.js +96 -4
- package/dist/run/merge.js +8 -8
- package/package.json +1 -1
package/dist/run/consult.js
CHANGED
|
@@ -19,6 +19,41 @@ export function renderRetryGuidance(v) {
|
|
|
19
19
|
lines.push(part);
|
|
20
20
|
return lines.slice(0, MAX_RETRY_GUIDANCE_LINES).map((l) => `- ${l}`).join("\n");
|
|
21
21
|
}
|
|
22
|
+
const LANDED_COMMIT_PREMISE_RE = /\b(?:already\s+committed|is\s+already\s+committed|landed\s+commit|implementation\s+is\s+already)\b/i;
|
|
23
|
+
const COMMIT_HASH_RE = /\b[0-9a-f]{7,40}\b/gi;
|
|
24
|
+
// OBS-58: name prior attempt commits by hash and drop consult premises about landed work that the
|
|
25
|
+
// fresh worktree does not satisfy — regenerated against post-recreation state, not inherited verbatim.
|
|
26
|
+
export function augmentRetryBrief(feedback, opts) {
|
|
27
|
+
const { attempted, carried, present } = opts;
|
|
28
|
+
const named = [...new Set([...attempted, ...carried])];
|
|
29
|
+
const presentLower = new Set([...present].map((h) => h.toLowerCase()));
|
|
30
|
+
const parts = [];
|
|
31
|
+
if (named.length > 0) {
|
|
32
|
+
parts.push("## Prior attempt commits (by hash)\n"
|
|
33
|
+
+ named.map((h) => `- ${h}${presentLower.has(h.toLowerCase()) ? " — present in this worktree" : " — not in this worktree"}`).join("\n"));
|
|
34
|
+
}
|
|
35
|
+
let body = feedback.trim();
|
|
36
|
+
if (body && LANDED_COMMIT_PREMISE_RE.test(body)) {
|
|
37
|
+
const falseHash = [...body.matchAll(COMMIT_HASH_RE)].some((m) => !presentLower.has(m[0].toLowerCase()));
|
|
38
|
+
const falseLanded = carried.length === 0 && attempted.length > 0;
|
|
39
|
+
if (falseHash || falseLanded) {
|
|
40
|
+
body = body
|
|
41
|
+
.split("\n")
|
|
42
|
+
.filter((line) => !LANDED_COMMIT_PREMISE_RE.test(line))
|
|
43
|
+
.join("\n")
|
|
44
|
+
.trim();
|
|
45
|
+
const correction = carried.length > 0
|
|
46
|
+
? `Prior attempt work is present at: ${carried.join(", ")}.`
|
|
47
|
+
: attempted.length > 0
|
|
48
|
+
? "Prior attempt commits could not be carried forward onto this worktree — re-land the work or continue from the integration tip."
|
|
49
|
+
: "No prior attempt commits are present in this worktree — implement from the integration tip.";
|
|
50
|
+
body = body ? `${correction}\n\n${body}` : correction;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (body)
|
|
54
|
+
parts.push(body);
|
|
55
|
+
return parts.join("\n\n");
|
|
56
|
+
}
|
|
22
57
|
const ACTIONS = ["retry", "reroute", "decompose", "human"];
|
|
23
58
|
export function buildDossierPrompt(d, nonce) {
|
|
24
59
|
return `TICKMARKR-CONSULT
|
package/dist/run/daemon.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { shq } from "../adapters/types.js";
|
|
3
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { join } from "node:path";
|
|
4
5
|
import { trailerPattern, writePrompt } from "../adapters/prompt.js";
|
|
5
6
|
import { allAdapters, discoverChannels, getAdapter, probeAll, readDoctor } from "../adapters/registry.js";
|
|
@@ -10,9 +11,9 @@ import { formatOwnedName } from "../drivers/types.js";
|
|
|
10
11
|
import { captureBaseline, detectGateCommands } from "../gates/baseline.js";
|
|
11
12
|
import { runGates } from "../gates/run-gates.js";
|
|
12
13
|
import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
|
|
13
|
-
import { consult, renderRetryGuidance } from "./consult.js";
|
|
14
|
-
import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, WORKTREE_LAYOUT_CONTRACT } from "./git.js";
|
|
15
|
-
import { engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
|
|
14
|
+
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
15
|
+
import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, shGit, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
16
|
+
import { classifyWorkerResultCause, engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
|
|
16
17
|
import { acquireRunLock, releaseRunLock } from "./lock.js";
|
|
17
18
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, verifyIntegrationTip } from "./merge.js";
|
|
18
19
|
import { nextChannel, route } from "../route/router.js";
|
|
@@ -27,6 +28,30 @@ export function formatSummary(s) {
|
|
|
27
28
|
}
|
|
28
29
|
const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
|
|
29
30
|
const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
|
|
31
|
+
const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
|
|
32
|
+
const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
|
|
33
|
+
const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
|
|
34
|
+
async function commitsAheadOf(base, wt) {
|
|
35
|
+
const head = await gitHead(wt);
|
|
36
|
+
if (head === base)
|
|
37
|
+
return [];
|
|
38
|
+
const r = await shGit(`git log --reverse --format=%H ${shq(base)}..${shq(head)}`, wt);
|
|
39
|
+
if (r.code !== 0)
|
|
40
|
+
return [];
|
|
41
|
+
return r.stdout.trim().split("\n").filter(Boolean);
|
|
42
|
+
}
|
|
43
|
+
async function cherryPickCommits(wt, commits) {
|
|
44
|
+
const carried = [];
|
|
45
|
+
for (const hash of commits) {
|
|
46
|
+
const r = await shGit(`git cherry-pick --no-gpg-sign ${shq(hash)}`, wt);
|
|
47
|
+
if (r.code !== 0) {
|
|
48
|
+
await shGit("git cherry-pick --abort", wt);
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
carried.push(hash);
|
|
52
|
+
}
|
|
53
|
+
return carried;
|
|
54
|
+
}
|
|
30
55
|
export async function runDaemon(repoRoot, opts = {}) {
|
|
31
56
|
const cfg = loadConfig(repoRoot, { globalDir: opts.globalDir });
|
|
32
57
|
const adapters = opts.adapters ?? allAdapters();
|
|
@@ -128,6 +153,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
128
153
|
const keepLlm = keepForever;
|
|
129
154
|
const keptSlots = [];
|
|
130
155
|
const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
|
|
156
|
+
// OBS-57: per-run in-run demotion — channels that burn consecutive no-trailer windows route around for later attempts.
|
|
157
|
+
const demotedChannels = new Set();
|
|
158
|
+
const noTrailerStreak = new Map();
|
|
131
159
|
// OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
|
|
132
160
|
// terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
|
|
133
161
|
// owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
|
|
@@ -163,14 +191,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
163
191
|
mergeChain = next.catch(() => undefined);
|
|
164
192
|
return next;
|
|
165
193
|
};
|
|
166
|
-
//
|
|
167
|
-
//
|
|
194
|
+
// gateFails/consults are execTask-scoped counters passed in so a park row is a rich verified-failure
|
|
195
|
+
// observation (e.g. ladder-exhausted + gateFails:4); every task-human row has a closed kind, never prose alone.
|
|
168
196
|
const park = async (t, reason, kind, assignment, attempts, startMs, gateFails = 0, consults = 0, tokens, metered = 0, retryMode = "fresh") => {
|
|
169
197
|
graph = setStatus(graph, t.id, "human");
|
|
170
198
|
saveGraph(repoRoot, graph);
|
|
171
|
-
journal.append("task-human", t.id, { reason });
|
|
199
|
+
journal.append("task-human", t.id, { reason, kind });
|
|
172
200
|
if (assignment) {
|
|
173
|
-
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind
|
|
201
|
+
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: assignment.adapter, model: assignment.model, channel: assignment.channel, attempts, outcome: "human", durationMs: Date.now() - startMs, parkKind: kind, gateFails, consults, tokens, meteredAttempts: tokens ? metered : undefined, retryMode });
|
|
174
202
|
}
|
|
175
203
|
await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
|
|
176
204
|
await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}`, { tier: "attention" });
|
|
@@ -183,10 +211,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
183
211
|
// term) would silently dispatch every unapproved gate that becomes ready during a resume — pinned
|
|
184
212
|
// by the resume-path guard pin in tests/run/daemon.test.ts (the only test that reaches this guard
|
|
185
213
|
// on the resume path; a park-then-resume task is filtered out by readyTasks() and never gets here).
|
|
186
|
-
await park(t, `humanGate: "${t.title}" requires approval before dispatch`,
|
|
214
|
+
await park(t, `humanGate: "${t.title}" requires approval before dispatch`, "human-gate", null, 0, startMs);
|
|
187
215
|
return;
|
|
188
216
|
}
|
|
189
|
-
const r = route(t, cfg, channels, profile);
|
|
217
|
+
const r = route(t, cfg, channels, profile, undefined, demotedChannels);
|
|
190
218
|
for (const lint of r.lints)
|
|
191
219
|
journal.append("routing-lint", t.id, { lint });
|
|
192
220
|
// VIS-02: journal a deviation from the static choice ONLY when one occurred (greppable absence = no deviation)
|
|
@@ -211,7 +239,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
211
239
|
// trailing-reroute edge (kill between verdict and dispatch), a stale fleet, OR a fresh-budget
|
|
212
240
|
// release (attempts 0 + non-empty tried): pick a failover over the replayed exclusions via the
|
|
213
241
|
// EXISTING nextChannel `tried` parameter — zero router changes (D-03).
|
|
214
|
-
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile);
|
|
242
|
+
const next = nextChannel(assignment, t, cfg, channels, rs.tried, profile, demotedChannels);
|
|
215
243
|
if (next)
|
|
216
244
|
assignment = next;
|
|
217
245
|
// ponytail: nextChannel null (every channel already tried / none available) — keep the static
|
|
@@ -245,9 +273,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
245
273
|
// ROUTE-13: learned within-band failover + deviation audit. nextChannel stays pure (route/ never
|
|
246
274
|
// journals); the daemon compares the learned pick against the static pick and owns the journal write.
|
|
247
275
|
const failover = (site) => {
|
|
248
|
-
const next = nextChannel(assignment, t, cfg, channels, tried, profile);
|
|
276
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
|
|
249
277
|
if (profile && next) {
|
|
250
|
-
const staticNext = nextChannel(assignment, t, cfg, channels, tried);
|
|
278
|
+
const staticNext = nextChannel(assignment, t, cfg, channels, tried, undefined, demotedChannels);
|
|
251
279
|
if (staticNext && channelKey(next) !== channelKey(staticNext)) {
|
|
252
280
|
journal.append("failover-deviation", t.id, { site, static: channelKey(staticNext), chosen: channelKey(next) });
|
|
253
281
|
}
|
|
@@ -311,7 +339,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
311
339
|
// the existing attempt-cap check below with zero new code. Fresh path: rs is undefined ⇒ 0.
|
|
312
340
|
// v1.24 OBS-18: after task-approved{release:attempt-cap}, replay zeros attempts so this loop
|
|
313
341
|
// starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
|
|
342
|
+
let providerDeathRequeues = 0;
|
|
343
|
+
let providerDeathAttempt = -1;
|
|
314
344
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
345
|
+
if (attempt !== providerDeathAttempt) {
|
|
346
|
+
providerDeathRequeues = 0;
|
|
347
|
+
providerDeathAttempt = attempt;
|
|
348
|
+
}
|
|
315
349
|
// v1.13 (VIS-09 safety): one FRESH nonce per attempt — see the run-scope comment above. A retained
|
|
316
350
|
// prior-attempt trailer (herdr scrollback / subprocess buffer) must never satisfy this attempt.
|
|
317
351
|
const nonce = randomBytes(4).toString("hex");
|
|
@@ -321,6 +355,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
321
355
|
await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
322
356
|
return;
|
|
323
357
|
}
|
|
358
|
+
// OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
|
|
359
|
+
if (demotedChannels.has(channelKey(assignment))) {
|
|
360
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
|
|
361
|
+
if (next) {
|
|
362
|
+
assignment = next;
|
|
363
|
+
const k = channelKey(next);
|
|
364
|
+
if (!tried.includes(k))
|
|
365
|
+
tried.push(k);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
324
368
|
// v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
|
|
325
369
|
// + adapter capability resumes; every other path is today's fresh dispatch.
|
|
326
370
|
const priorSession = retrySession;
|
|
@@ -347,7 +391,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
347
391
|
journal.append("task-dispatch", t.id, { assignment, attempt, provenance: r.provenance, retryMode });
|
|
348
392
|
const taskBase = await integrationHead(intWt); // deps are merged → visible to this task
|
|
349
393
|
const taskBranch = `${branch}--${t.id}`; // "--": a ref can't nest under the existing integration branch (locked decision 10)
|
|
394
|
+
const priorWt = worktreePath(repoRoot, taskBranch);
|
|
395
|
+
const commitsToCarry = existsSync(priorWt) ? await commitsAheadOf(taskBase, priorWt) : [];
|
|
350
396
|
const wt = await driver.worktree(repoRoot, taskBranch, taskBase);
|
|
397
|
+
// OBS-58: quota-failover and every retry recreate the task worktree from the integration tip —
|
|
398
|
+
// cherry-pick prior attempts' landed commits forward so a failover dispatch cannot silently
|
|
399
|
+
// orphan work a consult already verified as landed.
|
|
400
|
+
let carriedCommits = [];
|
|
401
|
+
if (commitsToCarry.length > 0) {
|
|
402
|
+
carriedCommits = await cherryPickCommits(wt, commitsToCarry);
|
|
403
|
+
journal.append("worktree-recreation", t.id, { attempted: commitsToCarry, carried: carriedCommits });
|
|
404
|
+
}
|
|
405
|
+
const priorNamed = [...new Set([...commitsToCarry, ...carriedCommits])];
|
|
406
|
+
const presentCommits = new Set(carriedCommits);
|
|
407
|
+
for (const h of commitsToCarry) {
|
|
408
|
+
if (!presentCommits.has(h) && (await shGit(`git merge-base --is-ancestor ${shq(h)} HEAD`, wt)).code === 0) {
|
|
409
|
+
presentCommits.add(h);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
if (feedback || priorNamed.length > 0) {
|
|
413
|
+
feedback = augmentRetryBrief(feedback, { attempted: commitsToCarry, carried: carriedCommits, present: presentCommits });
|
|
414
|
+
}
|
|
351
415
|
if (cfg.setup) {
|
|
352
416
|
// v1.22 T3: setup runs inside the task worktree — seal herdr control vars so a setup script
|
|
353
417
|
// cannot mutate the operator's panes. Worker/judge/review/consult are sealed at the driver
|
|
@@ -361,11 +425,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
361
425
|
}
|
|
362
426
|
}
|
|
363
427
|
const promptFile = writePrompt(journal.dir, t, attempt, feedback, nonce);
|
|
428
|
+
// OBS-56: state the non-interactive, one-pass finish contract and the OBS-54 stall budget in every
|
|
429
|
+
// worker prompt, not only consult retry guidance. Prepended so prompt.ts's completion trailer stays last.
|
|
430
|
+
const workerContract = `## Harness contract\n- This harness is non-interactive: make one continuous pass; do not stop for questions or follow-up input.\n- You have a ${taskTimeoutMinutes} minute stall window. Budget the full suite once, then commit and emit the completion trailer before it expires.`;
|
|
364
431
|
// OBS-47: state the worktree layout contract in the worker prompt (cheap-tier workers were
|
|
365
|
-
// committing/deleting node_modules and tripping the scope gate).
|
|
366
|
-
//
|
|
367
|
-
|
|
368
|
-
writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${readFileSync(promptFile, "utf8")}`);
|
|
432
|
+
// committing/deleting node_modules and tripping the scope gate). The harness re-asserts the link
|
|
433
|
+
// itself before gates regardless of what the worker does with it.
|
|
434
|
+
writeFileSync(promptFile, `${WORKTREE_LAYOUT_CONTRACT}\n\n${workerContract}\n\n${readFileSync(promptFile, "utf8")}`);
|
|
369
435
|
const adapter = getAdapter(assignment.adapter, adapters);
|
|
370
436
|
// VIS-04: workers share one role tab. T2: `owned` names the pane canonically (ownership contract);
|
|
371
437
|
// the legacy name stays the fallback for drivers without owned handling (subprocess spies).
|
|
@@ -419,6 +485,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
419
485
|
let finished;
|
|
420
486
|
let output;
|
|
421
487
|
let exitCode;
|
|
488
|
+
let timedOut = false;
|
|
422
489
|
if (interactive) {
|
|
423
490
|
// v1.2 interactive: the TUI doesn't exit on completion — the trailer is the finish line.
|
|
424
491
|
// The exit wrapper still fires if the TUI dies (crash/quit): fast-fail instead of burning the timeout.
|
|
@@ -429,11 +496,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
429
496
|
let trustAnswered = false;
|
|
430
497
|
finished = false;
|
|
431
498
|
exitCode = null;
|
|
432
|
-
output =
|
|
433
|
-
|
|
434
|
-
|
|
499
|
+
output = await driver.read(slot, 1000);
|
|
500
|
+
// OBS-54: reaping keys on new pane output, not dispatch wall clock. Poll at least twice per
|
|
501
|
+
// stall window (and at the existing 30s cadence for normal windows) so an active worker resets it.
|
|
502
|
+
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
503
|
+
let lastOutput = output;
|
|
504
|
+
let lastOutputAt = Date.now();
|
|
505
|
+
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
435
506
|
const sliceStart = Date.now();
|
|
436
|
-
const
|
|
507
|
+
const remaining = stallWindowMs - (sliceStart - lastOutputAt);
|
|
508
|
+
const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
437
509
|
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
438
510
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
439
511
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
@@ -447,6 +519,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
447
519
|
break;
|
|
448
520
|
}
|
|
449
521
|
}
|
|
522
|
+
const currentOutput = await driver.read(slot, 1000);
|
|
523
|
+
if (currentOutput !== lastOutput) {
|
|
524
|
+
lastOutput = currentOutput;
|
|
525
|
+
lastOutputAt = Date.now();
|
|
526
|
+
}
|
|
450
527
|
// v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
|
|
451
528
|
await sampleContext();
|
|
452
529
|
// page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
|
|
@@ -486,6 +563,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
486
563
|
}
|
|
487
564
|
if (!finished && exitCode === null) {
|
|
488
565
|
// timed out (or only ever saw false positives): harvest whatever the pane holds now
|
|
566
|
+
timedOut = Date.now() - lastOutputAt >= stallWindowMs;
|
|
489
567
|
output = await driver.read(slot, 1000);
|
|
490
568
|
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
491
569
|
const exit = exitRe.exec(output);
|
|
@@ -499,9 +577,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
499
577
|
else {
|
|
500
578
|
const inv = adapter.invoke(t, wt, assignment, { promptFile });
|
|
501
579
|
await driver.run(slot, `${inv.command}; ${exitMarkerCmd}`);
|
|
502
|
-
|
|
580
|
+
// OBS-54: headless workers have the same output-inactivity budget as visible panes.
|
|
581
|
+
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
582
|
+
let lastOutput = await driver.read(slot, 500);
|
|
583
|
+
let lastOutputAt = Date.now();
|
|
584
|
+
finished = false;
|
|
585
|
+
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
586
|
+
const remaining = stallWindowMs - (Date.now() - lastOutputAt);
|
|
587
|
+
const slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
588
|
+
if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
589
|
+
finished = true;
|
|
590
|
+
break;
|
|
591
|
+
}
|
|
592
|
+
const currentOutput = await driver.read(slot, 500);
|
|
593
|
+
if (currentOutput !== lastOutput) {
|
|
594
|
+
lastOutput = currentOutput;
|
|
595
|
+
lastOutputAt = Date.now();
|
|
596
|
+
}
|
|
597
|
+
}
|
|
503
598
|
output = await driver.read(slot, 500);
|
|
504
599
|
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
600
|
+
timedOut = !finished && Date.now() - lastOutputAt >= stallWindowMs;
|
|
505
601
|
}
|
|
506
602
|
// SPEND-01 interactive metering race: the harvest loop breaks on the trailer, but the worker
|
|
507
603
|
// shell may still be running post-trailer bookkeeping (session-store flush, fake usage stamp,
|
|
@@ -529,7 +625,31 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
529
625
|
metered++;
|
|
530
626
|
}
|
|
531
627
|
const result = adapter.parse(output, nonce);
|
|
532
|
-
|
|
628
|
+
const cause = classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut });
|
|
629
|
+
journal.append("worker-result", t.id, {
|
|
630
|
+
ok: result.ok, summary: result.summary, deviations: result.deviations, finished, exitCode,
|
|
631
|
+
mode: interactive ? "interactive" : "print", ...(cause ? { cause } : {}),
|
|
632
|
+
});
|
|
633
|
+
if (result.ok && finished)
|
|
634
|
+
noTrailerStreak.set(channelKey(assignment), 0);
|
|
635
|
+
else if (!finished && cause !== "provider-death") {
|
|
636
|
+
const ck = channelKey(assignment);
|
|
637
|
+
const streak = (noTrailerStreak.get(ck) ?? 0) + 1;
|
|
638
|
+
noTrailerStreak.set(ck, streak);
|
|
639
|
+
// OBS-57: two consecutive no-trailer windows in one run demote the channel for later attempts.
|
|
640
|
+
if (streak >= NO_TRAILER_DEMOTION_STREAK && !demotedChannels.has(ck)) {
|
|
641
|
+
demotedChannels.add(ck);
|
|
642
|
+
journal.append("channel-demotion", t.id, { channel: ck, streak });
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
// v1.46 T1: provider-outage requeue — same assignment, no attempt burn, no consult, capped.
|
|
646
|
+
if (cause === "provider-death" && providerDeathRequeues < PROVIDER_DEATH_REQUEUE_CAP) {
|
|
647
|
+
providerDeathRequeues++;
|
|
648
|
+
journal.append("provider-death-requeue", t.id, { attempt, requeue: providerDeathRequeues, assignment });
|
|
649
|
+
await new Promise((r) => setTimeout(r, PROVIDER_DEATH_BACKOFF_MS));
|
|
650
|
+
attempt--;
|
|
651
|
+
continue;
|
|
652
|
+
}
|
|
533
653
|
// quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
|
|
534
654
|
// print: guarded on exit code — exit-0 output that merely MENTIONS "rate limit" must not failover
|
|
535
655
|
// interactive: a harvested trailer beats quota mentions; without one, quota text fails over (spec v1.2 §2)
|
|
@@ -650,7 +770,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
650
770
|
journal.append("tip-moved", t.id, m.tipMoved);
|
|
651
771
|
if (tipMoves++ === 0)
|
|
652
772
|
continue gateLoop;
|
|
653
|
-
await park(t, "task branch tip moved twice after gating",
|
|
773
|
+
await park(t, "task branch tip moved twice after gating", "tip-moved", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
654
774
|
return;
|
|
655
775
|
}
|
|
656
776
|
if (!m.ok) {
|
|
@@ -795,7 +915,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
795
915
|
const tipFail = summary.tipVerify === "failed"
|
|
796
916
|
? ` — TIP VERIFY FAILED on ${summary.lastMergedTask ? `last merge ${summary.lastMergedTask}` : "integration tip"}`
|
|
797
917
|
: "";
|
|
798
|
-
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "
|
|
918
|
+
await driver.notify(`tickmarkr ${runId}: ${summary.done.length} done, ${summary.failed.length} failed, ${summary.human.length} awaiting human, ${summary.blocked.length} blocked, ${summary.pending.length} pending${attribution ? ` (${attribution})` : ""}${tipFail} — integration branch ${branch} (merge to main is yours)`, { tier: summary.tipVerify === "failed" ? "attention" : "routine" });
|
|
799
919
|
return summary;
|
|
800
920
|
}
|
|
801
921
|
finally {
|
package/dist/run/git.d.ts
CHANGED
|
@@ -5,7 +5,9 @@ export interface ShResult {
|
|
|
5
5
|
timedOut?: boolean;
|
|
6
6
|
}
|
|
7
7
|
export declare function sh(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
|
|
8
|
+
export declare function shGit(cmd: string, cwd: string, timeoutMs?: number): Promise<ShResult>;
|
|
8
9
|
export declare function shOk(cmd: string, cwd: string): Promise<string>;
|
|
10
|
+
export declare function shGitOk(cmd: string, cwd: string): Promise<string>;
|
|
9
11
|
export declare function gitHead(cwd: string): Promise<string>;
|
|
10
12
|
export declare const sanitizeBranch: (branch: string) => string;
|
|
11
13
|
export declare const WORKTREES_DIR = "worktrees.noindex";
|
package/dist/run/git.js
CHANGED
|
@@ -5,12 +5,12 @@ import { shq } from "../adapters/types.js";
|
|
|
5
5
|
import { tickmarkrDir } from "../graph/graph.js";
|
|
6
6
|
// stdin "ignore": same class as HARD-05 / SubprocessDriver — never leave an open pipe a child can block on
|
|
7
7
|
// (pi -p / codex exec wait for stdin EOF). timedOut distinguishes SIGKILL-timeout from a real nonzero exit.
|
|
8
|
-
|
|
8
|
+
function shell(cmd, cwd, timeoutMs, login) {
|
|
9
9
|
return new Promise((resolve) => {
|
|
10
10
|
// detached: bash gets its own process group so a timeout can kill the whole tree —
|
|
11
11
|
// SIGKILLing bash alone orphans grandchildren (codex/pi) that hold the stdio pipes
|
|
12
12
|
// open, so "close" never fires and the promise wedges forever (v1.33.1 init hang).
|
|
13
|
-
const p = spawn("bash", ["-lc", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
|
|
13
|
+
const p = spawn("bash", [login ? "-lc" : "-c", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"], detached: true });
|
|
14
14
|
let stdout = "", stderr = "";
|
|
15
15
|
let timedOut = false, done = false;
|
|
16
16
|
const finish = (code, err) => {
|
|
@@ -38,14 +38,27 @@ export function sh(cmd, cwd, timeoutMs = 600000) {
|
|
|
38
38
|
finish(code ?? 1); });
|
|
39
39
|
});
|
|
40
40
|
}
|
|
41
|
+
export function sh(cmd, cwd, timeoutMs = 600000) {
|
|
42
|
+
return shell(cmd, cwd, timeoutMs, true);
|
|
43
|
+
}
|
|
44
|
+
// Git plumbing never needs an operator profile; skip login-shell startup and its side effects.
|
|
45
|
+
export function shGit(cmd, cwd, timeoutMs = 600000) {
|
|
46
|
+
return shell(cmd, cwd, timeoutMs, false);
|
|
47
|
+
}
|
|
41
48
|
export async function shOk(cmd, cwd) {
|
|
42
49
|
const r = await sh(cmd, cwd);
|
|
43
50
|
if (r.code !== 0)
|
|
44
51
|
throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
|
|
45
52
|
return r.stdout;
|
|
46
53
|
}
|
|
54
|
+
export async function shGitOk(cmd, cwd) {
|
|
55
|
+
const r = await shGit(cmd, cwd);
|
|
56
|
+
if (r.code !== 0)
|
|
57
|
+
throw new Error(`command failed (${r.code}): ${cmd}\n${r.stderr || r.stdout}`);
|
|
58
|
+
return r.stdout;
|
|
59
|
+
}
|
|
47
60
|
export async function gitHead(cwd) {
|
|
48
|
-
return (await
|
|
61
|
+
return (await shGitOk("git rev-parse HEAD", cwd)).trim();
|
|
49
62
|
}
|
|
50
63
|
export const sanitizeBranch = (branch) => branch.replace(/[^\w.-]+/g, "-");
|
|
51
64
|
const sanitize = sanitizeBranch;
|
|
@@ -77,7 +90,7 @@ export async function createWorktree(repo, branch, baseRef) {
|
|
|
77
90
|
const dir = join(tickmarkrDir(repo), WORKTREES_DIR, sanitize(branch));
|
|
78
91
|
if (existsSync(dir))
|
|
79
92
|
await removeWorktree(repo, dir);
|
|
80
|
-
await
|
|
93
|
+
await shGitOk(`git worktree add -B ${shq(branch)} ${shq(dir)} ${shq(baseRef)}`, repo);
|
|
81
94
|
linkNodeModules(repo, dir);
|
|
82
95
|
return dir;
|
|
83
96
|
}
|
|
@@ -116,7 +129,7 @@ export function linkNodeModules(repo, dir, { force = false } = {}) {
|
|
|
116
129
|
export const WORKTREE_LAYOUT_CONTRACT = `## Worktree layout contract (harness-provisioned — do not modify)
|
|
117
130
|
- node_modules is a symlink into the main repo's node_modules, provisioned by tickmarkr. Never commit, delete, or replace it — the harness re-asserts this link before gates run, so modifying it cannot help and may fail your attempt.`;
|
|
118
131
|
export async function removeWorktree(repo, dir) {
|
|
119
|
-
await
|
|
120
|
-
await
|
|
121
|
-
await
|
|
132
|
+
await shGit(`git worktree remove --force ${shq(dir)}`, repo); // best-effort; stale dirs are re-added with -B
|
|
133
|
+
await shGit(`rm -rf ${shq(dir)}`, repo);
|
|
134
|
+
await shGit("git worktree prune", repo);
|
|
122
135
|
}
|
package/dist/run/journal.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import { type Assignment } from "../adapters/types.js";
|
|
3
3
|
import type { TickmarkrConfig } from "../config/config.js";
|
|
4
4
|
import { type TaskStatus } from "../graph/schema.js";
|
|
5
|
-
import { type RoutingProfile } from "../route/profile.js";
|
|
5
|
+
import { type ProfileDiscount, type RoutingProfile } from "../route/profile.js";
|
|
6
6
|
export interface JournalEvent {
|
|
7
7
|
ts: string;
|
|
8
8
|
event: string;
|
|
@@ -16,10 +16,21 @@ export interface ResumeState {
|
|
|
16
16
|
lastAssignment?: Assignment;
|
|
17
17
|
}
|
|
18
18
|
export declare const ATTEMPT_CAP_RELEASE: "attempt-cap";
|
|
19
|
-
export declare const PARK_KINDS: readonly ["ladder-exhausted", "attempt-cap", "gate-fail", "quota", "reroute-exhausted", "setup", "stall", "merge-conflict"];
|
|
19
|
+
export declare const PARK_KINDS: readonly ["human-gate", "ladder-exhausted", "attempt-cap", "gate-fail", "quota", "reroute-exhausted", "setup", "stall", "merge-conflict", "tip-moved"];
|
|
20
20
|
export type ParkKind = (typeof PARK_KINDS)[number];
|
|
21
21
|
export declare const RETRY_MODES: readonly ["resume", "fresh"];
|
|
22
22
|
export type RetryMode = (typeof RETRY_MODES)[number];
|
|
23
|
+
export declare const WORKER_RESULT_CAUSES: readonly ["provider-death", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer"];
|
|
24
|
+
export type WorkerResultCause = (typeof WORKER_RESULT_CAUSES)[number];
|
|
25
|
+
/** OBS-53: classify worker-result failures so retries and routing see the true signal, not one lumped bucket. */
|
|
26
|
+
export declare function classifyWorkerResultCause(opts: {
|
|
27
|
+
output: string;
|
|
28
|
+
ok: boolean;
|
|
29
|
+
finished: boolean;
|
|
30
|
+
exitCode: number | null;
|
|
31
|
+
summary: string;
|
|
32
|
+
timedOut: boolean;
|
|
33
|
+
}): WorkerResultCause | undefined;
|
|
23
34
|
export declare const TelemetryRowSchema: z.ZodObject<{
|
|
24
35
|
taskId: z.ZodString;
|
|
25
36
|
shape: z.ZodString;
|
|
@@ -42,9 +53,11 @@ export declare const TelemetryRowSchema: z.ZodObject<{
|
|
|
42
53
|
"attempt-cap": "attempt-cap";
|
|
43
54
|
"gate-fail": "gate-fail";
|
|
44
55
|
quota: "quota";
|
|
56
|
+
"human-gate": "human-gate";
|
|
45
57
|
"reroute-exhausted": "reroute-exhausted";
|
|
46
58
|
stall: "stall";
|
|
47
59
|
"merge-conflict": "merge-conflict";
|
|
60
|
+
"tip-moved": "tip-moved";
|
|
48
61
|
}>>;
|
|
49
62
|
tokens: z.ZodCatch<z.ZodOptional<z.ZodObject<{
|
|
50
63
|
input: z.ZodNumber;
|
|
@@ -60,8 +73,29 @@ export declare const TelemetryRowSchema: z.ZodObject<{
|
|
|
60
73
|
resume: "resume";
|
|
61
74
|
fresh: "fresh";
|
|
62
75
|
}>>;
|
|
76
|
+
signalQuality: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<0>, z.ZodLiteral<0.25>, z.ZodLiteral<0.5>, z.ZodLiteral<0.75>, z.ZodLiteral<1>]>>;
|
|
77
|
+
signalBasis: z.ZodOptional<z.ZodEnum<{
|
|
78
|
+
proved: "proved";
|
|
79
|
+
"review-agree": "review-agree";
|
|
80
|
+
"judge-only": "judge-only";
|
|
81
|
+
legacy: "legacy";
|
|
82
|
+
vacuous: "vacuous";
|
|
83
|
+
skipped: "skipped";
|
|
84
|
+
}>>;
|
|
63
85
|
}, z.core.$strip>;
|
|
64
86
|
export type TelemetryRow = z.infer<typeof TelemetryRowSchema>;
|
|
87
|
+
export declare const SIGNAL_BASIS: readonly ["proved", "review-agree", "judge-only", "legacy", "vacuous", "skipped"];
|
|
88
|
+
export type SignalBasis = (typeof SIGNAL_BASIS)[number];
|
|
89
|
+
export declare const SIGNAL_QUALITY: Record<SignalBasis, 0 | 0.25 | 0.5 | 0.75 | 1>;
|
|
90
|
+
export declare function signalQualityFromBasis(basis: SignalBasis): 0 | 0.25 | 0.5 | 0.75 | 1;
|
|
91
|
+
export declare function deriveSignalBasis(gate: string, pass: boolean, details: string, meta?: Record<string, unknown>): SignalBasis;
|
|
92
|
+
export declare function gateResultJournalData(gate: string, pass: boolean, details: string, meta?: Record<string, unknown>): {
|
|
93
|
+
gate: string;
|
|
94
|
+
pass: boolean;
|
|
95
|
+
details: string;
|
|
96
|
+
signalBasis: SignalBasis;
|
|
97
|
+
signalQuality: number;
|
|
98
|
+
} & Record<string, unknown>;
|
|
65
99
|
export declare function recordedGraphDefinitionHash(events: JournalEvent[]): string | undefined;
|
|
66
100
|
export type EngagementCompare = {
|
|
67
101
|
comparable: true;
|
|
@@ -84,6 +118,9 @@ export declare function readAllTelemetry(repoRoot: string, lastK: number, opts?:
|
|
|
84
118
|
})[];
|
|
85
119
|
export declare const RUNS_WINDOW = 50;
|
|
86
120
|
export declare function readProfileCursor(repoRoot: string): string | undefined;
|
|
121
|
+
export declare function profileDiscountsPath(repoRoot: string): string;
|
|
122
|
+
export declare function readProfileDiscounts(repoRoot: string): ProfileDiscount[];
|
|
123
|
+
export declare function appendProfileDiscount(repoRoot: string, discount: ProfileDiscount): void;
|
|
87
124
|
export declare function loadRoutingProfile(repoRoot: string, cfg: TickmarkrConfig, opts?: {
|
|
88
125
|
preview?: boolean;
|
|
89
126
|
}): RoutingProfile | undefined;
|