tickmarkr 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/catalog-remote.d.ts +11 -0
- package/dist/adapters/catalog-remote.js +55 -13
- package/dist/adapters/claude-code.js +7 -2
- package/dist/adapters/codex.d.ts +1 -0
- package/dist/adapters/codex.js +68 -8
- package/dist/adapters/qwen.js +3 -3
- package/dist/adapters/registry.js +13 -1
- package/dist/adapters/types.d.ts +4 -0
- package/dist/adapters/types.js +33 -0
- package/dist/cli/commands/compile.d.ts +3 -0
- package/dist/cli/commands/compile.js +77 -46
- package/dist/cli/commands/doctor.js +14 -9
- package/dist/cli/commands/fleet.js +11 -5
- package/dist/cli/commands/init.js +12 -13
- package/dist/cli/commands/plan.js +47 -7
- package/dist/cli/commands/run.js +20 -1
- package/dist/cli/commands/status.js +20 -21
- package/dist/cli/commands/version.js +2 -2
- package/dist/compile/collateral.d.ts +14 -5
- package/dist/compile/collateral.js +32 -26
- package/dist/compile/index.js +17 -6
- package/dist/compile/native.js +10 -2
- package/dist/compile/ownership.js +15 -9
- package/dist/drivers/herdr.d.ts +1 -0
- package/dist/drivers/herdr.js +32 -3
- package/dist/drivers/orca.d.ts +5 -1
- package/dist/drivers/orca.js +51 -2
- package/dist/drivers/types.d.ts +1 -1
- package/dist/gates/baseline.d.ts +26 -2
- package/dist/gates/baseline.js +90 -7
- package/dist/gates/review.d.ts +2 -2
- package/dist/gates/review.js +2 -18
- package/dist/graph/graph.d.ts +20 -0
- package/dist/graph/graph.js +66 -1
- package/dist/route/preference.d.ts +3 -1
- package/dist/route/preference.js +4 -4
- package/dist/run/consult.js +1 -0
- package/dist/run/daemon.d.ts +5 -0
- package/dist/run/daemon.js +151 -22
- package/dist/run/git.d.ts +2 -0
- package/dist/run/git.js +18 -4
- package/dist/run/journal.d.ts +1 -1
- package/dist/run/journal.js +1 -1
- package/dist/run/lock.d.ts +6 -0
- package/dist/run/lock.js +41 -1
- package/package.json +1 -1
- package/skills/tickmarkr-overseer/SKILL.md +39 -4
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +79 -0
- package/skills/tickmarkr-overseer/scripts/watch-context.sh +90 -4
package/dist/graph/graph.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
3
|
+
import { isAbsolute, join } from "node:path";
|
|
4
4
|
import { validateGraph } from "./schema.js";
|
|
5
5
|
export function stateDirName(_repoRoot) {
|
|
6
6
|
return ".tickmarkr";
|
|
@@ -8,6 +8,71 @@ export function stateDirName(_repoRoot) {
|
|
|
8
8
|
export function graphPath(repoRoot) {
|
|
9
9
|
return join(repoRoot, stateDirName(repoRoot), "graph.json");
|
|
10
10
|
}
|
|
11
|
+
export function compileRefusalPath(repoRoot) {
|
|
12
|
+
return join(repoRoot, stateDirName(repoRoot), "compile-refusal.json");
|
|
13
|
+
}
|
|
14
|
+
export function readCompileRefusal(repoRoot) {
|
|
15
|
+
const path = compileRefusalPath(repoRoot);
|
|
16
|
+
if (!existsSync(path))
|
|
17
|
+
return undefined;
|
|
18
|
+
let value;
|
|
19
|
+
try {
|
|
20
|
+
value = JSON.parse(readFileSync(path, "utf8"));
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
throw new Error(`compile refusal record at ${path} is unreadable: ${error instanceof Error ? error.message : String(error)}`);
|
|
24
|
+
}
|
|
25
|
+
if (typeof value !== "object" || value === null
|
|
26
|
+
|| typeof value.refusedAt !== "string"
|
|
27
|
+
|| typeof value.source !== "string"
|
|
28
|
+
|| typeof value.error !== "string") {
|
|
29
|
+
throw new Error(`compile refusal record at ${path} is malformed`);
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
export function saveCompileRefusal(repoRoot, record) {
|
|
34
|
+
tickmarkrDir(repoRoot);
|
|
35
|
+
const path = compileRefusalPath(repoRoot);
|
|
36
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
37
|
+
try {
|
|
38
|
+
writeFileSync(tmp, JSON.stringify(record, null, 2) + "\n");
|
|
39
|
+
renameSync(tmp, path);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
rmSync(tmp, { force: true });
|
|
43
|
+
throw error;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
export function clearCompileRefusal(repoRoot) {
|
|
47
|
+
rmSync(compileRefusalPath(repoRoot), { force: true });
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Re-hash the exact single source file recorded by CLI compiles. Native and PRD record the source
|
|
51
|
+
* markdown; Spec Kit records its tasks.md. GSD combines several plan bodies and is deliberately not
|
|
52
|
+
* reconstructed here. A missing file is the one fail-open case: the refusal record is the fail-closed
|
|
53
|
+
* evidence for failed recompiles, while moved/deleted source files need not strand a compiled graph.
|
|
54
|
+
*/
|
|
55
|
+
export function onDiskSpecHash(_repoRoot, graph) {
|
|
56
|
+
if (graph.spec.source === "gsd")
|
|
57
|
+
return undefined;
|
|
58
|
+
if (graph.spec.paths.length !== 1)
|
|
59
|
+
return undefined;
|
|
60
|
+
const recorded = graph.spec.paths[0];
|
|
61
|
+
// CLI compilation records an absolute source. Relative paths belong to legacy/programmatic
|
|
62
|
+
// graphs whose resolution context is unknowable here, so preserve their prior fail-open behavior.
|
|
63
|
+
if (!isAbsolute(recorded))
|
|
64
|
+
return undefined;
|
|
65
|
+
const path = recorded;
|
|
66
|
+
try {
|
|
67
|
+
const content = readFileSync(path);
|
|
68
|
+
return { path, hash: createHash("sha256").update(content).digest("hex") };
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
if (error.code === "ENOENT")
|
|
72
|
+
return undefined;
|
|
73
|
+
throw new Error(`cannot verify compiled spec hash from ${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
11
76
|
// T3 (Sol #2 / Fable F2): ONE canonical engagement identity over COMPILED TASK DEFINITIONS only.
|
|
12
77
|
// status/evidence are runtime-mutated (the daemon flips status, accumulates evidence every attempt) so
|
|
13
78
|
// they are excluded — the identity survives a status flip or evidence growth but changes the instant a
|
|
@@ -5,7 +5,9 @@ export interface Disallowed {
|
|
|
5
5
|
entry: string;
|
|
6
6
|
}
|
|
7
7
|
export type PreferenceRole = "worker" | "judge" | "review" | "consult";
|
|
8
|
-
|
|
8
|
+
/** Provider identity comes from the served model, not a gateway adapter's stamped vendor. */
|
|
9
|
+
export declare function modelProvider(model: string, fallback?: string): string;
|
|
10
|
+
export declare const routingModelProvider: typeof modelProvider;
|
|
9
11
|
export declare const modelRouteIdentity: (model: string, fallback?: string) => string;
|
|
10
12
|
export declare const channelRouteIdentity: (key: string, fallback?: string) => string;
|
|
11
13
|
export declare function routingEntrySeatLines(cfg: TickmarkrConfig): string[];
|
package/dist/route/preference.js
CHANGED
|
@@ -2,10 +2,8 @@ import { channelKey, channelsFromConfig } from "../adapters/types.js";
|
|
|
2
2
|
import { validateGraph } from "../graph/schema.js";
|
|
3
3
|
import { route, RoutingError } from "./router.js";
|
|
4
4
|
const PREFERENCE_ROLES = ["worker", "judge", "review", "consult"];
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
// avoids importing gates back into route/run and must move with that helper when the scope permits.
|
|
8
|
-
export function routingModelProvider(model, fallback = "unknown") {
|
|
5
|
+
/** Provider identity comes from the served model, not a gateway adapter's stamped vendor. */
|
|
6
|
+
export function modelProvider(model, fallback = "unknown") {
|
|
9
7
|
const id = model.toLowerCase();
|
|
10
8
|
const prefix = id.includes("/") ? id.slice(0, id.indexOf("/")) : "";
|
|
11
9
|
if (prefix === "openai" || prefix === "openai-codex" || /^(?:gpt|o\d)/.test(id))
|
|
@@ -22,6 +20,8 @@ export function routingModelProvider(model, fallback = "unknown") {
|
|
|
22
20
|
return "moonshot";
|
|
23
21
|
return fallback;
|
|
24
22
|
}
|
|
23
|
+
// Router naming remains explicit while sharing the review gate's exact function object.
|
|
24
|
+
export const routingModelProvider = modelProvider;
|
|
25
25
|
export const modelRouteIdentity = (model, fallback = "unknown") => `${routingModelProvider(model, fallback)}/${model.slice(model.lastIndexOf("/") + 1).toLowerCase()}`;
|
|
26
26
|
export const channelRouteIdentity = (key, fallback = "unknown") => {
|
|
27
27
|
const i = key.indexOf(":");
|
package/dist/run/consult.js
CHANGED
|
@@ -63,6 +63,7 @@ function excludedProviderFromDossier(d, adapter) {
|
|
|
63
63
|
try {
|
|
64
64
|
const events = JSON.parse(d.journalTail);
|
|
65
65
|
const assignment = [...events].reverse().find((event) => event.event === "task-dispatch"
|
|
66
|
+
&& event.taskId === d.taskId
|
|
66
67
|
&& event.data?.assignment?.adapter === adapter
|
|
67
68
|
&& typeof event.data.assignment.model === "string")?.data?.assignment;
|
|
68
69
|
if (typeof assignment?.model !== "string")
|
package/dist/run/daemon.d.ts
CHANGED
|
@@ -117,6 +117,11 @@ export declare const WORKER_NUDGE_MESSAGE = "tickmarkr liveness check: if the ta
|
|
|
117
117
|
/** Test seam — shrink the nudge gate and grace without minute-long sleeps. */
|
|
118
118
|
export declare function setNudgeTimingForTests(silentMs: number, graceMs: number): void;
|
|
119
119
|
export declare function resetNudgeTimingForTests(): void;
|
|
120
|
+
export declare const WORKER_STARTUP_WINDOW_MS = 60000;
|
|
121
|
+
export declare const WORKER_STARTUP_WINDOW_BYTES: number;
|
|
122
|
+
/** Test seam — exercises both sides of the startup boundary without a production-length fixture. */
|
|
123
|
+
export declare function setWorkerStartupWindowMsForTests(ms: number): void;
|
|
124
|
+
export declare function resetWorkerStartupWindowMsForTests(): void;
|
|
120
125
|
/** Test seam — shrink the quota-banner silence gate without minute-long sleeps. */
|
|
121
126
|
export declare function setQuotaBannerSilentMsForTests(ms: number): void;
|
|
122
127
|
export declare function resetQuotaBannerSilentMsForTests(): void;
|
package/dist/run/daemon.js
CHANGED
|
@@ -14,7 +14,7 @@ import { collateralHits } from "../compile/collateral.js";
|
|
|
14
14
|
import { DEFAULT_DIFF_CAP, globalConfigDir, loadConfigWithMode, readOverlayFile, repoOverlayPath, } from "../config/config.js";
|
|
15
15
|
import { DeliveryReadinessError } from "../drivers/herdr.js";
|
|
16
16
|
import { driverEvidence } from "../drivers/index.js";
|
|
17
|
-
import { herdrSealShellPrefix, SubprocessDriver } from "../drivers/subprocess.js";
|
|
17
|
+
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
18
18
|
import { formatOwnedName } from "../drivers/types.js";
|
|
19
19
|
import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../gates/baseline.js";
|
|
20
20
|
import { runGates } from "../gates/run-gates.js";
|
|
@@ -347,6 +347,48 @@ export function resetNudgeTimingForTests() {
|
|
|
347
347
|
const QUOTA_BANNER_SILENT_MS = 3 * 60_000;
|
|
348
348
|
let quotaBannerSilentMs = QUOTA_BANNER_SILENT_MS;
|
|
349
349
|
const WORKER_STARTUP_FAILURE_RE = /(?:not logged in|authentication(?:[ _-](?:required|failed|error))|401 unauthorized|model[_ -]?not[_ -]?found|(?:model|deployment)[^\n]{0,80}(?:not found|does not exist|is unavailable)|(?:404|not[_ -]?found)[^\n]{0,80}(?:model|deployment))/i;
|
|
350
|
+
// A startup banner is evidence only while the process is starting. Applying this matcher to every
|
|
351
|
+
// later poll let ordinary worker prose (including a diff that mentions model-not-found handling)
|
|
352
|
+
// condemn a live attempt. Bound both dimensions: time prevents a late line from becoming startup
|
|
353
|
+
// evidence, and bytes prevent a chatty launch from keeping an ever-growing classification surface.
|
|
354
|
+
export const WORKER_STARTUP_WINDOW_MS = EARLY_LAUNCH_LIVENESS_MS;
|
|
355
|
+
export const WORKER_STARTUP_WINDOW_BYTES = 64 * 1024;
|
|
356
|
+
let workerStartupWindowMs = WORKER_STARTUP_WINDOW_MS;
|
|
357
|
+
/** Test seam — exercises both sides of the startup boundary without a production-length fixture. */
|
|
358
|
+
export function setWorkerStartupWindowMsForTests(ms) {
|
|
359
|
+
workerStartupWindowMs = ms;
|
|
360
|
+
}
|
|
361
|
+
export function resetWorkerStartupWindowMsForTests() {
|
|
362
|
+
workerStartupWindowMs = WORKER_STARTUP_WINDOW_MS;
|
|
363
|
+
}
|
|
364
|
+
function startupFailureInWindow(output, launchedAt) {
|
|
365
|
+
if (launchedAt === undefined || Date.now() - launchedAt > workerStartupWindowMs)
|
|
366
|
+
return false;
|
|
367
|
+
const bytes = Buffer.from(output);
|
|
368
|
+
const bounded = bytes.length <= WORKER_STARTUP_WINDOW_BYTES
|
|
369
|
+
? output
|
|
370
|
+
: bytes.subarray(0, WORKER_STARTUP_WINDOW_BYTES).toString("utf8");
|
|
371
|
+
return WORKER_STARTUP_FAILURE_RE.test(stallSnapshotBannerRows(bounded));
|
|
372
|
+
}
|
|
373
|
+
// OBS-901/906: one daemon-owned writer for every execution surface. Drivers expose one retained
|
|
374
|
+
// stream through read(); the daemon persists that exact surface before any close/reconcile can make
|
|
375
|
+
// it disappear. MAX_BUF is the execution layer's existing two-MiB retention contract; truncate the
|
|
376
|
+
// encoded bytes from the head so the completion/error tail is always the part that survives.
|
|
377
|
+
async function captureWorkerStream(journalDir, taskId, attempt, driver, slot, fallback) {
|
|
378
|
+
let captured = fallback;
|
|
379
|
+
try {
|
|
380
|
+
captured = await driver.read(slot, PANE_READ_ROWS);
|
|
381
|
+
}
|
|
382
|
+
catch {
|
|
383
|
+
// Earlier successful reads are still a captured stream. Persist them rather than losing the
|
|
384
|
+
// only failure evidence because the pane vanished between its terminal read and this snapshot.
|
|
385
|
+
}
|
|
386
|
+
const bytes = Buffer.from(captured);
|
|
387
|
+
const tail = bytes.length > MAX_BUF ? bytes.subarray(bytes.length - MAX_BUF) : bytes;
|
|
388
|
+
const path = posix.join("prompts", `${taskId}-a${attempt}.out`);
|
|
389
|
+
writeFileSync(join(journalDir, path), tail);
|
|
390
|
+
return { path, output: captured };
|
|
391
|
+
}
|
|
350
392
|
/** Test seam — shrink the quota-banner silence gate without minute-long sleeps. */
|
|
351
393
|
export function setQuotaBannerSilentMsForTests(ms) {
|
|
352
394
|
quotaBannerSilentMs = ms;
|
|
@@ -2329,6 +2371,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2329
2371
|
// starts at 0 (fresh budget) instead of re-parking at the cap in the same tick.
|
|
2330
2372
|
let providerDeathRequeues = 0;
|
|
2331
2373
|
let providerDeathAttempt = -1;
|
|
2374
|
+
// A completed attempt may keep its pane for gate-failure context, but once another worker is
|
|
2375
|
+
// funded that context is superseded and — more importantly — must not remain a second writer on
|
|
2376
|
+
// the same worktree. The next loop iteration retires it before reading/recreating the tree.
|
|
2377
|
+
let supersededWorkerSlot;
|
|
2332
2378
|
attempts: for (let attempt = rs?.attempts ?? 0;; attempt++) {
|
|
2333
2379
|
if (attempt !== providerDeathAttempt) {
|
|
2334
2380
|
providerDeathRequeues = 0;
|
|
@@ -2343,6 +2389,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2343
2389
|
await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2344
2390
|
return;
|
|
2345
2391
|
}
|
|
2392
|
+
if (supersededWorkerSlot && !keepForever) {
|
|
2393
|
+
const prior = supersededWorkerSlot;
|
|
2394
|
+
supersededWorkerSlot = undefined;
|
|
2395
|
+
const idx = keptSlots.indexOf(prior);
|
|
2396
|
+
if (idx >= 0)
|
|
2397
|
+
keptSlots.splice(idx, 1);
|
|
2398
|
+
try {
|
|
2399
|
+
await closeSlot(prior);
|
|
2400
|
+
}
|
|
2401
|
+
catch (error) {
|
|
2402
|
+
await park(t, `superseded worker could not be swept before dispatch: ${error instanceof Error ? error.message : String(error)}`, "stall", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
2403
|
+
return;
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2346
2406
|
// OBS-419: read the approval-carried ceiling at the funding boundary, after every escalation
|
|
2347
2407
|
// decision but before the sole task-dispatch below. Both interactive and headless worker-launch
|
|
2348
2408
|
// branches are dominated by this check, so no route can fund another worker without consulting it.
|
|
@@ -2657,11 +2717,17 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2657
2717
|
// between task-dispatch and this line — worktree recreation, setup, prompt write, slot allocation,
|
|
2658
2718
|
// the launch itself — can still die with no worker having seen the brief, and `--retry-failed`
|
|
2659
2719
|
// must then re-send that same brief rather than a fresh prompt on a possibly banned channel.
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2720
|
+
let workerLaunchedAt;
|
|
2721
|
+
const noteLaunched = async () => {
|
|
2722
|
+
// Startup evidence begins when the worker exists, not while prompt preparation, slot
|
|
2723
|
+
// allocation, or interactive readiness is still consuming the attempt wall clock.
|
|
2724
|
+
workerLaunchedAt ??= Date.now();
|
|
2725
|
+
journal.append("worker-launch", t.id, {
|
|
2726
|
+
attempt,
|
|
2727
|
+
retryMode,
|
|
2728
|
+
...(trackedDriver.describe ? await trackedDriver.describe(slot) : {}),
|
|
2729
|
+
});
|
|
2730
|
+
};
|
|
2665
2731
|
const noteWorkerLiveness = (event, data) => journal.append(event, t.id, { ...data, source: trackedDriver.readSource ?? "driver.read" });
|
|
2666
2732
|
let cpuFlat;
|
|
2667
2733
|
let cpuAccountant;
|
|
@@ -2898,7 +2964,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2898
2964
|
// latch, and the second is a no-op. A seed that answered is never re-answered by the loop.
|
|
2899
2965
|
if (seedResult?.trustAnswered)
|
|
2900
2966
|
noteSeedTrustAnswered();
|
|
2901
|
-
startupFailure =
|
|
2967
|
+
startupFailure = startupFailureInWindow(output, workerLaunchedAt);
|
|
2902
2968
|
if (seedResult?.seedFailed || startupFailure) {
|
|
2903
2969
|
finished = false;
|
|
2904
2970
|
}
|
|
@@ -2942,6 +3008,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2942
3008
|
const sliceStart = Date.now();
|
|
2943
3009
|
const remaining = stallWindowMs - (sliceStart - lastProgressAt);
|
|
2944
3010
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3011
|
+
const startupLeft = workerLaunchedAt === undefined
|
|
3012
|
+
? 0
|
|
3013
|
+
: workerStartupWindowMs - (sliceStart - workerLaunchedAt);
|
|
3014
|
+
if (startupLeft > 0)
|
|
3015
|
+
slice = Math.min(slice, Math.max(1, startupLeft));
|
|
2945
3016
|
if (!everHadOutput) {
|
|
2946
3017
|
const earlyLeft = earlyLaunchLivenessMs - (sliceStart - attemptStart);
|
|
2947
3018
|
if (earlyLeft > 0)
|
|
@@ -2961,6 +3032,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2961
3032
|
const exit = exitRe.exec(output);
|
|
2962
3033
|
if (finished || exit) {
|
|
2963
3034
|
exitCode = exit ? Number(exit[1]) : null; // null ⇔ the TUI is still alive
|
|
3035
|
+
processExited = exit !== null;
|
|
2964
3036
|
await sampleContext(); // final poll-seam sample before leaving the wait
|
|
2965
3037
|
break;
|
|
2966
3038
|
}
|
|
@@ -3000,7 +3072,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3000
3072
|
}
|
|
3001
3073
|
if (paneText.length > 0)
|
|
3002
3074
|
everHadOutput = true;
|
|
3003
|
-
if (
|
|
3075
|
+
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
3004
3076
|
startupFailure = true;
|
|
3005
3077
|
output = paneText;
|
|
3006
3078
|
break;
|
|
@@ -3353,6 +3425,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3353
3425
|
const exit = exitRe.exec(output);
|
|
3354
3426
|
if (finished || exit) {
|
|
3355
3427
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3428
|
+
processExited = exit !== null;
|
|
3356
3429
|
await sampleContext();
|
|
3357
3430
|
break;
|
|
3358
3431
|
}
|
|
@@ -3398,6 +3471,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3398
3471
|
finished = new RegExp(trailerPattern(nonce)).test(output);
|
|
3399
3472
|
const exit = exitRe.exec(output);
|
|
3400
3473
|
exitCode = exit ? Number(exit[1]) : null;
|
|
3474
|
+
processExited = exit !== null;
|
|
3401
3475
|
}
|
|
3402
3476
|
if (finished) {
|
|
3403
3477
|
await driver.waitAgentStatus(slot, "idle", 5_000); // settle, then re-harvest the final render
|
|
@@ -3444,7 +3518,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3444
3518
|
// v1.76: same monotonic-progress measure as the interactive site; harvest stays raw.
|
|
3445
3519
|
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
3446
3520
|
const initialPane = await driver.read(slot, 500);
|
|
3447
|
-
startupFailure =
|
|
3521
|
+
startupFailure = startupFailureInWindow(initialPane, workerLaunchedAt);
|
|
3448
3522
|
let everHadOutput = initialPane.length > 0;
|
|
3449
3523
|
const stallProgress = new StallProgressTracker();
|
|
3450
3524
|
stallProgress.observe({ paneText: initialPane, seedSubmitted: true, contextTokens });
|
|
@@ -3460,6 +3534,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3460
3534
|
while (!startupFailure && Date.now() - lastProgressAt < stallWindowMs) {
|
|
3461
3535
|
const remaining = stallWindowMs - (Date.now() - lastProgressAt);
|
|
3462
3536
|
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
3537
|
+
const startupLeft = workerLaunchedAt === undefined
|
|
3538
|
+
? 0
|
|
3539
|
+
: workerStartupWindowMs - (Date.now() - workerLaunchedAt);
|
|
3540
|
+
if (startupLeft > 0)
|
|
3541
|
+
slice = Math.min(slice, Math.max(1, startupLeft));
|
|
3463
3542
|
if (!everHadOutput) {
|
|
3464
3543
|
const earlyLeft = earlyLaunchLivenessMs - (Date.now() - attemptStart);
|
|
3465
3544
|
if (earlyLeft > 0)
|
|
@@ -3474,7 +3553,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3474
3553
|
const paneText = await driver.read(slot, 500);
|
|
3475
3554
|
if (paneText.length > 0)
|
|
3476
3555
|
everHadOutput = true;
|
|
3477
|
-
if (
|
|
3556
|
+
if (startupFailureInWindow(paneText, workerLaunchedAt)) {
|
|
3478
3557
|
startupFailure = true;
|
|
3479
3558
|
break;
|
|
3480
3559
|
}
|
|
@@ -3516,12 +3595,52 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3516
3595
|
if (interactive && finished && !exitRe.test(output)) {
|
|
3517
3596
|
await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, 2_000, { regex: true });
|
|
3518
3597
|
}
|
|
3519
|
-
//
|
|
3520
|
-
//
|
|
3521
|
-
|
|
3598
|
+
// Free availability reroutes deliberately reuse the charged-attempt ordinal. Stream artifacts
|
|
3599
|
+
// cannot: every dispatch owns bytes that must remain independently recoverable, so derive the
|
|
3600
|
+
// append-only dispatch ordinal from the journal instead of overwriting an earlier a<n>.out.
|
|
3601
|
+
const streamAttempt = journal.read().filter((event) => event.event === "task-dispatch" && event.taskId === t.id).length - 1;
|
|
3602
|
+
const capturedStream = await captureWorkerStream(journal.dir, t.id, streamAttempt, driver, slot, output);
|
|
3603
|
+
// An interactive settle already selected the bounded parse it is allowed to use. Every other
|
|
3604
|
+
// path parses the common final stream snapshot — notably envelope adapters whose trailer is
|
|
3605
|
+
// encoded and therefore invisible to the raw wait regex. Keep `output` bounded: every text
|
|
3606
|
+
// classifier below retains its pre-artifact tail surface.
|
|
3607
|
+
let result = settleParsed ?? adapter.parse(capturedStream.output, nonce);
|
|
3608
|
+
const adapterCause = result.cause;
|
|
3609
|
+
// Law 46: there is one completion fact. A classified adapter result with no parse-failure cause
|
|
3610
|
+
// means its decoder found the nonce-bound trailer even when the raw JSON envelope escaped it.
|
|
3611
|
+
const workerFinished = finished || adapterCause === undefined;
|
|
3612
|
+
finished = workerFinished;
|
|
3613
|
+
const adapterStartupFailure = adapterCause === "startup-failure";
|
|
3614
|
+
const workerCause = adapterStartupFailure
|
|
3615
|
+
? "startup-failure"
|
|
3616
|
+
: startupFailure
|
|
3617
|
+
? "startup-failure"
|
|
3618
|
+
: classifyWorkerResultCause({ output, ok: result.ok, finished: workerFinished, exitCode, summary: result.summary, timedOut, deadChannel: deadChannelKilled });
|
|
3619
|
+
// OBS-897/889(4): neither a timeout/quiet-harvest nor a trailer is a verdict on a tree while
|
|
3620
|
+
// its writer remains alive. Reap through the same driver lifecycle every execution surface
|
|
3621
|
+
// already implements, then (and only then) permit commitsAheadOf/gates below to read the tree.
|
|
3622
|
+
// A failed reap is a stall disposition, never permission to gate a moving branch.
|
|
3623
|
+
let reapFailure;
|
|
3624
|
+
if (!processExited && !deadWorkerPark) {
|
|
3625
|
+
try {
|
|
3626
|
+
await closeSlot(slot);
|
|
3627
|
+
if (!workerFinished) {
|
|
3628
|
+
journal.append("worker-reaped-before-harvest", t.id, {
|
|
3629
|
+
slot: slot.name, attempt, cause: workerCause ?? "stall-timeout",
|
|
3630
|
+
});
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
catch (error) {
|
|
3634
|
+
reapFailure = error instanceof Error ? error.message : String(error);
|
|
3635
|
+
}
|
|
3636
|
+
}
|
|
3637
|
+
else if (keepOpen && (workerFinished || processExited || driver.id !== "subprocess")) {
|
|
3522
3638
|
keptSlots.push(slot);
|
|
3523
|
-
|
|
3639
|
+
supersededWorkerSlot = slot;
|
|
3640
|
+
}
|
|
3641
|
+
else {
|
|
3524
3642
|
await closeSlot(slot);
|
|
3643
|
+
}
|
|
3525
3644
|
// SPEND-01: usage from the harness's own cwd-keyed structured store, read POST-HOC from disk —
|
|
3526
3645
|
// `wt` is this task's private worktree, so the path is unique; the read is sliced to records
|
|
3527
3646
|
// stamped at/after this attempt's dispatch instant. Never the harvested pane text, never the
|
|
@@ -3537,15 +3656,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3537
3656
|
await park(t, deadWorkerPark.reason, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { ref: deadWorkerPark.ref });
|
|
3538
3657
|
return;
|
|
3539
3658
|
}
|
|
3540
|
-
let result = settleParsed ?? adapter.parse(output, nonce);
|
|
3541
|
-
const workerFinished = finished;
|
|
3542
|
-
const workerCause = startupFailure
|
|
3543
|
-
? "startup-failure"
|
|
3544
|
-
: classifyWorkerResultCause({ output, ok: result.ok, finished, exitCode, summary: result.summary, timedOut, deadChannel: deadChannelKilled });
|
|
3545
3659
|
journal.append("worker-result", t.id, {
|
|
3546
3660
|
ok: result.ok, summary: result.summary, deviations: result.deviations, finished: workerFinished, exitCode,
|
|
3547
|
-
mode: interactive ? "interactive" : "print",
|
|
3661
|
+
mode: interactive ? "interactive" : "print", stream: capturedStream.path,
|
|
3662
|
+
...(workerCause ? { cause: workerCause } : {}),
|
|
3548
3663
|
});
|
|
3664
|
+
if (reapFailure) {
|
|
3665
|
+
await park(t, `worker could not be reaped before harvest: ${reapFailure}`, "stall", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { reapFailure });
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3549
3668
|
// T2 review (routing precedence): provider-death, quota and dead-channel classification
|
|
3550
3669
|
// derive from ONE rule — the worker's OWN outcome, workerFinished and this PRE-HARVEST
|
|
3551
3670
|
// parse — never from the synthesized result below. A worker that committed and then walled
|
|
@@ -3632,6 +3751,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3632
3751
|
}
|
|
3633
3752
|
catch { /* cosmetic — reconcile is the backstop */ }
|
|
3634
3753
|
}
|
|
3754
|
+
if (supersededWorkerSlot === slot)
|
|
3755
|
+
supersededWorkerSlot = undefined;
|
|
3635
3756
|
}
|
|
3636
3757
|
// v1.8 TEL-05 — FROM-channel attribution for mid-task quota failover: `assignment` is still the
|
|
3637
3758
|
// throttled-away-FROM channel here (before the reassign below). durationMs:0 marks this as a
|
|
@@ -3658,7 +3779,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3658
3779
|
// T2 review: classify the PRE-HARVEST parse — classifyDeadChannel bails on any ok:true
|
|
3659
3780
|
// result, so reading the synthesized harvest result would swallow auth-required /
|
|
3660
3781
|
// setup-required / provider-outage for every committed-but-walled attempt (in both modes).
|
|
3661
|
-
const dead = classifyDeadChannel(preHarvestResult
|
|
3782
|
+
const dead = classifyDeadChannel({ ...preHarvestResult, raw: output })
|
|
3783
|
+
?? (earlyLaunchDead || startupFailure || adapterStartupFailure ? "setup-required" : undefined);
|
|
3662
3784
|
if (dead) {
|
|
3663
3785
|
const from = channelKey(assignment);
|
|
3664
3786
|
demotedChannels.add(from); // excluded for later attempts AND later tasks in this run
|
|
@@ -3677,12 +3799,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3677
3799
|
}
|
|
3678
3800
|
catch { /* cosmetic — reconcile is the backstop */ }
|
|
3679
3801
|
}
|
|
3802
|
+
if (supersededWorkerSlot === slot)
|
|
3803
|
+
supersededWorkerSlot = undefined;
|
|
3680
3804
|
}
|
|
3681
3805
|
assignment = next;
|
|
3682
3806
|
tried.push(channelKey(next));
|
|
3807
|
+
// Startup failures bought no worker turn: preserve the attempt ordinal/budget while the
|
|
3808
|
+
// verified-dead channel remains excluded for every later dispatch in this run.
|
|
3809
|
+
if (adapterStartupFailure || startupFailure)
|
|
3810
|
+
attempt--;
|
|
3683
3811
|
continue;
|
|
3684
3812
|
}
|
|
3685
|
-
|
|
3813
|
+
const chargedAttempts = adapterStartupFailure || startupFailure ? attempt : attempt + 1;
|
|
3814
|
+
await park(t, `dead channel (${dead}) and no eligible channel remains`, "reroute-exhausted", assignment, chargedAttempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3686
3815
|
return;
|
|
3687
3816
|
}
|
|
3688
3817
|
if (!finished) {
|
package/dist/run/git.d.ts
CHANGED
|
@@ -92,6 +92,8 @@ export interface ShResult {
|
|
|
92
92
|
timedOut?: boolean;
|
|
93
93
|
/** The shell exited, but descendants still held its process group and pipes past the grace. */
|
|
94
94
|
reapedGroup?: boolean;
|
|
95
|
+
/** The grace reap's group kill failed with something other than ESRCH (for example EPERM). */
|
|
96
|
+
reapError?: string;
|
|
95
97
|
durationMs?: number;
|
|
96
98
|
/** T7: the capacity THIS child ran under, stamped where its environment was built (see `shell`). */
|
|
97
99
|
capacity?: RunCapacity;
|
package/dist/run/git.js
CHANGED
|
@@ -174,6 +174,7 @@ function shell(cmd, cwd, timeoutMs, login) {
|
|
|
174
174
|
const stdoutDecoder = new StringDecoder("utf8");
|
|
175
175
|
const stderrDecoder = new StringDecoder("utf8");
|
|
176
176
|
let timedOut = false, reapedGroup = false, done = false, started = false, outputSeen = false;
|
|
177
|
+
let reapError, exitedCode;
|
|
177
178
|
let reapTimer;
|
|
178
179
|
const finish = (code, err) => {
|
|
179
180
|
if (done)
|
|
@@ -191,6 +192,7 @@ function shell(cmd, cwd, timeoutMs, login) {
|
|
|
191
192
|
durationMs: Date.now() - startedAt,
|
|
192
193
|
capacity,
|
|
193
194
|
...(reapedGroup ? { reapedGroup: true } : {}),
|
|
195
|
+
...(reapError ? { reapError } : {}),
|
|
194
196
|
});
|
|
195
197
|
};
|
|
196
198
|
const timer = setTimeout(() => {
|
|
@@ -198,8 +200,15 @@ function shell(cmd, cwd, timeoutMs, login) {
|
|
|
198
200
|
try {
|
|
199
201
|
process.kill(-p.pid, "SIGKILL");
|
|
200
202
|
}
|
|
201
|
-
catch {
|
|
203
|
+
catch (error) {
|
|
204
|
+
const code = error?.code;
|
|
205
|
+
if (code !== "ESRCH") {
|
|
206
|
+
reapError ??= `${code ?? "kill"}: ${error instanceof Error ? error.message : String(error)}`;
|
|
207
|
+
}
|
|
202
208
|
p.kill("SIGKILL");
|
|
209
|
+
// A failed group kill can leave inherited pipes open forever; the command ceiling still wins.
|
|
210
|
+
if (code !== "ESRCH")
|
|
211
|
+
finish(exitedCode ?? 1);
|
|
203
212
|
}
|
|
204
213
|
}, timeoutMs);
|
|
205
214
|
p.on("spawn", () => { started = true; }); // the command exists from here on — never retryable past it
|
|
@@ -233,8 +242,9 @@ function shell(cmd, cwd, timeoutMs, login) {
|
|
|
233
242
|
// cannot hold this promise until the command ceiling. A real timeout wins first and is never
|
|
234
243
|
// reclassified as a grace reap.
|
|
235
244
|
p.on("exit", (code) => {
|
|
245
|
+
exitedCode = code ?? 1;
|
|
236
246
|
if (timedOut) {
|
|
237
|
-
finish(
|
|
247
|
+
finish(exitedCode);
|
|
238
248
|
return;
|
|
239
249
|
}
|
|
240
250
|
reapTimer = setTimeout(() => {
|
|
@@ -244,8 +254,12 @@ function shell(cmd, cwd, timeoutMs, login) {
|
|
|
244
254
|
process.kill(-p.pid, "SIGKILL");
|
|
245
255
|
reapedGroup = true;
|
|
246
256
|
}
|
|
247
|
-
catch {
|
|
248
|
-
//
|
|
257
|
+
catch (error) {
|
|
258
|
+
// ESRCH means the group closed in the grace race; every other error must survive to the
|
|
259
|
+
// result while the command timeout remains the explicit backstop.
|
|
260
|
+
const code = error?.code;
|
|
261
|
+
if (code !== "ESRCH")
|
|
262
|
+
reapError = `${code ?? "kill"}: ${error instanceof Error ? error.message : String(error)}`;
|
|
249
263
|
}
|
|
250
264
|
}, SHELL_REAP_GRACE_MS);
|
|
251
265
|
});
|
package/dist/run/journal.d.ts
CHANGED
|
@@ -196,7 +196,7 @@ export declare const PARK_KINDS: readonly ["human-gate", "ladder-exhausted", "at
|
|
|
196
196
|
export type ParkKind = (typeof PARK_KINDS)[number];
|
|
197
197
|
export declare const RETRY_MODES: readonly ["resume", "fresh", "repair"];
|
|
198
198
|
export type RetryMode = (typeof RETRY_MODES)[number];
|
|
199
|
-
export declare const WORKER_RESULT_CAUSES: readonly ["provider-death", "dead-channel", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer"];
|
|
199
|
+
export declare const WORKER_RESULT_CAUSES: readonly ["provider-death", "dead-channel", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer", "startup-failure"];
|
|
200
200
|
export type WorkerResultCause = (typeof WORKER_RESULT_CAUSES)[number];
|
|
201
201
|
export declare function isQualityFailureParkKind(kind: ParkKind): boolean;
|
|
202
202
|
export declare function classifyTaskFailure(taskEvents: JournalEvent[]): ParkKind;
|
package/dist/run/journal.js
CHANGED
|
@@ -782,7 +782,7 @@ export const RETRY_MODES = ["resume", "fresh", "repair"];
|
|
|
782
782
|
// concludes a worker while the rolling stall window still has most of its time left, so labelling it
|
|
783
783
|
// stall-timeout points every downstream reader — the repair brief, the consult — at a mechanism that
|
|
784
784
|
// cannot have fired.
|
|
785
|
-
export const WORKER_RESULT_CAUSES = ["provider-death", "dead-channel", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer"];
|
|
785
|
+
export const WORKER_RESULT_CAUSES = ["provider-death", "dead-channel", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer", "startup-failure"];
|
|
786
786
|
// Status consumes the routing profile's existing quality split directly: verified park kinds classify
|
|
787
787
|
// to 0, while availability/recovery noise classifies to null. Keep the synthetic row here at the
|
|
788
788
|
// run↔route seam so presentation code never grows a second list of "bad" park kinds.
|
package/dist/run/lock.d.ts
CHANGED
|
@@ -9,6 +9,10 @@ export interface Inspection {
|
|
|
9
9
|
mtimeMs: number;
|
|
10
10
|
ino: number;
|
|
11
11
|
}
|
|
12
|
+
export type RunLineEvent = {
|
|
13
|
+
event: string;
|
|
14
|
+
ts: string;
|
|
15
|
+
};
|
|
12
16
|
export declare function shouldRefuse(i: Pick<Inspection, "garbage" | "dead">): boolean;
|
|
13
17
|
export declare function isPidLive(pid: number): boolean;
|
|
14
18
|
export declare function acquireRunLock(repoRoot: string, runId: string): {
|
|
@@ -29,6 +33,8 @@ export declare function runLockOwner(repoRoot: string): {
|
|
|
29
33
|
runId?: string;
|
|
30
34
|
live: boolean;
|
|
31
35
|
} | undefined;
|
|
36
|
+
export declare function runLockRunId(repoRoot: string): string | undefined;
|
|
37
|
+
export declare function runStatusLine(repoRoot: string, runId: string, events: readonly RunLineEvent[]): string | null;
|
|
32
38
|
export declare function isRunLockLive(repoRoot: string): boolean;
|
|
33
39
|
export declare function unlockRun(repoRoot: string): {
|
|
34
40
|
held: false;
|
package/dist/run/lock.js
CHANGED
|
@@ -2,6 +2,7 @@ import { linkSync, readFileSync, statSync, unlinkSync, utimesSync, writeFileSync
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { tickmarkrDir, stateDirName } from "../graph/graph.js";
|
|
5
|
+
import { parseRunId } from "./journal.js";
|
|
5
6
|
// HARD-01/02: coarse per-run advisory lock over .tickmarkr/graph.json. LOCK-02: the lock is created by
|
|
6
7
|
// the link(2) idiom — write the full payload to graph.lock.<pid>.tmp, then linkSync(tmp, lockPath),
|
|
7
8
|
// which is atomic and throws EEXIST if the lock already exists (the mutual-exclusion primitive).
|
|
@@ -238,7 +239,7 @@ export async function acquireApprovalSerialization(repoRoot, runId) {
|
|
|
238
239
|
export function runLockOwner(repoRoot) {
|
|
239
240
|
let insp;
|
|
240
241
|
try {
|
|
241
|
-
insp = inspect(
|
|
242
|
+
insp = inspect(join(repoRoot, stateDirName(repoRoot), "graph.lock"));
|
|
242
243
|
}
|
|
243
244
|
catch {
|
|
244
245
|
return undefined;
|
|
@@ -247,6 +248,45 @@ export function runLockOwner(repoRoot) {
|
|
|
247
248
|
// REPOSITORY-wide, so a live pid here is not proof it is running the run the caller cares about.
|
|
248
249
|
return { pid: insp.pid, runId: insp.runId, live: shouldRefuse(insp) };
|
|
249
250
|
}
|
|
251
|
+
export function runLockRunId(repoRoot) {
|
|
252
|
+
const runId = runLockOwner(repoRoot)?.runId;
|
|
253
|
+
if (runId === undefined)
|
|
254
|
+
return undefined;
|
|
255
|
+
try {
|
|
256
|
+
return parseRunId(runId);
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
export function runStatusLine(repoRoot, runId, events) {
|
|
263
|
+
let parsedRunId;
|
|
264
|
+
try {
|
|
265
|
+
parsedRunId = parseRunId(runId);
|
|
266
|
+
}
|
|
267
|
+
catch {
|
|
268
|
+
return null;
|
|
269
|
+
}
|
|
270
|
+
const owner = runLockOwner(repoRoot);
|
|
271
|
+
const ownerRunId = owner?.runId === undefined ? undefined : (() => {
|
|
272
|
+
try {
|
|
273
|
+
return parseRunId(owner.runId);
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
return undefined;
|
|
277
|
+
}
|
|
278
|
+
})();
|
|
279
|
+
if (owner && ownerRunId === parsedRunId) {
|
|
280
|
+
if (owner.live)
|
|
281
|
+
return `run ${parsedRunId} active`;
|
|
282
|
+
return owner.pid === undefined
|
|
283
|
+
? `run ${parsedRunId} stale lock`
|
|
284
|
+
: `run ${parsedRunId} stale lock naming dead holder pid ${owner.pid}`;
|
|
285
|
+
}
|
|
286
|
+
if (events.some((event) => event.event === "run-end"))
|
|
287
|
+
return null;
|
|
288
|
+
return `run ${parsedRunId} abandoned since ${events.at(-1)?.ts ?? "unknown"}`;
|
|
289
|
+
}
|
|
250
290
|
// Read-only predicate: true iff a lock exists that the decision table would REFUSE on (alive,
|
|
251
291
|
// EPERM, or ANY garbage). A provably-dead holder (ESRCH) reads not-live (LOCK-02/OBS-05). Never
|
|
252
292
|
// mutates the lock. compile now acquires via acquireRunLock; this remains for drift oracles/tests.
|