tickmarkr 1.70.0 → 1.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/kimi.js +12 -35
- package/dist/adapters/registry.d.ts +25 -1
- package/dist/adapters/registry.js +95 -4
- package/dist/adapters/types.d.ts +8 -0
- package/dist/cli/commands/doctor.js +8 -3
- package/dist/cli/commands/resume.js +7 -1
- package/dist/cli/commands/status.js +20 -8
- package/dist/drivers/herdr.d.ts +5 -0
- package/dist/drivers/herdr.js +22 -3
- package/dist/route/preference.d.ts +9 -0
- package/dist/route/preference.js +61 -0
- package/dist/run/daemon.d.ts +4 -0
- package/dist/run/daemon.js +67 -10
- package/dist/run/interactive-seed.d.ts +1 -0
- package/dist/run/interactive-seed.js +12 -4
- package/dist/run/journal.d.ts +9 -1
- package/dist/run/journal.js +53 -2
- package/package.json +1 -1
package/dist/adapters/kimi.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { spawnSync } from "node:child_process";
|
|
2
1
|
import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
3
|
import { join } from "node:path";
|
|
5
4
|
import { probeVersion } from "./claude-code.js";
|
|
6
5
|
import { parseWorkerResult } from "./prompt.js";
|
|
6
|
+
import { runInteractiveSeed } from "../run/interactive-seed.js";
|
|
7
|
+
import { sh } from "../run/git.js";
|
|
7
8
|
import { channelsFromConfig, MODEL_ID_RE, shq, TokenUsageSchema } from "./types.js";
|
|
8
9
|
// KIMI-03 → v1.58 T5: the "no harness-readable counter" block (research F-6, 2026-07-17) is
|
|
9
10
|
// LIFTED for collectUsage — kimi 0.27.0 writes a wire journal per agent at
|
|
@@ -117,40 +118,18 @@ export function confirmKimiSeedBanner(banner, assignedModel) {
|
|
|
117
118
|
}
|
|
118
119
|
return { ok: true, sessionId: kimiBannerSessionId(banner) };
|
|
119
120
|
}
|
|
120
|
-
// Shared launch-then-seed surface (T6) + banner confirm
|
|
121
|
-
//
|
|
121
|
+
// Shared launch-then-seed surface (T6) + banner confirm (T7/T2). One definition so the adapter
|
|
122
|
+
// property and the daemon's generic runInteractiveSeed path cannot drift.
|
|
122
123
|
const KIMI_SEED = {
|
|
123
124
|
launch: (model) => `kimi -y -m ${shq(kimiAlias(model))}`,
|
|
124
125
|
readinessMatch: "Send /help for help information.",
|
|
125
126
|
seedLine: (promptFile) => `Read ${promptFile} and do exactly what it says.`,
|
|
127
|
+
confirmBanner: confirmKimiSeedBanner,
|
|
126
128
|
};
|
|
127
|
-
// v1.69 T7:
|
|
128
|
-
// from the SAME pane text already present for the readiness match. No extra probe, no extra
|
|
129
|
-
// dispatch — one read of the launch banner, then seed or fail closed.
|
|
129
|
+
// v1.69 T7 / v1.71 T2: thin wrapper over the daemon's generic dispatch path for kimi-only tests.
|
|
130
130
|
export async function runKimiInteractiveSeed(opts) {
|
|
131
|
-
await
|
|
132
|
-
|
|
133
|
-
// Banner text already on the pane for the readiness match — the only text model/session use.
|
|
134
|
-
const banner = await opts.driver.read(opts.slot, 1000);
|
|
135
|
-
if (!ready) {
|
|
136
|
-
return { output: banner, seedFailed: true, seedError: `readiness pattern not seen: ${KIMI_SEED.readinessMatch}` };
|
|
137
|
-
}
|
|
138
|
-
const confirm = confirmKimiSeedBanner(banner, opts.assignment.model);
|
|
139
|
-
if (!confirm.ok) {
|
|
140
|
-
return { output: banner, seedFailed: true, seedError: confirm.error };
|
|
141
|
-
}
|
|
142
|
-
const seedText = KIMI_SEED.seedLine(opts.promptFile);
|
|
143
|
-
await opts.driver.run(opts.slot, seedText);
|
|
144
|
-
let output = "";
|
|
145
|
-
for (let attempt = 0; attempt < 5; attempt++) {
|
|
146
|
-
await new Promise((r) => setTimeout(r, 200));
|
|
147
|
-
output = await opts.driver.read(opts.slot, 1000);
|
|
148
|
-
const bottom = output.trimEnd().split("\n").pop() ?? "";
|
|
149
|
-
if (!bottom.includes(seedText)) {
|
|
150
|
-
return { output, seedFailed: false, sessionId: confirm.sessionId };
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
return { output, seedFailed: true, seedError: "seed line never left the input box", sessionId: confirm.sessionId };
|
|
131
|
+
const r = await runInteractiveSeed({ ...opts, adapter: kimi });
|
|
132
|
+
return { output: r.output, seedFailed: r.seedFailed, seedError: r.seedError, sessionId: r.sessionId };
|
|
154
133
|
}
|
|
155
134
|
export const kimi = {
|
|
156
135
|
id: "kimi",
|
|
@@ -183,10 +162,8 @@ export const kimi = {
|
|
|
183
162
|
// null → the daemon's print fallback (types.ts:101) keeps kimi workers visible without a TUI.
|
|
184
163
|
interactiveCommand: () => null,
|
|
185
164
|
// v1.69 T6/T7: launch the real TUI, wait for the cold-start readiness marker, then submit the
|
|
186
|
-
// prompt as one user turn. Banner model/session confirmation
|
|
187
|
-
//
|
|
188
|
-
// extra probe). Live-probed 2026-07-22 (kimi 0.28.1): `kimi -y` opens the TUI with yolo active
|
|
189
|
-
// and prints the model/session lines plus "Send /help for help information." before the input box.
|
|
165
|
+
// prompt as one user turn. Banner model/session confirmation runs on the daemon's generic
|
|
166
|
+
// runInteractiveSeed path via confirmBanner on KIMI_SEED (not a separate dispatch helper).
|
|
190
167
|
interactiveSeed: KIMI_SEED,
|
|
191
168
|
// v1.53 T3 resume — live-probed 2026-07-18: `-p` + `-S <id>` compose cleanly (no OBS-67-class
|
|
192
169
|
// flag rejection) and the resumed session carries prior conversation state. `-S <id>` is the
|
|
@@ -298,7 +275,7 @@ export const kimi = {
|
|
|
298
275
|
}
|
|
299
276
|
},
|
|
300
277
|
listModels: async () => {
|
|
301
|
-
const r =
|
|
302
|
-
return r.
|
|
278
|
+
const r = await sh("kimi provider list --json", process.cwd(), 15000);
|
|
279
|
+
return r.code !== 0 ? [] : parseKimiModels(r.stdout || "");
|
|
303
280
|
},
|
|
304
281
|
};
|
|
@@ -3,6 +3,28 @@ import { type RoutingProfile } from "../route/profile.js";
|
|
|
3
3
|
import { type AuthHealth, type BillingChannel, type WorkerAdapter } from "./types.js";
|
|
4
4
|
export declare const CANDIDATE_CLI_CATALOG: readonly ["kimi", "gemini", "qwen", "aider", "goose", "amp", "droid", "auggie", "crush"];
|
|
5
5
|
export declare const REGISTERED_ADAPTER_BINARIES: readonly ["claude", "codex", "cursor-agent", "opencode", "pi", "grok"];
|
|
6
|
+
export declare function resolveShellBinary(bin: string, cwd?: string): {
|
|
7
|
+
resolved?: string;
|
|
8
|
+
all: string[];
|
|
9
|
+
};
|
|
10
|
+
export declare function probeVersionShell(bin: string, cwd?: string): AuthHealth & {
|
|
11
|
+
binaryPaths?: string[];
|
|
12
|
+
};
|
|
13
|
+
export declare function binaryShadowWarnings(adapters: WorkerAdapter[], health: Record<string, AuthHealth>, cwd?: string, resolve?: (bin: string, cwd: string) => {
|
|
14
|
+
resolved?: string;
|
|
15
|
+
all: string[];
|
|
16
|
+
}): string[];
|
|
17
|
+
export declare function invalidConfiguredModels(cfg: TickmarkrConfig, adapterId: string, h: AuthHealth | undefined): string[];
|
|
18
|
+
export declare function modelAliasExclusions(cfg: TickmarkrConfig, adapters: WorkerAdapter[], health: Record<string, AuthHealth>): {
|
|
19
|
+
key: string;
|
|
20
|
+
adapter: string;
|
|
21
|
+
model: string;
|
|
22
|
+
}[];
|
|
23
|
+
export declare function modelAliasLine(excluded: {
|
|
24
|
+
key: string;
|
|
25
|
+
adapter: string;
|
|
26
|
+
model: string;
|
|
27
|
+
}[]): string;
|
|
6
28
|
export type CandidateCliDetection = {
|
|
7
29
|
binary: string;
|
|
8
30
|
version: string | undefined;
|
|
@@ -19,7 +41,9 @@ export declare function allAdapters(opts?: {
|
|
|
19
41
|
fakeScriptPath?: string;
|
|
20
42
|
}): WorkerAdapter[];
|
|
21
43
|
export declare function getAdapter(id: string, adapters: WorkerAdapter[]): WorkerAdapter;
|
|
22
|
-
export declare function probeAll(adapters: WorkerAdapter[]
|
|
44
|
+
export declare function probeAll(adapters: WorkerAdapter[], opts?: {
|
|
45
|
+
cwd?: string;
|
|
46
|
+
}): Promise<Record<string, AuthHealth>>;
|
|
23
47
|
export type ProbeModelStatus = "ok" | "timeout" | "failed";
|
|
24
48
|
export type ProbeModelProgress = (adapter: string, model: string, status: ProbeModelStatus, durationMs: number) => void;
|
|
25
49
|
export interface AutoPreferDoc {
|
|
@@ -18,11 +18,89 @@ import { grok } from "./grok.js";
|
|
|
18
18
|
import { kimi } from "./kimi.js";
|
|
19
19
|
import { opencode } from "./opencode.js";
|
|
20
20
|
import { pi } from "./pi.js";
|
|
21
|
-
import {
|
|
21
|
+
import { sealHerdrEnv } from "../drivers/subprocess.js";
|
|
22
|
+
import { channelKey, modelAuthed, QUOTA_RE, shq } from "./types.js";
|
|
22
23
|
// Agent-CLI binary names with no tickmarkr adapter — doctor sweeps these advisory-only (v1.48 T1).
|
|
23
24
|
export const CANDIDATE_CLI_CATALOG = ["kimi", "gemini", "qwen", "aider", "goose", "amp", "droid", "auggie", "crush"];
|
|
24
25
|
// Binaries probed by registered adapters — kept beside the catalog so a test can forbid overlap.
|
|
25
26
|
export const REGISTERED_ADAPTER_BINARIES = ["claude", "codex", "cursor-agent", "opencode", "pi", "grok"];
|
|
27
|
+
// OBS-117: worker panes run commands through `bash -lc` (SubprocessDriver + sh()), not bare spawnSync(bin).
|
|
28
|
+
// Doctor must resolve adapter binaries the same way so a shadow install on PATH cannot disagree with dispatch.
|
|
29
|
+
function shellSpawnSync(cmd, cwd, timeoutMs = 10000) {
|
|
30
|
+
const r = spawnSync("bash", ["-lc", cmd], {
|
|
31
|
+
encoding: "utf8",
|
|
32
|
+
cwd,
|
|
33
|
+
timeout: timeoutMs,
|
|
34
|
+
env: sealHerdrEnv(process.env),
|
|
35
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
36
|
+
});
|
|
37
|
+
return { code: r.status ?? 1, stdout: (r.stdout || "").trim(), stderr: (r.stderr || "").trim() };
|
|
38
|
+
}
|
|
39
|
+
export function resolveShellBinary(bin, cwd = process.cwd()) {
|
|
40
|
+
const r = shellSpawnSync(`type -a ${shq(bin)} 2>/dev/null | sed -n 's/^.* is \\(.*\\)$/\\1/p'`, cwd);
|
|
41
|
+
const all = r.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
42
|
+
return { resolved: all[0], all };
|
|
43
|
+
}
|
|
44
|
+
export function probeVersionShell(bin, cwd = process.cwd()) {
|
|
45
|
+
const { resolved, all } = resolveShellBinary(bin, cwd);
|
|
46
|
+
if (!resolved)
|
|
47
|
+
return { installed: false, authed: false, models: [] };
|
|
48
|
+
const r = shellSpawnSync(`${shq(resolved)} --version`, cwd);
|
|
49
|
+
if (r.code !== 0)
|
|
50
|
+
return { installed: false, authed: false, models: [], binaryPaths: all };
|
|
51
|
+
return {
|
|
52
|
+
installed: true,
|
|
53
|
+
authed: true,
|
|
54
|
+
version: (r.stdout || r.stderr).trim().split("\n")[0],
|
|
55
|
+
models: [],
|
|
56
|
+
note: "auth assumed; verified at dispatch (failover on auth/quota errors)",
|
|
57
|
+
binaryPaths: all,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
export function binaryShadowWarnings(adapters, health, cwd = process.cwd(), resolve) {
|
|
61
|
+
const resolveFn = resolve ?? resolveShellBinary;
|
|
62
|
+
const out = [];
|
|
63
|
+
for (const a of adapters) {
|
|
64
|
+
const bin = a.hardcodedFlags?.binary;
|
|
65
|
+
if (!bin || !health[a.id]?.installed)
|
|
66
|
+
continue;
|
|
67
|
+
const { resolved, all } = resolveFn(bin, cwd);
|
|
68
|
+
if (all.length <= 1)
|
|
69
|
+
continue;
|
|
70
|
+
out.push(`binary shadow: ${a.id} resolves to ${resolved ?? bin} but ${all.length} installs on PATH — ${all.join(", ")}`);
|
|
71
|
+
}
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
74
|
+
function modelListValidated(h) {
|
|
75
|
+
return !!h?.modelsDetectedAt && Array.isArray(h.models);
|
|
76
|
+
}
|
|
77
|
+
export function invalidConfiguredModels(cfg, adapterId, h) {
|
|
78
|
+
if (!modelListValidated(h))
|
|
79
|
+
return [];
|
|
80
|
+
const listed = new Set(h.models);
|
|
81
|
+
const configured = Object.keys(cfg.tiers[adapterId]?.models ?? {});
|
|
82
|
+
// Fail open until the CLI list intersects at least one configured model — proves the list is authoritative
|
|
83
|
+
// for this adapter's tier namespace (MODEL-04: unrelated detected ids must not invalidate routing).
|
|
84
|
+
if (!configured.some((m) => listed.has(m)))
|
|
85
|
+
return [];
|
|
86
|
+
return configured.filter((m) => !listed.has(m));
|
|
87
|
+
}
|
|
88
|
+
export function modelAliasExclusions(cfg, adapters, health) {
|
|
89
|
+
const out = [];
|
|
90
|
+
for (const a of adapters) {
|
|
91
|
+
const h = health[a.id];
|
|
92
|
+
if (!h?.installed || !h?.authed)
|
|
93
|
+
continue;
|
|
94
|
+
for (const model of invalidConfiguredModels(cfg, a.id, h)) {
|
|
95
|
+
out.push({ key: channelKey({ adapter: a.id, model }), adapter: a.id, model });
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
export function modelAliasLine(excluded) {
|
|
101
|
+
const parts = excluded.map(({ key, adapter }) => `${key} (not in ${adapter}'s reported model list)`);
|
|
102
|
+
return `model alias: ${excluded.length} channel(s) invalid — ${parts.join(", ")}`;
|
|
103
|
+
}
|
|
26
104
|
function candidateOnPath(bin, pathEnv) {
|
|
27
105
|
const r = spawnSync("which", [bin], { encoding: "utf8", env: { ...process.env, PATH: pathEnv } });
|
|
28
106
|
return r.status === 0 && r.stdout.trim().length > 0;
|
|
@@ -105,9 +183,20 @@ export function getAdapter(id, adapters) {
|
|
|
105
183
|
throw new Error(`unknown adapter ${id} (have: ${adapters.map((a) => a.id).join(", ")})`);
|
|
106
184
|
return a;
|
|
107
185
|
}
|
|
108
|
-
export async function probeAll(adapters) {
|
|
186
|
+
export async function probeAll(adapters, opts = {}) {
|
|
187
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
109
188
|
const out = {};
|
|
110
|
-
await Promise.all(adapters.map(async (a) => {
|
|
189
|
+
await Promise.all(adapters.map(async (a) => {
|
|
190
|
+
let h = await a.probe();
|
|
191
|
+
const bin = a.hardcodedFlags?.binary;
|
|
192
|
+
if (bin) {
|
|
193
|
+
const shell = probeVersionShell(bin, cwd);
|
|
194
|
+
h = { ...h, installed: shell.installed, version: shell.version ?? h.version };
|
|
195
|
+
if (!shell.installed)
|
|
196
|
+
h = { ...h, installed: false, authed: false };
|
|
197
|
+
}
|
|
198
|
+
out[a.id] = h;
|
|
199
|
+
}));
|
|
111
200
|
return out;
|
|
112
201
|
}
|
|
113
202
|
const MODEL_PROBE_PROMPT = "Reply with exactly OK and nothing else.";
|
|
@@ -317,10 +406,12 @@ export function discoverChannels(cfg, adapters, health) {
|
|
|
317
406
|
.flatMap((a) => {
|
|
318
407
|
const h = health[a.id];
|
|
319
408
|
const s = h?.servable;
|
|
409
|
+
const invalid = new Set(invalidConfiguredModels(cfg, a.id, h));
|
|
320
410
|
// T2 (2026-07-13): only a model doctor marked authed (modelAuth[model].authed===true) advertises a
|
|
321
411
|
// channel — routing an unknown or 403 into dispatch is fail-closed. Operators with pre-v1.21 doctor.json
|
|
322
412
|
// can opt into the prior unknown-is-routable behavior with routing.allowUnverifiedModels.
|
|
323
|
-
|
|
413
|
+
// OBS-117: configured aliases absent from the adapter CLI's own model list are not dispatchable.
|
|
414
|
+
return a.channels(cfg).filter((c) => !invalid.has(c.model) && modelAuthed(h, c.model, cfg.routing.allowUnverifiedModels) && (!s || s.includes(c.model)));
|
|
324
415
|
});
|
|
325
416
|
if (!cfg.routing.allow && !cfg.routing.deny)
|
|
326
417
|
return base;
|
package/dist/adapters/types.d.ts
CHANGED
|
@@ -48,10 +48,18 @@ export interface WorkerResult {
|
|
|
48
48
|
deviations: string[];
|
|
49
49
|
raw: string;
|
|
50
50
|
}
|
|
51
|
+
export type SeedBannerConfirmResult = {
|
|
52
|
+
ok: true;
|
|
53
|
+
sessionId?: string;
|
|
54
|
+
} | {
|
|
55
|
+
ok: false;
|
|
56
|
+
error: string;
|
|
57
|
+
};
|
|
51
58
|
export interface InteractiveSeed {
|
|
52
59
|
launch(model: string): string;
|
|
53
60
|
readinessMatch: string;
|
|
54
61
|
seedLine(promptFile: string): string;
|
|
62
|
+
confirmBanner?(banner: string, assignedModel: string): SeedBannerConfirmResult;
|
|
55
63
|
}
|
|
56
64
|
export interface ContextUsage {
|
|
57
65
|
tokens: number;
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { allAdapters, detectCandidateClis, flagDriftWarnings, probeAll, probeModels, readAutoPrefer, servableExclusions, servabilityLine, writeDoctor } from "../../adapters/registry.js";
|
|
3
|
+
import { allAdapters, binaryShadowWarnings, detectCandidateClis, flagDriftWarnings, modelAliasExclusions, modelAliasLine, probeAll, probeModels, readAutoPrefer, servableExclusions, servabilityLine, writeDoctor } from "../../adapters/registry.js";
|
|
4
4
|
import { BANNER, dim, fail, kvRow, legend, ok, rule, statusRow, title } from "../../brand.js";
|
|
5
5
|
import { tickmarkrDir, stateDirName } from "../../graph/graph.js";
|
|
6
6
|
import { declaredModelWindow, hasWindowsConfig, modelLints, suggestOverlay, ttyVisual } from "../../adapters/model-lints.js";
|
|
7
7
|
import { DEFAULT_CONFIG, loadConfig, overlayPreferShapes } from "../../config/config.js";
|
|
8
8
|
import { HerdrDriver } from "../../drivers/herdr.js";
|
|
9
|
-
import { disallowedBy, excludedChannels, exclusionLine, preferRanks } from "../../route/preference.js";
|
|
9
|
+
import { denyPreferCollisionLine, denyPreferCollisions, disallowedBy, excludedChannels, exclusionLine, preferRanks } from "../../route/preference.js";
|
|
10
10
|
const visual = () => process.stdout.isTTY === true && process.env.NO_COLOR === undefined;
|
|
11
11
|
const alignedStatusRow = (verdict, key, value) => ` ${statusRow(verdict, kvRow(key, value).slice(2))}`;
|
|
12
12
|
const attentionRow = (text) => ` ${statusRow("warn", text)}`;
|
|
@@ -19,7 +19,7 @@ export async function doctor(_argv, cwd = process.cwd(), adapters = allAdapters(
|
|
|
19
19
|
// otherwise prints nothing until the end — silence here reads as a hang (v1.33.1)
|
|
20
20
|
console.error("probing installed agent CLIs — one short LLM call per configured model, may take a minute...");
|
|
21
21
|
const probeProgressTTY = process.stderr.isTTY === true;
|
|
22
|
-
const health = await probeAll(adapters);
|
|
22
|
+
const health = await probeAll(adapters, { cwd });
|
|
23
23
|
// MODEL-02: detect models where the adapter exposes a list surface, BEFORE writing doctor.json (write once, below).
|
|
24
24
|
// Fail OPEN — the inverse of gates' fail-closed: detection is advisory, so a broken list surface NEVER fails doctor.
|
|
25
25
|
for (const a of adapters) {
|
|
@@ -81,6 +81,11 @@ export async function doctor(_argv, cwd = process.cwd(), adapters = allAdapters(
|
|
|
81
81
|
const excluded = excludedChannels(cfg, adapters, health);
|
|
82
82
|
if (excluded.length)
|
|
83
83
|
rows.push(attentionRow(exclusionLine(excluded)));
|
|
84
|
+
rows.push(...denyPreferCollisions(cfg).map((c) => attentionRow(denyPreferCollisionLine(c))));
|
|
85
|
+
const aliasExcluded = modelAliasExclusions(cfg, adapters, health);
|
|
86
|
+
if (aliasExcluded.length)
|
|
87
|
+
rows.push(attentionRow(modelAliasLine(aliasExcluded)));
|
|
88
|
+
rows.push(...binaryShadowWarnings(adapters, health, cwd).map(attentionRow));
|
|
84
89
|
// HYG-07(a): doctor just probed fresh (probeAll above), so servability attribution is current by construction.
|
|
85
90
|
const servable = servableExclusions(cfg, adapters, health);
|
|
86
91
|
if (servable.length)
|
|
@@ -2,6 +2,7 @@ import { loadConfig } from "../../config/config.js";
|
|
|
2
2
|
import { pickDriver } from "../../drivers/index.js";
|
|
3
3
|
import { formatSummary, runDaemon } from "../../run/daemon.js";
|
|
4
4
|
import { formatJournalNarration } from "../../run/journal.js";
|
|
5
|
+
import { denyPreferCollisionLine, denyPreferCollisions } from "../../route/preference.js";
|
|
5
6
|
const summaryGreen = (s) => s.failed.length === 0 && s.human.length === 0 && s.blocked.length === 0 && s.pending.length === 0
|
|
6
7
|
&& s.tipVerify !== "failed";
|
|
7
8
|
export async function resume(argv, cwd = process.cwd()) {
|
|
@@ -12,7 +13,12 @@ export async function resume(argv, cwd = process.cwd()) {
|
|
|
12
13
|
// Fable F2) — the daemon refuses a mismatched/unbound journal unless this is set, then journals a
|
|
13
14
|
// graph-rehash event naming both hashes. Strip the flag before runId resolution so a bare id still wins.
|
|
14
15
|
const graphChanged = argv.includes("--graph-changed");
|
|
15
|
-
const
|
|
16
|
+
const cfg = loadConfig(cwd);
|
|
17
|
+
const collisions = denyPreferCollisions(cfg);
|
|
18
|
+
if (collisions.length) {
|
|
19
|
+
throw new Error(collisions.map(denyPreferCollisionLine).join("; "));
|
|
20
|
+
}
|
|
21
|
+
const s = await runDaemon(cwd, { runId, resume: true, graphChanged, driver: pickDriver(cfg), narrate: (event) => console.log(formatJournalNarration(event)) });
|
|
16
22
|
const out = `resumed ${s.runId} — ${formatSummary(s)}`;
|
|
17
23
|
return { out, code: summaryGreen(s) ? 0 : 2 };
|
|
18
24
|
}
|
|
@@ -2,7 +2,7 @@ import { BANNER, GLYPHS, dim, fail, legend, ok, rule, statusRow, title, warn } f
|
|
|
2
2
|
import { blockedTasks, graphDefinitionHash, loadGraph } from "../../graph/graph.js";
|
|
3
3
|
import { GATE_NAMES } from "../../graph/schema.js";
|
|
4
4
|
import { foldActivity } from "../../run/activity.js";
|
|
5
|
-
import { Journal, engagementComparable } from "../../run/journal.js";
|
|
5
|
+
import { Journal, engagementComparable, isQualityFailureParkKind, recordedTaskFailureKind, runHasEnded, } from "../../run/journal.js";
|
|
6
6
|
// ponytail: fixed 2s refresh; promote to config.visibility.* only when an operator asks.
|
|
7
7
|
const REFRESH_MS = 2000;
|
|
8
8
|
const NOT_COMPARABLE_NOTICE = "graph recompiled since this run — task states not comparable; run `tickmarkr run` to execute";
|
|
@@ -178,15 +178,23 @@ const renderFrame = (cwd) => {
|
|
|
178
178
|
const divider = unicode ? " · " : " / ";
|
|
179
179
|
const width = process.stdout.columns ?? 120;
|
|
180
180
|
const done = effective.tasks.filter((t) => t.status === "done").length;
|
|
181
|
+
const ended = comparable && runHasEnded(events);
|
|
181
182
|
const cells = g.tasks.map((t) => {
|
|
182
183
|
const st = replayed?.get(t.id) ?? t.status;
|
|
184
|
+
const failureKind = comparable ? recordedTaskFailureKind(events, t.id) : undefined;
|
|
185
|
+
const terminal = st === "failed" || st === "human";
|
|
186
|
+
// Unknown legacy task-failed events stay red. Typed availability noise is warn-tier only while
|
|
187
|
+
// the run is live; verified quality parks and every unresolved task in an ended run are red.
|
|
188
|
+
const redTier = terminal && (ended
|
|
189
|
+
|| (failureKind !== undefined && isQualityFailureParkKind(failureKind))
|
|
190
|
+
|| (st === "failed" && failureKind === undefined));
|
|
183
191
|
const isStarved = starved.has(t.id);
|
|
184
192
|
const phrase = isStarved ? undefined : activity.cells.get(t.id);
|
|
185
193
|
const label = isStarved ? " starved" : phrase ? ` ${phrase}` : "";
|
|
186
194
|
const channel = assignments.get(t.id) ?? "-";
|
|
187
195
|
const ctx = contexts.get(t.id);
|
|
188
196
|
const assignCol = ctx !== undefined ? `${channel}${divider}ctx ${ctx}` : channel;
|
|
189
|
-
return { t, st, label, assignCol, isStarved, phrase, channel, ctx, states: comparable ? gateStates(t, events) : defaultGateStates(t) };
|
|
197
|
+
return { t, st, failureKind, redTier, label, assignCol, isStarved, phrase, channel, ctx, states: comparable ? gateStates(t, events) : defaultGateStates(t) };
|
|
190
198
|
});
|
|
191
199
|
if (!unicode) {
|
|
192
200
|
// machine/CI surface — layout unchanged (pipes, greps, and the golden pins depend on it)
|
|
@@ -207,7 +215,7 @@ const renderFrame = (cwd) => {
|
|
|
207
215
|
// rows (pad plain text FIRST, colorize after: ANSI has zero display width and would corrupt
|
|
208
216
|
// padEnd math)
|
|
209
217
|
const dot = dim(" · ");
|
|
210
|
-
const anyFailed = cells.some((c) => c.
|
|
218
|
+
const anyFailed = cells.some((c) => c.redTier);
|
|
211
219
|
const gaugeCells = 10;
|
|
212
220
|
const fill = g.tasks.length ? Math.round((done / g.tasks.length) * gaugeCells) : 0;
|
|
213
221
|
const gauge = (fill ? (anyFailed ? fail : ok)("█".repeat(fill)) : "") + (fill < gaugeCells ? dim("░".repeat(gaugeCells - fill)) : "");
|
|
@@ -232,17 +240,20 @@ const renderFrame = (cwd) => {
|
|
|
232
240
|
// id, goal at full width, and only SHORT status words. Line 2 carries the machinery, dim and
|
|
233
241
|
// aligned under the goal: gate chain, live activity phrase (or channel), ctx. Long channel names
|
|
234
242
|
// and activity phrases live on line 2 only, so they can never squeeze the goal or wrap line 1.
|
|
235
|
-
const taskVerdict = (
|
|
243
|
+
const taskVerdict = (c) => c.st === "done" ? "pass" : c.redTier ? "fail" : c.st === "failed" || c.st === "human" ? "warn" : "neutral";
|
|
244
|
+
const statusWord = (c) => c.redTier ? "failed" : c.st === "failed" ? "warn" : String(c.st);
|
|
236
245
|
const idW = Math.max(...cells.map((c) => c.t.id.length), 2);
|
|
237
246
|
// plain-text status suffixes for width math only — starved / failed gate names / approval hint
|
|
238
247
|
const suffixPlain = (c) => (c.isStarved ? " · starved" : "") + failedSuffix(c.states) + humanGateSuffix(c.t, c.st, c.states);
|
|
239
|
-
const stW = Math.max(...cells.map((c) =>
|
|
248
|
+
const stW = Math.max(...cells.map((c) => statusWord(c).length + suffixPlain(c).length));
|
|
240
249
|
const avail = Math.max(8, width - (5 + idW) - 2 - stW);
|
|
241
250
|
const goals = cells.map((c) => shortGoal(c.t.goal, avail));
|
|
242
251
|
const goalW = Math.max(8, ...goals.map((s) => s.length));
|
|
243
252
|
const indent = " ".repeat(idW + 5); // line 2 starts under the goal column
|
|
244
|
-
const rows = cells.map((
|
|
245
|
-
const
|
|
253
|
+
const rows = cells.map((c, i) => {
|
|
254
|
+
const { t, st, failureKind, redTier, states, isStarved, phrase, channel, ctx } = c;
|
|
255
|
+
const word = statusWord(c);
|
|
256
|
+
const stWord = st === "done" ? ok(word) : redTier ? fail(word) : st === "failed" || st === "human" ? warn(word) : word;
|
|
246
257
|
// a fail names its gate in words right here — the one moment gate identity is needed on a row
|
|
247
258
|
const f = failedGates(states);
|
|
248
259
|
const human = humanGateSuffix(t, st, states);
|
|
@@ -250,10 +261,11 @@ const renderFrame = (cwd) => {
|
|
|
250
261
|
(isStarved ? dot + fail("starved") : "") +
|
|
251
262
|
(f.length ? dot + fail(f.join(", ")) : "") +
|
|
252
263
|
(human ? dot + warn("awaiting approval") : "");
|
|
253
|
-
const line1 = ` ${statusRow(taskVerdict(
|
|
264
|
+
const line1 = ` ${statusRow(taskVerdict(c), `${t.id.padEnd(idW)} ${goals[i].padEnd(goalW)} ${statusCell}`)}`;
|
|
254
265
|
// activity already names its channel for in-flight attempts — never repeat it
|
|
255
266
|
const detail = [
|
|
256
267
|
...(phrase ? [phrase, ...(phrase.includes(channel) || channel === "-" ? [] : [channel])] : [channel]),
|
|
268
|
+
...(failureKind && !phrase?.includes(failureKind) ? [failureKind] : []),
|
|
257
269
|
...(ctx !== undefined ? [`ctx ${ctx}`] : []),
|
|
258
270
|
].join(" · ");
|
|
259
271
|
return [line1, `${indent}${gateChain(states, true)} ${dim(detail)}`, ""];
|
package/dist/drivers/herdr.d.ts
CHANGED
|
@@ -11,11 +11,14 @@ export declare class HerdrDriver implements ExecutorDriver {
|
|
|
11
11
|
interactive: boolean;
|
|
12
12
|
private groups;
|
|
13
13
|
private groupSerial;
|
|
14
|
+
private deliverySerial;
|
|
14
15
|
private ws;
|
|
15
16
|
private callerPane;
|
|
16
17
|
private watches;
|
|
17
18
|
constructor(bin?: string, workersPerTab?: number);
|
|
18
19
|
private serial;
|
|
20
|
+
private deliveryQueue;
|
|
21
|
+
private deliveryMatches;
|
|
19
22
|
static available(): boolean;
|
|
20
23
|
private herdr;
|
|
21
24
|
private namedPaneId;
|
|
@@ -29,6 +32,8 @@ export declare class HerdrDriver implements ExecutorDriver {
|
|
|
29
32
|
private glyphFor;
|
|
30
33
|
private joinGroup;
|
|
31
34
|
run(slot: Slot, cmd: string): Promise<void>;
|
|
35
|
+
private deliver;
|
|
36
|
+
private deliveryReadMatches;
|
|
32
37
|
private waitOk;
|
|
33
38
|
waitOutput(slot: Slot, pattern: string, timeoutMs: number, opts?: {
|
|
34
39
|
regex?: boolean;
|
package/dist/drivers/herdr.js
CHANGED
|
@@ -25,6 +25,8 @@ export class HerdrDriver {
|
|
|
25
25
|
// grouped slot()/close() mutate shared group state across awaits — serialize them so two
|
|
26
26
|
// concurrent first members can never both create the group tab (mergeSerial idiom, daemon.ts)
|
|
27
27
|
groupSerial = Promise.resolve();
|
|
28
|
+
// OBS-119: concurrent deliveries contend on herdr's send path — one chain per driver (mergeSerial idiom)
|
|
29
|
+
deliverySerial = Promise.resolve();
|
|
28
30
|
// VIS-10: the run's workspace id, captured once at construction (the daemon inherits it from the
|
|
29
31
|
// operator's env before the driver is built). Required at slot() time, never in the constructor —
|
|
30
32
|
// pickDriver and its unit test construct HerdrDriver without env, so slot() is the trust gate.
|
|
@@ -40,6 +42,18 @@ export class HerdrDriver {
|
|
|
40
42
|
this.groupSerial = p.catch(() => undefined);
|
|
41
43
|
return p;
|
|
42
44
|
}
|
|
45
|
+
deliveryQueue(fn) {
|
|
46
|
+
const p = this.deliverySerial.then(fn, fn);
|
|
47
|
+
this.deliverySerial = p.catch(() => undefined);
|
|
48
|
+
return p;
|
|
49
|
+
}
|
|
50
|
+
// ponytail: narrow panes hard-wrap the input line — collapse whitespace before comparing.
|
|
51
|
+
deliveryMatches(transcript, cmd) {
|
|
52
|
+
const norm = (s) => s.replace(/\s+/g, "");
|
|
53
|
+
const hay = norm(transcript);
|
|
54
|
+
const needle = norm(cmd);
|
|
55
|
+
return needle.length > 0 && hay.includes(needle);
|
|
56
|
+
}
|
|
43
57
|
static available() {
|
|
44
58
|
return process.env.HERDR_ENV === "1";
|
|
45
59
|
}
|
|
@@ -286,6 +300,9 @@ export class HerdrDriver {
|
|
|
286
300
|
// cleared (C-u), and retyped, bounded; persistent corruption fails closed WITH the captured
|
|
287
301
|
// transcript — the dispatch-time pincer the ledger asks for, not post-hoc `git:` archaeology.
|
|
288
302
|
async run(slot, cmd) {
|
|
303
|
+
return this.deliveryQueue(() => this.deliver(slot, cmd));
|
|
304
|
+
}
|
|
305
|
+
async deliver(slot, cmd) {
|
|
289
306
|
const pane = await this.paneId(slot);
|
|
290
307
|
let transcript = "";
|
|
291
308
|
for (let attempt = 0; attempt < DELIVERY_ATTEMPTS; attempt++) {
|
|
@@ -300,9 +317,7 @@ export class HerdrDriver {
|
|
|
300
317
|
if (typed.code !== 0)
|
|
301
318
|
throw new Error(`herdr pane send-text failed: ${typed.stderr || typed.stdout}`);
|
|
302
319
|
const back = await this.herdr(`wait output ${shq(pane)} --match ${shq(cmd)} --timeout ${DELIVERY_VERIFY_TIMEOUT_MS}`, slot.cwd, DELIVERY_VERIFY_TIMEOUT_MS + 15_000);
|
|
303
|
-
|
|
304
|
-
// prior echo of the same command ever false-verifies. Dead pane → error envelope → unverified.
|
|
305
|
-
if (this.waitOk(back.code, back.stdout)) {
|
|
320
|
+
if (this.waitOk(back.code, back.stdout) || await this.deliveryReadMatches(pane, cmd, slot.cwd)) {
|
|
306
321
|
const enter = await this.herdr(`pane send-keys ${shq(pane)} Enter`, slot.cwd);
|
|
307
322
|
if (enter.code !== 0)
|
|
308
323
|
throw new Error(`herdr pane send-keys Enter failed: ${enter.stderr || enter.stdout}`);
|
|
@@ -313,6 +328,10 @@ export class HerdrDriver {
|
|
|
313
328
|
}
|
|
314
329
|
throw new Error(`herdr delivery corrupted after ${DELIVERY_ATTEMPTS} attempts — enter never pressed (OBS-85); pane transcript:\n${transcript}`);
|
|
315
330
|
}
|
|
331
|
+
async deliveryReadMatches(pane, cmd, cwd) {
|
|
332
|
+
const read = await this.herdr(`pane read ${shq(pane)} --source recent-unwrapped --lines ${DELIVERY_READ_LINES}`, cwd);
|
|
333
|
+
return read.code === 0 && this.deliveryMatches(read.stdout, cmd);
|
|
334
|
+
}
|
|
316
335
|
waitOk(code, stdout) {
|
|
317
336
|
if (code !== 0 || !stdout.trim())
|
|
318
337
|
return code === 0; // herdr's successful waits may be silent
|
|
@@ -25,3 +25,12 @@ export declare function disallowedBy(c: {
|
|
|
25
25
|
adapter: string;
|
|
26
26
|
model: string;
|
|
27
27
|
}, routing: TickmarkrConfig["routing"]): Disallowed | null;
|
|
28
|
+
export interface DenyPreferCollision {
|
|
29
|
+
kind: "prefer" | "pin";
|
|
30
|
+
shape: string;
|
|
31
|
+
detail: string;
|
|
32
|
+
disallowed: Disallowed;
|
|
33
|
+
}
|
|
34
|
+
export declare function preferEntryDenied(p: string, cfg: TickmarkrConfig): Disallowed | null;
|
|
35
|
+
export declare function denyPreferCollisions(cfg: TickmarkrConfig): DenyPreferCollision[];
|
|
36
|
+
export declare function denyPreferCollisionLine({ kind, shape, detail, disallowed }: DenyPreferCollision): string;
|
package/dist/route/preference.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { channelKey, channelsFromConfig } from "../adapters/types.js";
|
|
2
|
+
import { validateGraph } from "../graph/schema.js";
|
|
3
|
+
import { route, RoutingError } from "./router.js";
|
|
2
4
|
const adapterIds = (adapters) => typeof adapters[0] === "string" ? adapters : adapters.map((a) => a.id);
|
|
3
5
|
export function excludedChannels(cfg, adapters, health) {
|
|
4
6
|
const { allow, deny } = cfg.routing;
|
|
@@ -47,3 +49,62 @@ export function disallowedBy(c, routing) {
|
|
|
47
49
|
}
|
|
48
50
|
return null;
|
|
49
51
|
}
|
|
52
|
+
// Delegates to route()'s preflightPrefer — doctor/resume preflight reuses the router grammar,
|
|
53
|
+
// not a second colon/tier/model parser of the same config.
|
|
54
|
+
const PREFLIGHT_SHAPE = "chore";
|
|
55
|
+
const preflightTask = validateGraph({
|
|
56
|
+
version: 1, spec: { source: "prd", paths: ["p"], hash: "h" },
|
|
57
|
+
tasks: [{ id: "T0", title: "preflight", goal: "preflight", shape: PREFLIGHT_SHAPE, complexity: 1, acceptance: ["preflight"] }],
|
|
58
|
+
}).tasks[0];
|
|
59
|
+
const disallowedFromPreferError = (msg) => {
|
|
60
|
+
const m = msg.match(/prefer entry .+ is disallowed by routing\.(deny|allow) \(([^)]+)\)/);
|
|
61
|
+
return m ? { by: m[1], entry: m[2] } : null;
|
|
62
|
+
};
|
|
63
|
+
export function preferEntryDenied(p, cfg) {
|
|
64
|
+
if (!cfg.routing.allow && !cfg.routing.deny)
|
|
65
|
+
return null;
|
|
66
|
+
const probe = structuredClone(cfg);
|
|
67
|
+
probe.routing.map = { ...probe.routing.map, [PREFLIGHT_SHAPE]: { prefer: [p] } };
|
|
68
|
+
try {
|
|
69
|
+
route(preflightTask, probe, []);
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
if (e instanceof RoutingError)
|
|
74
|
+
return disallowedFromPreferError(e.message);
|
|
75
|
+
throw e;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
export function denyPreferCollisions(cfg) {
|
|
79
|
+
if (!cfg.routing.allow && !cfg.routing.deny)
|
|
80
|
+
return [];
|
|
81
|
+
const out = [];
|
|
82
|
+
for (const [shape, entry] of Object.entries(cfg.routing.map)) {
|
|
83
|
+
if (entry.pin) {
|
|
84
|
+
const d = disallowedBy({ adapter: entry.pin.via, model: entry.pin.model }, cfg.routing);
|
|
85
|
+
if (d) {
|
|
86
|
+
out.push({
|
|
87
|
+
kind: "pin",
|
|
88
|
+
shape,
|
|
89
|
+
detail: `${entry.pin.via}:${entry.pin.model}`,
|
|
90
|
+
disallowed: d,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const prefer = entry.prefer ?? [];
|
|
95
|
+
if (prefer.length && prefer.every((p) => preferEntryDenied(p, cfg) !== null)) {
|
|
96
|
+
out.push({
|
|
97
|
+
kind: "prefer",
|
|
98
|
+
shape,
|
|
99
|
+
detail: prefer.join(" > "),
|
|
100
|
+
disallowed: preferEntryDenied(prefer[0], cfg),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
export function denyPreferCollisionLine({ kind, shape, detail, disallowed }) {
|
|
107
|
+
const surface = kind === "pin" ? `routing.map.${shape}.pin` : `routing.map.${shape}.prefer`;
|
|
108
|
+
const verb = kind === "pin" ? "is disallowed" : "fully disallowed";
|
|
109
|
+
return `deny∩prefer: ${surface} ${detail} ${verb} by routing.${disallowed.by} (${disallowed.entry}) — remove the ${disallowed.by} entry or adjust ${kind === "pin" ? "pin" : "prefer"}`;
|
|
110
|
+
}
|
package/dist/run/daemon.d.ts
CHANGED
|
@@ -41,4 +41,8 @@ export interface RunSummary {
|
|
|
41
41
|
lastMergedTask?: string;
|
|
42
42
|
}
|
|
43
43
|
export declare function formatSummary(s: RunSummary): string;
|
|
44
|
+
export declare const EARLY_LAUNCH_LIVENESS_MS = 60000;
|
|
45
|
+
/** Test seam — lowers the empty-pane liveness window without sleeping 60s per case. */
|
|
46
|
+
export declare function setEarlyLaunchLivenessMsForTests(ms: number): void;
|
|
47
|
+
export declare function resetEarlyLaunchLivenessMsForTests(): void;
|
|
44
48
|
export declare function runDaemon(repoRoot: string, opts?: RunOptions): Promise<RunSummary>;
|
package/dist/run/daemon.js
CHANGED
|
@@ -74,6 +74,18 @@ const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether th
|
|
|
74
74
|
const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
|
|
75
75
|
const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
|
|
76
76
|
const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
|
|
77
|
+
// OBS-117 (v1.71 T6): a worker pane that never prints a byte by T+60s after dispatch is a dead
|
|
78
|
+
// channel — don't burn the full stall window waiting for a silent launch failure. Checked on the
|
|
79
|
+
// existing stall-wait poll cadence only (no new timer). Spinner/ANSI repaints count as output.
|
|
80
|
+
export const EARLY_LAUNCH_LIVENESS_MS = 60_000;
|
|
81
|
+
let earlyLaunchLivenessMs = EARLY_LAUNCH_LIVENESS_MS;
|
|
82
|
+
/** Test seam — lowers the empty-pane liveness window without sleeping 60s per case. */
|
|
83
|
+
export function setEarlyLaunchLivenessMsForTests(ms) {
|
|
84
|
+
earlyLaunchLivenessMs = ms;
|
|
85
|
+
}
|
|
86
|
+
export function resetEarlyLaunchLivenessMsForTests() {
|
|
87
|
+
earlyLaunchLivenessMs = EARLY_LAUNCH_LIVENESS_MS;
|
|
88
|
+
}
|
|
77
89
|
async function commitsAheadOf(base, wt) {
|
|
78
90
|
const head = await gitHead(wt);
|
|
79
91
|
if (head === base)
|
|
@@ -217,6 +229,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
217
229
|
// GATE-08 lesson at the humanGate guard: condition on the data, not the code path). Dead-code
|
|
218
230
|
// equivalence to the router.ts:194 profile⇒undefined pattern: no map entry ⇒ today's literal.
|
|
219
231
|
const resume = opts.resume ? journal.replayResumeState() : new Map();
|
|
232
|
+
const replayedExclusions = opts.resume ? journal.replayExcludedChannels() : new Set();
|
|
220
233
|
if (opts.resume) {
|
|
221
234
|
// v1.53 T5: a superseded run is dead — resuming it beside its successor is the exact
|
|
222
235
|
// two-concurrent-runs hazard supersession exists to prevent. Fail closed, naming the successor.
|
|
@@ -252,7 +265,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
252
265
|
continue;
|
|
253
266
|
graph = setStatus(graph, id, st);
|
|
254
267
|
}
|
|
255
|
-
journal.append("run-resume", undefined, {
|
|
268
|
+
journal.append("run-resume", undefined, {
|
|
269
|
+
pid: process.pid, // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
270
|
+
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
271
|
+
});
|
|
256
272
|
}
|
|
257
273
|
else {
|
|
258
274
|
baseRef = await gitHead(repoRoot);
|
|
@@ -301,7 +317,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
301
317
|
const keptSlots = [];
|
|
302
318
|
const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
|
|
303
319
|
// OBS-57: per-run in-run demotion — channels that burn consecutive no-trailer windows route around for later attempts.
|
|
304
|
-
|
|
320
|
+
// v1.71 OBS-119: on resume, re-seed from the journal fold (replayedExclusions) before any dispatch.
|
|
321
|
+
const demotedChannels = new Set(replayedExclusions);
|
|
305
322
|
const noTrailerStreak = new Map();
|
|
306
323
|
// OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
|
|
307
324
|
// terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
|
|
@@ -394,7 +411,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
394
411
|
// fresh-budget release, prefer nextChannel over the surviving tried-list so burned channels are
|
|
395
412
|
// not re-tried first (consult bans / prior failovers survive the release).
|
|
396
413
|
const rs = resume.get(t.id);
|
|
397
|
-
if (rs?.lastAssignment && rs.attempts > 0
|
|
414
|
+
if (rs?.lastAssignment && rs.attempts > 0
|
|
415
|
+
&& channels.some((c) => channelKey(c) === channelKey(rs.lastAssignment))
|
|
416
|
+
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
398
417
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
399
418
|
}
|
|
400
419
|
else if (rs && rs.tried.length) {
|
|
@@ -690,6 +709,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
690
709
|
let output;
|
|
691
710
|
let exitCode;
|
|
692
711
|
let timedOut = false;
|
|
712
|
+
let earlyLaunchDead = false;
|
|
693
713
|
let settleParsed;
|
|
694
714
|
let seedResult;
|
|
695
715
|
if (interactive) {
|
|
@@ -724,12 +744,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
724
744
|
// OBS-82: the stall clock compares NORMALIZED snapshots so a spinner glyph/elapsed-time
|
|
725
745
|
// repaint is silence, not activity. ONLY this inactivity compare sees normalized text —
|
|
726
746
|
// trailer detection, harvest, paging, and quota checks all read the raw pane.
|
|
747
|
+
let everHadOutput = output.length > 0;
|
|
727
748
|
let lastStallSnapshot = normalizeStallSnapshot(output);
|
|
728
749
|
let lastOutputAt = Date.now();
|
|
729
750
|
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
730
751
|
const sliceStart = Date.now();
|
|
731
752
|
const remaining = stallWindowMs - (sliceStart - lastOutputAt);
|
|
732
|
-
|
|
753
|
+
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
754
|
+
if (!everHadOutput) {
|
|
755
|
+
const earlyLeft = earlyLaunchLivenessMs - (sliceStart - attemptStart);
|
|
756
|
+
if (earlyLeft > 0)
|
|
757
|
+
slice = Math.min(slice, earlyLeft);
|
|
758
|
+
}
|
|
733
759
|
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
734
760
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
735
761
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
@@ -743,11 +769,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
743
769
|
break;
|
|
744
770
|
}
|
|
745
771
|
}
|
|
746
|
-
const
|
|
772
|
+
const paneText = await driver.read(slot, 1000);
|
|
773
|
+
if (paneText.length > 0)
|
|
774
|
+
everHadOutput = true;
|
|
775
|
+
const currentStallSnapshot = normalizeStallSnapshot(paneText);
|
|
747
776
|
if (currentStallSnapshot !== lastStallSnapshot) {
|
|
748
777
|
lastStallSnapshot = currentStallSnapshot;
|
|
749
778
|
lastOutputAt = Date.now();
|
|
750
779
|
}
|
|
780
|
+
// OBS-117 (v1.71 T6): zero raw output by the early-launch deadline is a dead channel now.
|
|
781
|
+
if (!everHadOutput && Date.now() - attemptStart >= earlyLaunchLivenessMs) {
|
|
782
|
+
earlyLaunchDead = true;
|
|
783
|
+
output = paneText;
|
|
784
|
+
break;
|
|
785
|
+
}
|
|
751
786
|
// v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
|
|
752
787
|
await sampleContext();
|
|
753
788
|
// page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
|
|
@@ -828,21 +863,35 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
828
863
|
// OBS-82: same normalized-snapshot compare as the interactive site — spinner-only repaints
|
|
829
864
|
// exhaust the budget here too; harvest below still reads the raw pane.
|
|
830
865
|
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
831
|
-
|
|
866
|
+
const initialPane = await driver.read(slot, 500);
|
|
867
|
+
let everHadOutput = initialPane.length > 0;
|
|
868
|
+
let lastStallSnapshot = normalizeStallSnapshot(initialPane);
|
|
832
869
|
let lastOutputAt = Date.now();
|
|
833
870
|
finished = false;
|
|
834
871
|
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
835
872
|
const remaining = stallWindowMs - (Date.now() - lastOutputAt);
|
|
836
|
-
|
|
873
|
+
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
874
|
+
if (!everHadOutput) {
|
|
875
|
+
const earlyLeft = earlyLaunchLivenessMs - (Date.now() - attemptStart);
|
|
876
|
+
if (earlyLeft > 0)
|
|
877
|
+
slice = Math.min(slice, earlyLeft);
|
|
878
|
+
}
|
|
837
879
|
if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
838
880
|
finished = true;
|
|
839
881
|
break;
|
|
840
882
|
}
|
|
841
|
-
const
|
|
883
|
+
const paneText = await driver.read(slot, 500);
|
|
884
|
+
if (paneText.length > 0)
|
|
885
|
+
everHadOutput = true;
|
|
886
|
+
const currentStallSnapshot = normalizeStallSnapshot(paneText);
|
|
842
887
|
if (currentStallSnapshot !== lastStallSnapshot) {
|
|
843
888
|
lastStallSnapshot = currentStallSnapshot;
|
|
844
889
|
lastOutputAt = Date.now();
|
|
845
890
|
}
|
|
891
|
+
if (!everHadOutput && Date.now() - attemptStart >= earlyLaunchLivenessMs) {
|
|
892
|
+
earlyLaunchDead = true;
|
|
893
|
+
break;
|
|
894
|
+
}
|
|
846
895
|
}
|
|
847
896
|
output = await driver.read(slot, 500);
|
|
848
897
|
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
@@ -940,10 +989,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
940
989
|
// AFTER the quota check so quota behavior stays byte-identical (a quota hit returns/continues
|
|
941
990
|
// before reaching here); provider-outage lands here only once the v1.46 same-channel requeue
|
|
942
991
|
// cap above is spent, so a transient blip still recovers in place.
|
|
943
|
-
|
|
992
|
+
// OBS-117 (v1.71 T6): a silent launch failure has no CLI signature to parse — the same
|
|
993
|
+
// setup-required typed dead-channel path a late-harvest "command not found" would take.
|
|
994
|
+
const dead = classifyDeadChannel(result) ?? (earlyLaunchDead ? "setup-required" : undefined);
|
|
944
995
|
if (dead) {
|
|
945
996
|
const from = channelKey(assignment);
|
|
946
997
|
demotedChannels.add(from); // excluded for later attempts AND later tasks in this run
|
|
998
|
+
journal.append("channel-exclusion", t.id, { channel: from, reason: dead, kind: "dead-channel" });
|
|
947
999
|
const next = failover("dead-channel");
|
|
948
1000
|
journal.append("dead-channel-failover", t.id, { reason: dead, from, to: next ? channelKey(next) : null });
|
|
949
1001
|
if (next) {
|
|
@@ -1130,9 +1182,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1130
1182
|
for (const t of ready) {
|
|
1131
1183
|
const p = execTask(t)
|
|
1132
1184
|
.catch(async (err) => {
|
|
1185
|
+
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
1186
|
+
const dispatch = [...taskEvents].reverse().find((e) => e.event === "task-dispatch");
|
|
1187
|
+
const verifiedFailure = taskEvents.some((e) => e.event === "gate-result" && e.data.pass === false);
|
|
1188
|
+
const kind = verifiedFailure ? "gate-fail" : dispatch ? "dispatch" : "infra";
|
|
1189
|
+
const attempts = dispatch && Number.isInteger(dispatch.data.attempt) ? dispatch.data.attempt : 0;
|
|
1133
1190
|
graph = setStatus(graph, t.id, "failed");
|
|
1134
1191
|
saveGraph(repoRoot, graph);
|
|
1135
|
-
journal.append("task-failed", t.id, { error: String(err) });
|
|
1192
|
+
journal.append("task-failed", t.id, { error: String(err), kind, attempts });
|
|
1136
1193
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
1137
1194
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
1138
1195
|
})
|
|
@@ -6,9 +6,17 @@ export async function runInteractiveSeed(opts) {
|
|
|
6
6
|
const seed = opts.adapter.interactiveSeed;
|
|
7
7
|
await opts.driver.run(opts.slot, seed.launch(opts.assignment.model));
|
|
8
8
|
const ready = await opts.driver.waitOutput(opts.slot, seed.readinessMatch, opts.taskTimeoutMinutes * 60_000);
|
|
9
|
+
const banner = await opts.driver.read(opts.slot, 1000);
|
|
9
10
|
if (!ready) {
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
return { output: banner, seedFailed: true, seedError: `readiness pattern not seen: ${seed.readinessMatch}` };
|
|
12
|
+
}
|
|
13
|
+
let sessionId;
|
|
14
|
+
if (seed.confirmBanner) {
|
|
15
|
+
const confirm = seed.confirmBanner(banner, opts.assignment.model);
|
|
16
|
+
if (!confirm.ok) {
|
|
17
|
+
return { output: banner, seedFailed: true, seedError: confirm.error };
|
|
18
|
+
}
|
|
19
|
+
sessionId = confirm.sessionId;
|
|
12
20
|
}
|
|
13
21
|
const seedText = seed.seedLine(opts.promptFile);
|
|
14
22
|
await opts.driver.run(opts.slot, seedText);
|
|
@@ -18,8 +26,8 @@ export async function runInteractiveSeed(opts) {
|
|
|
18
26
|
output = await opts.driver.read(opts.slot, 1000);
|
|
19
27
|
const bottom = output.trimEnd().split("\n").pop() ?? "";
|
|
20
28
|
if (!bottom.includes(seedText)) {
|
|
21
|
-
return { output, seedFailed: false };
|
|
29
|
+
return { output, seedFailed: false, ...(sessionId ? { sessionId } : {}) };
|
|
22
30
|
}
|
|
23
31
|
}
|
|
24
|
-
return { output, seedFailed: true, seedError: "seed line never left the input box" };
|
|
32
|
+
return { output, seedFailed: true, seedError: "seed line never left the input box", ...(sessionId ? { sessionId } : {}) };
|
|
25
33
|
}
|
package/dist/run/journal.d.ts
CHANGED
|
@@ -15,13 +15,18 @@ export interface ResumeState {
|
|
|
15
15
|
tried: string[];
|
|
16
16
|
lastAssignment?: Assignment;
|
|
17
17
|
}
|
|
18
|
+
export declare const CHANNEL_EXCLUSION_KINDS: readonly ["dead-channel"];
|
|
19
|
+
export type ChannelExclusionKind = (typeof CHANNEL_EXCLUSION_KINDS)[number];
|
|
18
20
|
export declare const ATTEMPT_CAP_RELEASE: "attempt-cap";
|
|
19
|
-
export declare const PARK_KINDS: readonly ["human-gate", "ladder-exhausted", "attempt-cap", "gate-fail", "quota", "reroute-exhausted", "setup", "stall", "merge-conflict", "tip-moved"];
|
|
21
|
+
export declare const PARK_KINDS: readonly ["human-gate", "ladder-exhausted", "attempt-cap", "gate-fail", "quota", "reroute-exhausted", "setup", "stall", "merge-conflict", "tip-moved", "infra", "dispatch"];
|
|
20
22
|
export type ParkKind = (typeof PARK_KINDS)[number];
|
|
21
23
|
export declare const RETRY_MODES: readonly ["resume", "fresh"];
|
|
22
24
|
export type RetryMode = (typeof RETRY_MODES)[number];
|
|
23
25
|
export declare const WORKER_RESULT_CAUSES: readonly ["provider-death", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer"];
|
|
24
26
|
export type WorkerResultCause = (typeof WORKER_RESULT_CAUSES)[number];
|
|
27
|
+
export declare function isQualityFailureParkKind(kind: ParkKind): boolean;
|
|
28
|
+
export declare function recordedTaskFailureKind(events: JournalEvent[], taskId: string): ParkKind | undefined;
|
|
29
|
+
export declare function runHasEnded(events: JournalEvent[]): boolean;
|
|
25
30
|
/** OBS-53: classify worker-result failures so retries and routing see the true signal, not one lumped bucket. */
|
|
26
31
|
export declare function classifyWorkerResultCause(opts: {
|
|
27
32
|
output: string;
|
|
@@ -58,6 +63,8 @@ export declare const TelemetryRowSchema: z.ZodObject<{
|
|
|
58
63
|
stall: "stall";
|
|
59
64
|
"merge-conflict": "merge-conflict";
|
|
60
65
|
"tip-moved": "tip-moved";
|
|
66
|
+
infra: "infra";
|
|
67
|
+
dispatch: "dispatch";
|
|
61
68
|
}>>;
|
|
62
69
|
tokens: z.ZodCatch<z.ZodOptional<z.ZodObject<{
|
|
63
70
|
input: z.ZodNumber;
|
|
@@ -139,6 +146,7 @@ export declare class Journal {
|
|
|
139
146
|
read(): JournalEvent[];
|
|
140
147
|
replayStatuses(): Map<string, TaskStatus>;
|
|
141
148
|
replayResumeState(): Map<string, ResumeState>;
|
|
149
|
+
replayExcludedChannels(): Set<string>;
|
|
142
150
|
telemetry(row: TelemetryRow): void;
|
|
143
151
|
readTelemetry(): TelemetryRow[];
|
|
144
152
|
}
|
package/dist/run/journal.js
CHANGED
|
@@ -4,7 +4,7 @@ import { z } from "zod";
|
|
|
4
4
|
import { channelKey, TokenUsageSchema } from "../adapters/types.js";
|
|
5
5
|
import { stateDirName, tickmarkrDir } from "../graph/graph.js";
|
|
6
6
|
import { TIERS } from "../graph/schema.js";
|
|
7
|
-
import { buildProfile } from "../route/profile.js";
|
|
7
|
+
import { buildProfile, classify } from "../route/profile.js";
|
|
8
8
|
import { redactSecrets } from "./redact.js";
|
|
9
9
|
export function formatJournalNarration({ event, taskId, data }) {
|
|
10
10
|
const assignment = data.assignment;
|
|
@@ -26,6 +26,9 @@ export function formatJournalNarration({ event, taskId, data }) {
|
|
|
26
26
|
: String(direct);
|
|
27
27
|
return [event, taskId, detail?.replace(/\s+/g, " ").slice(0, 120)].filter(Boolean).join(" — ");
|
|
28
28
|
}
|
|
29
|
+
// v1.71 OBS-119: run-wide channel exclusions derived from journal events — companion to
|
|
30
|
+
// replayResumeState(), consumed by the daemon on resume to re-seed demotedChannels.
|
|
31
|
+
export const CHANNEL_EXCLUSION_KINDS = ["dead-channel"];
|
|
29
32
|
// v1.24 OBS-18: explicit data on task-approved when the operator releases an attempt-cap park.
|
|
30
33
|
// Approve stamps this; replayResumeState zeros the attempt counter (fresh budget) while keeping
|
|
31
34
|
// tried (consult bans / burned channels). Absent on pre-v1.24 events ⇒ inert (corpus outcome-identical).
|
|
@@ -40,9 +43,43 @@ const DispatchAssignmentSchema = z.object({
|
|
|
40
43
|
tier: z.enum(TIERS),
|
|
41
44
|
});
|
|
42
45
|
export const PARK_KINDS = ["human-gate", "ladder-exhausted", "attempt-cap", "gate-fail", "quota",
|
|
43
|
-
"reroute-exhausted", "setup", "stall", "merge-conflict", "tip-moved"];
|
|
46
|
+
"reroute-exhausted", "setup", "stall", "merge-conflict", "tip-moved", "infra", "dispatch"];
|
|
44
47
|
export const RETRY_MODES = ["resume", "fresh"];
|
|
45
48
|
export const WORKER_RESULT_CAUSES = ["provider-death", "stall-timeout", "malformed-trailer", "clean-exit-no-trailer"];
|
|
49
|
+
// Status consumes the routing profile's existing quality split directly: verified park kinds classify
|
|
50
|
+
// to 0, while availability/recovery noise classifies to null. Keep the synthetic row here at the
|
|
51
|
+
// run↔route seam so presentation code never grows a second list of "bad" park kinds.
|
|
52
|
+
export function isQualityFailureParkKind(kind) {
|
|
53
|
+
return classify({
|
|
54
|
+
shape: "", adapter: "-", model: "-", channel: "-", attempts: 0,
|
|
55
|
+
outcome: "human", durationMs: 0, parkKind: kind,
|
|
56
|
+
}) === 0;
|
|
57
|
+
}
|
|
58
|
+
// The newest terminal event owns the cause. Unknown/malformed kinds fail toward undefined so a
|
|
59
|
+
// task-failed row keeps the legacy red treatment instead of being mistaken for recoverable noise.
|
|
60
|
+
export function recordedTaskFailureKind(events, taskId) {
|
|
61
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
62
|
+
const e = events[i];
|
|
63
|
+
if (e.taskId !== taskId || (e.event !== "task-human" && e.event !== "task-failed"))
|
|
64
|
+
continue;
|
|
65
|
+
return typeof e.data.kind === "string" && PARK_KINDS.includes(e.data.kind)
|
|
66
|
+
? e.data.kind
|
|
67
|
+
: undefined;
|
|
68
|
+
}
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
// Runs can end and later resume in the same journal. The newest lifecycle marker decides whether
|
|
72
|
+
// an unresolved task is still recoverable by this live daemon or belongs to an ended run.
|
|
73
|
+
export function runHasEnded(events) {
|
|
74
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
75
|
+
const event = events[i].event;
|
|
76
|
+
if (event === "run-end" || event === "superseded")
|
|
77
|
+
return true;
|
|
78
|
+
if (event === "run-start" || event === "run-resume")
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
46
83
|
// OBS-53: provider-outage signatures in dead worker output ("Unable to reach the model provider" and kin).
|
|
47
84
|
const PROVIDER_OUTAGE_RE = /Unable to reach the model provider|cannot reach the model provider|model provider.*(?:unavailable|unreachable)/i;
|
|
48
85
|
/** OBS-53: classify worker-result failures so retries and routing see the true signal, not one lumped bucket. */
|
|
@@ -424,6 +461,20 @@ export class Journal {
|
|
|
424
461
|
}
|
|
425
462
|
return m;
|
|
426
463
|
}
|
|
464
|
+
// v1.71 OBS-119: run-wide exclusion fold — same replay discipline as replayResumeState().
|
|
465
|
+
// channel-exclusion is the typed event; dead-channel-failover.from is the pre-v1.71 compat seam.
|
|
466
|
+
replayExcludedChannels() {
|
|
467
|
+
const excluded = new Set();
|
|
468
|
+
for (const e of this.read()) {
|
|
469
|
+
if (e.event === "channel-exclusion" && typeof e.data.channel === "string") {
|
|
470
|
+
excluded.add(e.data.channel);
|
|
471
|
+
}
|
|
472
|
+
else if (e.event === "dead-channel-failover" && typeof e.data.from === "string") {
|
|
473
|
+
excluded.add(e.data.from);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
return excluded;
|
|
477
|
+
}
|
|
427
478
|
telemetry(row) {
|
|
428
479
|
// T3 secret redaction: same persistence seam as append — credential-free rows are byte-identical.
|
|
429
480
|
appendFileSync(join(this.dir, "telemetry.jsonl"), redactSecrets(JSON.stringify(row)) + "\n");
|