tickmarkr 1.69.0 → 1.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/kimi.js +12 -35
- package/dist/adapters/registry.d.ts +25 -1
- package/dist/adapters/registry.js +95 -4
- package/dist/adapters/types.d.ts +8 -0
- package/dist/cli/commands/doctor.js +8 -3
- package/dist/cli/commands/report.js +40 -5
- package/dist/cli/commands/resume.js +7 -1
- package/dist/cli/commands/status.js +20 -8
- package/dist/drivers/herdr.d.ts +5 -0
- package/dist/drivers/herdr.js +22 -3
- package/dist/gates/acceptance.d.ts +5 -1
- package/dist/gates/acceptance.js +72 -10
- package/dist/gates/review.d.ts +10 -2
- package/dist/gates/review.js +72 -19
- package/dist/report/bundle.d.ts +45 -0
- package/dist/report/bundle.js +158 -0
- package/dist/report/compare.d.ts +64 -0
- package/dist/report/compare.js +231 -0
- package/dist/route/preference.d.ts +9 -0
- package/dist/route/preference.js +61 -0
- package/dist/run/daemon.d.ts +4 -0
- package/dist/run/daemon.js +94 -11
- package/dist/run/environment.d.ts +12 -0
- package/dist/run/environment.js +41 -0
- package/dist/run/interactive-seed.d.ts +1 -0
- package/dist/run/interactive-seed.js +12 -4
- package/dist/run/journal.d.ts +9 -1
- package/dist/run/journal.js +53 -2
- package/package.json +1 -1
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { estimateCosts } from "./cost.js";
|
|
2
|
+
export function hasRunStart(events) {
|
|
3
|
+
return events.some((e) => e.event === "run-start");
|
|
4
|
+
}
|
|
5
|
+
// First run-start's environment, fail-closed on a missing or malformed stamp.
|
|
6
|
+
export function recordedEnvironment(events) {
|
|
7
|
+
for (const e of events) {
|
|
8
|
+
if (e.event !== "run-start")
|
|
9
|
+
continue;
|
|
10
|
+
const env = e.data.environment;
|
|
11
|
+
if (!env || typeof env !== "object" || Array.isArray(env))
|
|
12
|
+
return undefined;
|
|
13
|
+
const o = env;
|
|
14
|
+
if (typeof o.tickmarkrVersion !== "string" || typeof o.configHash !== "string")
|
|
15
|
+
return undefined;
|
|
16
|
+
if (!o.adapterVersions || typeof o.adapterVersions !== "object" || Array.isArray(o.adapterVersions))
|
|
17
|
+
return undefined;
|
|
18
|
+
const adapterVersions = {};
|
|
19
|
+
for (const [k, v] of Object.entries(o.adapterVersions)) {
|
|
20
|
+
if (typeof v !== "string")
|
|
21
|
+
return undefined;
|
|
22
|
+
adapterVersions[k] = v;
|
|
23
|
+
}
|
|
24
|
+
return { tickmarkrVersion: o.tickmarkrVersion, configHash: o.configHash, adapterVersions };
|
|
25
|
+
}
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
// Canonical identity string for the `recorded` field — configHash is the axis the acceptance
|
|
29
|
+
// criteria name; full environment equality still decides .comparable.
|
|
30
|
+
function envFingerprint(env) {
|
|
31
|
+
return env.configHash;
|
|
32
|
+
}
|
|
33
|
+
function envEqual(a, b) {
|
|
34
|
+
if (a.tickmarkrVersion !== b.tickmarkrVersion || a.configHash !== b.configHash)
|
|
35
|
+
return false;
|
|
36
|
+
const ak = Object.keys(a.adapterVersions).sort();
|
|
37
|
+
const bk = Object.keys(b.adapterVersions).sort();
|
|
38
|
+
if (ak.length !== bk.length)
|
|
39
|
+
return false;
|
|
40
|
+
for (let i = 0; i < ak.length; i++) {
|
|
41
|
+
if (ak[i] !== bk[i])
|
|
42
|
+
return false;
|
|
43
|
+
if (a.adapterVersions[ak[i]] !== b.adapterVersions[bk[i]])
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
// THE environment-identity comparator (criterion: reuses engagementComparable's shape).
|
|
49
|
+
// unbound = either side lacks a usable stamp; mismatch = stamps disagree; comparable = equal.
|
|
50
|
+
export function environmentComparable(baselineEnv, currentEnv) {
|
|
51
|
+
if (baselineEnv === undefined || currentEnv === undefined)
|
|
52
|
+
return { comparable: false, reason: "unbound" };
|
|
53
|
+
const recorded = envFingerprint(baselineEnv);
|
|
54
|
+
return envEqual(baselineEnv, currentEnv)
|
|
55
|
+
? { comparable: true, recorded }
|
|
56
|
+
: { comparable: false, reason: "mismatch", recorded };
|
|
57
|
+
}
|
|
58
|
+
export function runMetrics(events, rows = [], cost = {}) {
|
|
59
|
+
const start = events.find((e) => e.event === "run-start");
|
|
60
|
+
const end = [...events].reverse().find((e) => e.event === "run-end");
|
|
61
|
+
let durationMs;
|
|
62
|
+
if (start && end) {
|
|
63
|
+
const from = Date.parse(start.ts);
|
|
64
|
+
const to = Date.parse(end.ts);
|
|
65
|
+
if (Number.isFinite(from) && Number.isFinite(to) && to >= from)
|
|
66
|
+
durationMs = to - from;
|
|
67
|
+
}
|
|
68
|
+
const gates = events.filter((e) => e.event === "gate-result");
|
|
69
|
+
const gatePass = gates.filter((e) => e.data.pass === true).length;
|
|
70
|
+
const gateFail = gates.filter((e) => e.data.pass === false).length;
|
|
71
|
+
let costUsd;
|
|
72
|
+
let costSum = 0;
|
|
73
|
+
let hasCost = false;
|
|
74
|
+
for (const p of estimateCosts(rows, cost)) {
|
|
75
|
+
if (p.apiUsd !== undefined) {
|
|
76
|
+
costSum += p.apiUsd;
|
|
77
|
+
hasCost = true;
|
|
78
|
+
}
|
|
79
|
+
else if (p.amortizedUsd) {
|
|
80
|
+
costSum += (p.amortizedUsd[0] + p.amortizedUsd[1]) / 2;
|
|
81
|
+
hasCost = true;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (hasCost)
|
|
85
|
+
costUsd = Math.round(costSum * 1e6) / 1e6;
|
|
86
|
+
let tokensTotal;
|
|
87
|
+
let tokenSum = 0;
|
|
88
|
+
let hasTokens = false;
|
|
89
|
+
for (const r of rows) {
|
|
90
|
+
if (!r.tokens)
|
|
91
|
+
continue;
|
|
92
|
+
hasTokens = true;
|
|
93
|
+
const t = r.tokens;
|
|
94
|
+
tokenSum += t.input + t.output + (t.cacheRead ?? 0) + (t.cacheWrite ?? 0) + (t.reasoning ?? 0);
|
|
95
|
+
}
|
|
96
|
+
if (hasTokens)
|
|
97
|
+
tokensTotal = tokenSum;
|
|
98
|
+
return { durationMs, gatePass, gateFail, gateTotal: gates.length, costUsd, tokensTotal };
|
|
99
|
+
}
|
|
100
|
+
const n = (x) => x.toLocaleString("en-US");
|
|
101
|
+
const EM = "—";
|
|
102
|
+
function fmtDuration(ms) {
|
|
103
|
+
if (ms === undefined)
|
|
104
|
+
return EM;
|
|
105
|
+
const seconds = Math.round(ms / 1_000);
|
|
106
|
+
const minutes = Math.floor(seconds / 60);
|
|
107
|
+
return minutes ? `${minutes}m ${seconds % 60}s` : `${seconds}s`;
|
|
108
|
+
}
|
|
109
|
+
function fmtSignedDuration(ms) {
|
|
110
|
+
if (ms === undefined)
|
|
111
|
+
return EM;
|
|
112
|
+
if (ms === 0)
|
|
113
|
+
return "0s";
|
|
114
|
+
const sign = ms > 0 ? "+" : "-";
|
|
115
|
+
return `${sign}${fmtDuration(Math.abs(ms))}`;
|
|
116
|
+
}
|
|
117
|
+
function fmtUsd(v) {
|
|
118
|
+
if (v === undefined)
|
|
119
|
+
return EM;
|
|
120
|
+
return `$${v.toFixed(6)}`;
|
|
121
|
+
}
|
|
122
|
+
function fmtSignedUsd(v) {
|
|
123
|
+
if (v === undefined)
|
|
124
|
+
return EM;
|
|
125
|
+
if (v === 0)
|
|
126
|
+
return "$0.000000";
|
|
127
|
+
const sign = v > 0 ? "+" : "-";
|
|
128
|
+
return `${sign}$${Math.abs(v).toFixed(6)}`;
|
|
129
|
+
}
|
|
130
|
+
function fmtInt(v) {
|
|
131
|
+
if (v === undefined)
|
|
132
|
+
return EM;
|
|
133
|
+
return n(v);
|
|
134
|
+
}
|
|
135
|
+
function fmtSignedInt(v) {
|
|
136
|
+
if (v === undefined)
|
|
137
|
+
return EM;
|
|
138
|
+
if (v === 0)
|
|
139
|
+
return "0";
|
|
140
|
+
return v > 0 ? `+${n(v)}` : `-${n(Math.abs(v))}`;
|
|
141
|
+
}
|
|
142
|
+
function comparabilityLine(cmp, baselineEnv, currentEnv) {
|
|
143
|
+
if (cmp.comparable) {
|
|
144
|
+
return `full comparability (environment identity matches; configHash=${cmp.recorded})`;
|
|
145
|
+
}
|
|
146
|
+
if (cmp.reason === "unbound") {
|
|
147
|
+
return "comparability caveat — one or both runs lack a recorded environment identity; not apples-to-apples";
|
|
148
|
+
}
|
|
149
|
+
const base = baselineEnv?.configHash ?? EM;
|
|
150
|
+
const cur = currentEnv?.configHash ?? EM;
|
|
151
|
+
return `comparability caveat — environment identity disagrees (baseline configHash=${base} ≠ current configHash=${cur}; recorded baseline ${cmp.recorded}); not apples-to-apples`;
|
|
152
|
+
}
|
|
153
|
+
export function renderComparison(opts) {
|
|
154
|
+
const { runId, baselineRunId, comparability, baselineEnv, currentEnv, current, baseline, delta } = opts;
|
|
155
|
+
// Both run ids are always named so the reader knows which is the baseline.
|
|
156
|
+
const lines = [
|
|
157
|
+
`## Comparison`,
|
|
158
|
+
"",
|
|
159
|
+
`- **run:** ${runId}`,
|
|
160
|
+
`- **baseline:** ${baselineRunId}`,
|
|
161
|
+
`- **comparability:** ${comparabilityLine(comparability, baselineEnv, currentEnv)}`,
|
|
162
|
+
"",
|
|
163
|
+
"### Delta (current − baseline)",
|
|
164
|
+
"",
|
|
165
|
+
`| metric | baseline (${baselineRunId}) | current (${runId}) | delta |`,
|
|
166
|
+
`| --- | --- | --- | --- |`,
|
|
167
|
+
`| duration | ${fmtDuration(baseline.durationMs)} | ${fmtDuration(current.durationMs)} | ${fmtSignedDuration(delta.durationMs)} |`,
|
|
168
|
+
`| gate failures | ${fmtInt(baseline.gateFail)} | ${fmtInt(current.gateFail)} | ${fmtSignedInt(delta.gateFail)} |`,
|
|
169
|
+
`| gate pass rate | ${baseline.gateTotal ? `${baseline.gatePass}/${baseline.gateTotal}` : EM} | ${current.gateTotal ? `${current.gatePass}/${current.gateTotal}` : EM} | ${EM} |`,
|
|
170
|
+
`| cost | ${fmtUsd(baseline.costUsd)} | ${fmtUsd(current.costUsd)} | ${fmtSignedUsd(delta.costUsd)} |`,
|
|
171
|
+
`| tokens | ${fmtInt(baseline.tokensTotal)} | ${fmtInt(current.tokensTotal)} | ${fmtSignedInt(delta.tokensTotal)} |`,
|
|
172
|
+
"",
|
|
173
|
+
];
|
|
174
|
+
if (!comparability.comparable) {
|
|
175
|
+
lines.push("_Caveat: deltas are shown for inspection only — environment identity disagrees, so this is not an apples-to-apples table._", "");
|
|
176
|
+
}
|
|
177
|
+
return lines.join("\n").trimEnd() + "\n";
|
|
178
|
+
}
|
|
179
|
+
// Fail closed when either journal has no run-start (no partial / fabricated comparison).
|
|
180
|
+
// Mismatch of environment identity still yields a rendered delta, with an explicit caveat.
|
|
181
|
+
export function compareRuns(opts) {
|
|
182
|
+
if (!hasRunStart(opts.baselineEvents)) {
|
|
183
|
+
return {
|
|
184
|
+
ok: false,
|
|
185
|
+
reason: `baseline run ${opts.baselineRunId} has no recorded run-start event — cannot compare`,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
if (!hasRunStart(opts.events)) {
|
|
189
|
+
return {
|
|
190
|
+
ok: false,
|
|
191
|
+
reason: `run ${opts.runId} has no recorded run-start event — cannot compare`,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
const baselineEnv = recordedEnvironment(opts.baselineEvents);
|
|
195
|
+
const currentEnv = recordedEnvironment(opts.events);
|
|
196
|
+
const comparability = environmentComparable(baselineEnv, currentEnv);
|
|
197
|
+
const baseline = runMetrics(opts.baselineEvents, opts.baselineRows ?? [], opts.cost ?? {});
|
|
198
|
+
const current = runMetrics(opts.events, opts.rows ?? [], opts.cost ?? {});
|
|
199
|
+
const delta = {
|
|
200
|
+
durationMs: current.durationMs !== undefined && baseline.durationMs !== undefined
|
|
201
|
+
? current.durationMs - baseline.durationMs
|
|
202
|
+
: undefined,
|
|
203
|
+
gateFail: current.gateFail - baseline.gateFail,
|
|
204
|
+
costUsd: current.costUsd !== undefined && baseline.costUsd !== undefined
|
|
205
|
+
? Math.round((current.costUsd - baseline.costUsd) * 1e6) / 1e6
|
|
206
|
+
: undefined,
|
|
207
|
+
tokensTotal: current.tokensTotal !== undefined && baseline.tokensTotal !== undefined
|
|
208
|
+
? current.tokensTotal - baseline.tokensTotal
|
|
209
|
+
: undefined,
|
|
210
|
+
};
|
|
211
|
+
const text = renderComparison({
|
|
212
|
+
runId: opts.runId,
|
|
213
|
+
baselineRunId: opts.baselineRunId,
|
|
214
|
+
comparability,
|
|
215
|
+
baselineEnv,
|
|
216
|
+
currentEnv,
|
|
217
|
+
current,
|
|
218
|
+
baseline,
|
|
219
|
+
delta,
|
|
220
|
+
});
|
|
221
|
+
return {
|
|
222
|
+
ok: true,
|
|
223
|
+
runId: opts.runId,
|
|
224
|
+
baselineRunId: opts.baselineRunId,
|
|
225
|
+
comparability,
|
|
226
|
+
current,
|
|
227
|
+
baseline,
|
|
228
|
+
delta,
|
|
229
|
+
text,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
@@ -25,3 +25,12 @@ export declare function disallowedBy(c: {
|
|
|
25
25
|
adapter: string;
|
|
26
26
|
model: string;
|
|
27
27
|
}, routing: TickmarkrConfig["routing"]): Disallowed | null;
|
|
28
|
+
export interface DenyPreferCollision {
|
|
29
|
+
kind: "prefer" | "pin";
|
|
30
|
+
shape: string;
|
|
31
|
+
detail: string;
|
|
32
|
+
disallowed: Disallowed;
|
|
33
|
+
}
|
|
34
|
+
export declare function preferEntryDenied(p: string, cfg: TickmarkrConfig): Disallowed | null;
|
|
35
|
+
export declare function denyPreferCollisions(cfg: TickmarkrConfig): DenyPreferCollision[];
|
|
36
|
+
export declare function denyPreferCollisionLine({ kind, shape, detail, disallowed }: DenyPreferCollision): string;
|
package/dist/route/preference.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { channelKey, channelsFromConfig } from "../adapters/types.js";
|
|
2
|
+
import { validateGraph } from "../graph/schema.js";
|
|
3
|
+
import { route, RoutingError } from "./router.js";
|
|
2
4
|
const adapterIds = (adapters) => typeof adapters[0] === "string" ? adapters : adapters.map((a) => a.id);
|
|
3
5
|
export function excludedChannels(cfg, adapters, health) {
|
|
4
6
|
const { allow, deny } = cfg.routing;
|
|
@@ -47,3 +49,62 @@ export function disallowedBy(c, routing) {
|
|
|
47
49
|
}
|
|
48
50
|
return null;
|
|
49
51
|
}
|
|
52
|
+
// Delegates to route()'s preflightPrefer — doctor/resume preflight reuses the router grammar,
|
|
53
|
+
// not a second colon/tier/model parser of the same config.
|
|
54
|
+
const PREFLIGHT_SHAPE = "chore";
|
|
55
|
+
const preflightTask = validateGraph({
|
|
56
|
+
version: 1, spec: { source: "prd", paths: ["p"], hash: "h" },
|
|
57
|
+
tasks: [{ id: "T0", title: "preflight", goal: "preflight", shape: PREFLIGHT_SHAPE, complexity: 1, acceptance: ["preflight"] }],
|
|
58
|
+
}).tasks[0];
|
|
59
|
+
const disallowedFromPreferError = (msg) => {
|
|
60
|
+
const m = msg.match(/prefer entry .+ is disallowed by routing\.(deny|allow) \(([^)]+)\)/);
|
|
61
|
+
return m ? { by: m[1], entry: m[2] } : null;
|
|
62
|
+
};
|
|
63
|
+
export function preferEntryDenied(p, cfg) {
|
|
64
|
+
if (!cfg.routing.allow && !cfg.routing.deny)
|
|
65
|
+
return null;
|
|
66
|
+
const probe = structuredClone(cfg);
|
|
67
|
+
probe.routing.map = { ...probe.routing.map, [PREFLIGHT_SHAPE]: { prefer: [p] } };
|
|
68
|
+
try {
|
|
69
|
+
route(preflightTask, probe, []);
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
if (e instanceof RoutingError)
|
|
74
|
+
return disallowedFromPreferError(e.message);
|
|
75
|
+
throw e;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
export function denyPreferCollisions(cfg) {
|
|
79
|
+
if (!cfg.routing.allow && !cfg.routing.deny)
|
|
80
|
+
return [];
|
|
81
|
+
const out = [];
|
|
82
|
+
for (const [shape, entry] of Object.entries(cfg.routing.map)) {
|
|
83
|
+
if (entry.pin) {
|
|
84
|
+
const d = disallowedBy({ adapter: entry.pin.via, model: entry.pin.model }, cfg.routing);
|
|
85
|
+
if (d) {
|
|
86
|
+
out.push({
|
|
87
|
+
kind: "pin",
|
|
88
|
+
shape,
|
|
89
|
+
detail: `${entry.pin.via}:${entry.pin.model}`,
|
|
90
|
+
disallowed: d,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const prefer = entry.prefer ?? [];
|
|
95
|
+
if (prefer.length && prefer.every((p) => preferEntryDenied(p, cfg) !== null)) {
|
|
96
|
+
out.push({
|
|
97
|
+
kind: "prefer",
|
|
98
|
+
shape,
|
|
99
|
+
detail: prefer.join(" > "),
|
|
100
|
+
disallowed: preferEntryDenied(prefer[0], cfg),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
export function denyPreferCollisionLine({ kind, shape, detail, disallowed }) {
|
|
107
|
+
const surface = kind === "pin" ? `routing.map.${shape}.pin` : `routing.map.${shape}.prefer`;
|
|
108
|
+
const verb = kind === "pin" ? "is disallowed" : "fully disallowed";
|
|
109
|
+
return `deny∩prefer: ${surface} ${detail} ${verb} by routing.${disallowed.by} (${disallowed.entry}) — remove the ${disallowed.by} entry or adjust ${kind === "pin" ? "pin" : "prefer"}`;
|
|
110
|
+
}
|
package/dist/run/daemon.d.ts
CHANGED
|
@@ -41,4 +41,8 @@ export interface RunSummary {
|
|
|
41
41
|
lastMergedTask?: string;
|
|
42
42
|
}
|
|
43
43
|
export declare function formatSummary(s: RunSummary): string;
|
|
44
|
+
export declare const EARLY_LAUNCH_LIVENESS_MS = 60000;
|
|
45
|
+
/** Test seam — lowers the empty-pane liveness window without sleeping 60s per case. */
|
|
46
|
+
export declare function setEarlyLaunchLivenessMsForTests(ms: number): void;
|
|
47
|
+
export declare function resetEarlyLaunchLivenessMsForTests(): void;
|
|
44
48
|
export declare function runDaemon(repoRoot: string, opts?: RunOptions): Promise<RunSummary>;
|
package/dist/run/daemon.js
CHANGED
|
@@ -15,6 +15,7 @@ import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../ga
|
|
|
15
15
|
import { runGates } from "../gates/run-gates.js";
|
|
16
16
|
import { addEvidence, attributeBlocked, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus } from "../graph/graph.js";
|
|
17
17
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
18
|
+
import { runEnvironment } from "./environment.js";
|
|
18
19
|
import { cleanupRunWorktrees, gitHead, linkNodeModules, sh, shGit, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
19
20
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
20
21
|
import { classifyWorkerResultCause, engagementComparable, Journal, loadRoutingProfile, newRunId } from "./journal.js";
|
|
@@ -65,10 +66,26 @@ export function formatSummary(s) {
|
|
|
65
66
|
return `done: ${s.done.length}, failed: ${s.failed.length}, human: ${s.human.length}, blocked: ${s.blocked.length}, pending: ${s.pending.length}\nintegration branch: ${s.branch}${tip}`;
|
|
66
67
|
}
|
|
67
68
|
const MAX_ATTEMPTS = 10; // ponytail: hard cap so a pathological ladder can never loop forever
|
|
69
|
+
// v1.70 T5: request-changes review rounds a single task may draw before it parks for a human decision
|
|
70
|
+
// instead of cycling. Well below MAX_ATTEMPTS so review non-convergence is caught long before the
|
|
71
|
+
// global cap. ponytail: literal constant; lift to cfg.review.roundCap only if a second knob-turner appears.
|
|
72
|
+
const REVIEW_ROUND_CAP = 3;
|
|
68
73
|
const BLOCKED_POLL_MS = 30_000; // between trailer-wait slices, check whether the pane is blocked on a prompt
|
|
69
74
|
const PROVIDER_DEATH_REQUEUE_CAP = 2; // v1.46 T1: requeue same assignment twice, then fall through to the normal ladder
|
|
70
75
|
const PROVIDER_DEATH_BACKOFF_MS = 500; // short backoff before provider-death requeue
|
|
71
76
|
const NO_TRAILER_DEMOTION_STREAK = 2; // OBS-57: consecutive no-trailer windows demote a channel for the rest of the run
|
|
77
|
+
// OBS-117 (v1.71 T6): a worker pane that never prints a byte by T+60s after dispatch is a dead
|
|
78
|
+
// channel — don't burn the full stall window waiting for a silent launch failure. Checked on the
|
|
79
|
+
// existing stall-wait poll cadence only (no new timer). Spinner/ANSI repaints count as output.
|
|
80
|
+
export const EARLY_LAUNCH_LIVENESS_MS = 60_000;
|
|
81
|
+
let earlyLaunchLivenessMs = EARLY_LAUNCH_LIVENESS_MS;
|
|
82
|
+
/** Test seam — lowers the empty-pane liveness window without sleeping 60s per case. */
|
|
83
|
+
export function setEarlyLaunchLivenessMsForTests(ms) {
|
|
84
|
+
earlyLaunchLivenessMs = ms;
|
|
85
|
+
}
|
|
86
|
+
export function resetEarlyLaunchLivenessMsForTests() {
|
|
87
|
+
earlyLaunchLivenessMs = EARLY_LAUNCH_LIVENESS_MS;
|
|
88
|
+
}
|
|
72
89
|
async function commitsAheadOf(base, wt) {
|
|
73
90
|
const head = await gitHead(wt);
|
|
74
91
|
if (head === base)
|
|
@@ -212,6 +229,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
212
229
|
// GATE-08 lesson at the humanGate guard: condition on the data, not the code path). Dead-code
|
|
213
230
|
// equivalence to the router.ts:194 profile⇒undefined pattern: no map entry ⇒ today's literal.
|
|
214
231
|
const resume = opts.resume ? journal.replayResumeState() : new Map();
|
|
232
|
+
const replayedExclusions = opts.resume ? journal.replayExcludedChannels() : new Set();
|
|
215
233
|
if (opts.resume) {
|
|
216
234
|
// v1.53 T5: a superseded run is dead — resuming it beside its successor is the exact
|
|
217
235
|
// two-concurrent-runs hazard supersession exists to prevent. Fail closed, naming the successor.
|
|
@@ -247,13 +265,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
247
265
|
continue;
|
|
248
266
|
graph = setStatus(graph, id, st);
|
|
249
267
|
}
|
|
250
|
-
journal.append("run-resume", undefined, {
|
|
268
|
+
journal.append("run-resume", undefined, {
|
|
269
|
+
pid: process.pid, // v1.13 (VIS-11): record the live daemon pid for status liveness
|
|
270
|
+
...(replayedExclusions.size > 0 ? { excludedChannels: [...replayedExclusions].sort() } : {}),
|
|
271
|
+
});
|
|
251
272
|
}
|
|
252
273
|
else {
|
|
253
274
|
baseRef = await gitHead(repoRoot);
|
|
254
275
|
baseline = await captureBaseline(repoRoot, commands);
|
|
255
276
|
writeFileSync(join(journal.dir, "baseline.json"), JSON.stringify(baseline, null, 2));
|
|
256
|
-
|
|
277
|
+
// v1.70 T2: environment identity beside the graph/branch identity — running tickmarkr version,
|
|
278
|
+
// loaded-config hash, and the probed CLI version of each adapter holding a channel in the run,
|
|
279
|
+
// gathered through the existing probe/config-load paths (no second mechanism).
|
|
280
|
+
const environment = runEnvironment(cfg, channels, health);
|
|
281
|
+
journal.append("run-start", undefined, { pid: process.pid, baseRef, commands, channels: channels.map(channelKey), branch, graphDefinitionHash: graphDefinitionHash(graph), mode: rm.mode.mode, modeSource: rm.source, environment, ...(prior ? { supersedes: prior.runId } : {}) }); // graphDefinitionHash: T3 engagement identity (status+resume share it); pid: v1.13 (VIS-11) liveness; mode/modeSource: v1.51 T2; supersedes: v1.53 T5
|
|
257
282
|
runStarted = true;
|
|
258
283
|
// v1.53 T5: mark the prior run AFTER this run's run-start exists, so the prior journal never
|
|
259
284
|
// names a successor that has no journal. Append-only — the prior journal is never rewritten.
|
|
@@ -292,7 +317,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
292
317
|
const keptSlots = [];
|
|
293
318
|
const runTag = runId.replace(/^run-/, ""); // full date-time — cross-run unique even across days
|
|
294
319
|
// OBS-57: per-run in-run demotion — channels that burn consecutive no-trailer windows route around for later attempts.
|
|
295
|
-
|
|
320
|
+
// v1.71 OBS-119: on resume, re-seed from the journal fold (replayedExclusions) before any dispatch.
|
|
321
|
+
const demotedChannels = new Set(replayedExclusions);
|
|
296
322
|
const noTrailerStreak = new Map();
|
|
297
323
|
// OBS-17 T2: reconcile at every safe point — run start/resume (just journaled above), each task
|
|
298
324
|
// terminal event, and run-end. The desired set is the pure journal fold (reconcile.ts); the driver
|
|
@@ -385,7 +411,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
385
411
|
// fresh-budget release, prefer nextChannel over the surviving tried-list so burned channels are
|
|
386
412
|
// not re-tried first (consult bans / prior failovers survive the release).
|
|
387
413
|
const rs = resume.get(t.id);
|
|
388
|
-
if (rs?.lastAssignment && rs.attempts > 0
|
|
414
|
+
if (rs?.lastAssignment && rs.attempts > 0
|
|
415
|
+
&& channels.some((c) => channelKey(c) === channelKey(rs.lastAssignment))
|
|
416
|
+
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
389
417
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
390
418
|
}
|
|
391
419
|
else if (rs && rs.tried.length) {
|
|
@@ -410,6 +438,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
410
438
|
if (rs)
|
|
411
439
|
journal.append("resume-restore", t.id, { attempts: rs.attempts, tried: [...tried], assignment });
|
|
412
440
|
const badReviewers = []; // v1.1: reviewer channels that produced unparseable output for this task
|
|
441
|
+
// v1.70 T5 (review-convergence): failed review rounds this task has drawn, counted from the per-task
|
|
442
|
+
// review history already in the journal — the SAME review gate-result stream onGate reads to grow the
|
|
443
|
+
// reviewer-exclusion list (badReviewers), never a second parallel counter. request-changes rounds do
|
|
444
|
+
// not exclude their reviewer (a fix is re-checked by the same seat), so their count lives here in the
|
|
445
|
+
// shared history rather than in badReviewers' garbage-only exclusion subset.
|
|
446
|
+
const reviewRoundsDrawn = () => journal.read().filter((e) => e.taskId === t.id && e.event === "gate-result" &&
|
|
447
|
+
e.data.gate === "review" &&
|
|
448
|
+
e.data.pass === false).length;
|
|
413
449
|
let feedback = "";
|
|
414
450
|
let ladderIdx = 0;
|
|
415
451
|
let modeFallbackNoted = false; // v1.2: journal the interactive→print fallback once per task, not per attempt
|
|
@@ -510,6 +546,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
510
546
|
await park(t, `attempt cap (${MAX_ATTEMPTS}) reached`, "attempt-cap", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
511
547
|
return;
|
|
512
548
|
}
|
|
549
|
+
// v1.70 T5: a task that has already drawn REVIEW_ROUND_CAP request-changes review rounds parks for
|
|
550
|
+
// a human decision instead of dispatching another round. Condition on the review history (data),
|
|
551
|
+
// never the code path — and let an operator's approval (the human decision the cap asked for)
|
|
552
|
+
// release it, mirroring the humanGate guard's condition-on-approval precedent. Rounds only accrue
|
|
553
|
+
// after attempt 0, so the guard skips the journal read on the happy path.
|
|
554
|
+
if (attempt > 0 && !approved.has(t.id) && reviewRoundsDrawn() >= REVIEW_ROUND_CAP) {
|
|
555
|
+
await park(t, `review round cap (${REVIEW_ROUND_CAP}) reached — request-changes reviews not converging; a human should decide`, "gate-fail", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
513
558
|
// OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
|
|
514
559
|
if (demotedChannels.has(channelKey(assignment))) {
|
|
515
560
|
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels);
|
|
@@ -664,6 +709,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
664
709
|
let output;
|
|
665
710
|
let exitCode;
|
|
666
711
|
let timedOut = false;
|
|
712
|
+
let earlyLaunchDead = false;
|
|
667
713
|
let settleParsed;
|
|
668
714
|
let seedResult;
|
|
669
715
|
if (interactive) {
|
|
@@ -698,12 +744,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
698
744
|
// OBS-82: the stall clock compares NORMALIZED snapshots so a spinner glyph/elapsed-time
|
|
699
745
|
// repaint is silence, not activity. ONLY this inactivity compare sees normalized text —
|
|
700
746
|
// trailer detection, harvest, paging, and quota checks all read the raw pane.
|
|
747
|
+
let everHadOutput = output.length > 0;
|
|
701
748
|
let lastStallSnapshot = normalizeStallSnapshot(output);
|
|
702
749
|
let lastOutputAt = Date.now();
|
|
703
750
|
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
704
751
|
const sliceStart = Date.now();
|
|
705
752
|
const remaining = stallWindowMs - (sliceStart - lastOutputAt);
|
|
706
|
-
|
|
753
|
+
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
754
|
+
if (!everHadOutput) {
|
|
755
|
+
const earlyLeft = earlyLaunchLivenessMs - (sliceStart - attemptStart);
|
|
756
|
+
if (earlyLeft > 0)
|
|
757
|
+
slice = Math.min(slice, earlyLeft);
|
|
758
|
+
}
|
|
707
759
|
if (await driver.waitOutput(slot, `(${trailerPattern(nonce)})|TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
708
760
|
// verify before accepting: a worker that merely DISPLAYS a marker (e.g. editing tickmarkr's
|
|
709
761
|
// own source, where "TICKMARKR_EXIT:" is a string literal) must not end the wait. Only a
|
|
@@ -717,11 +769,20 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
717
769
|
break;
|
|
718
770
|
}
|
|
719
771
|
}
|
|
720
|
-
const
|
|
772
|
+
const paneText = await driver.read(slot, 1000);
|
|
773
|
+
if (paneText.length > 0)
|
|
774
|
+
everHadOutput = true;
|
|
775
|
+
const currentStallSnapshot = normalizeStallSnapshot(paneText);
|
|
721
776
|
if (currentStallSnapshot !== lastStallSnapshot) {
|
|
722
777
|
lastStallSnapshot = currentStallSnapshot;
|
|
723
778
|
lastOutputAt = Date.now();
|
|
724
779
|
}
|
|
780
|
+
// OBS-117 (v1.71 T6): zero raw output by the early-launch deadline is a dead channel now.
|
|
781
|
+
if (!everHadOutput && Date.now() - attemptStart >= earlyLaunchLivenessMs) {
|
|
782
|
+
earlyLaunchDead = true;
|
|
783
|
+
output = paneText;
|
|
784
|
+
break;
|
|
785
|
+
}
|
|
725
786
|
// v1.23 T2: piggyback on this poll slice — same cadence as blocked/idle checks, no new timer.
|
|
726
787
|
await sampleContext();
|
|
727
788
|
// page on "idle" too: herdr's blocked-scrape is strict and proved flaky for TUI dialogs
|
|
@@ -802,21 +863,35 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
802
863
|
// OBS-82: same normalized-snapshot compare as the interactive site — spinner-only repaints
|
|
803
864
|
// exhaust the budget here too; harvest below still reads the raw pane.
|
|
804
865
|
const stallWindowMs = taskTimeoutMinutes * 60_000;
|
|
805
|
-
|
|
866
|
+
const initialPane = await driver.read(slot, 500);
|
|
867
|
+
let everHadOutput = initialPane.length > 0;
|
|
868
|
+
let lastStallSnapshot = normalizeStallSnapshot(initialPane);
|
|
806
869
|
let lastOutputAt = Date.now();
|
|
807
870
|
finished = false;
|
|
808
871
|
while (Date.now() - lastOutputAt < stallWindowMs) {
|
|
809
872
|
const remaining = stallWindowMs - (Date.now() - lastOutputAt);
|
|
810
|
-
|
|
873
|
+
let slice = Math.min(BLOCKED_POLL_MS, Math.max(100, Math.min(stallWindowMs / 2, remaining)));
|
|
874
|
+
if (!everHadOutput) {
|
|
875
|
+
const earlyLeft = earlyLaunchLivenessMs - (Date.now() - attemptStart);
|
|
876
|
+
if (earlyLeft > 0)
|
|
877
|
+
slice = Math.min(slice, earlyLeft);
|
|
878
|
+
}
|
|
811
879
|
if (await driver.waitOutput(slot, `TICKMARKR_EXIT_${nonce}:\\d`, slice, { regex: true })) {
|
|
812
880
|
finished = true;
|
|
813
881
|
break;
|
|
814
882
|
}
|
|
815
|
-
const
|
|
883
|
+
const paneText = await driver.read(slot, 500);
|
|
884
|
+
if (paneText.length > 0)
|
|
885
|
+
everHadOutput = true;
|
|
886
|
+
const currentStallSnapshot = normalizeStallSnapshot(paneText);
|
|
816
887
|
if (currentStallSnapshot !== lastStallSnapshot) {
|
|
817
888
|
lastStallSnapshot = currentStallSnapshot;
|
|
818
889
|
lastOutputAt = Date.now();
|
|
819
890
|
}
|
|
891
|
+
if (!everHadOutput && Date.now() - attemptStart >= earlyLaunchLivenessMs) {
|
|
892
|
+
earlyLaunchDead = true;
|
|
893
|
+
break;
|
|
894
|
+
}
|
|
820
895
|
}
|
|
821
896
|
output = await driver.read(slot, 500);
|
|
822
897
|
exitCode = Number(exitRe.exec(output)?.[1] ?? 1);
|
|
@@ -914,10 +989,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
914
989
|
// AFTER the quota check so quota behavior stays byte-identical (a quota hit returns/continues
|
|
915
990
|
// before reaching here); provider-outage lands here only once the v1.46 same-channel requeue
|
|
916
991
|
// cap above is spent, so a transient blip still recovers in place.
|
|
917
|
-
|
|
992
|
+
// OBS-117 (v1.71 T6): a silent launch failure has no CLI signature to parse — the same
|
|
993
|
+
// setup-required typed dead-channel path a late-harvest "command not found" would take.
|
|
994
|
+
const dead = classifyDeadChannel(result) ?? (earlyLaunchDead ? "setup-required" : undefined);
|
|
918
995
|
if (dead) {
|
|
919
996
|
const from = channelKey(assignment);
|
|
920
997
|
demotedChannels.add(from); // excluded for later attempts AND later tasks in this run
|
|
998
|
+
journal.append("channel-exclusion", t.id, { channel: from, reason: dead, kind: "dead-channel" });
|
|
921
999
|
const next = failover("dead-channel");
|
|
922
1000
|
journal.append("dead-channel-failover", t.id, { reason: dead, from, to: next ? channelKey(next) : null });
|
|
923
1001
|
if (next) {
|
|
@@ -1104,9 +1182,14 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1104
1182
|
for (const t of ready) {
|
|
1105
1183
|
const p = execTask(t)
|
|
1106
1184
|
.catch(async (err) => {
|
|
1185
|
+
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
1186
|
+
const dispatch = [...taskEvents].reverse().find((e) => e.event === "task-dispatch");
|
|
1187
|
+
const verifiedFailure = taskEvents.some((e) => e.event === "gate-result" && e.data.pass === false);
|
|
1188
|
+
const kind = verifiedFailure ? "gate-fail" : dispatch ? "dispatch" : "infra";
|
|
1189
|
+
const attempts = dispatch && Number.isInteger(dispatch.data.attempt) ? dispatch.data.attempt : 0;
|
|
1107
1190
|
graph = setStatus(graph, t.id, "failed");
|
|
1108
1191
|
saveGraph(repoRoot, graph);
|
|
1109
|
-
journal.append("task-failed", t.id, { error: String(err) });
|
|
1192
|
+
journal.append("task-failed", t.id, { error: String(err), kind, attempts });
|
|
1110
1193
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
1111
1194
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
1112
1195
|
})
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { AuthHealth, BillingChannel } from "../adapters/types.js";
|
|
2
|
+
import type { TickmarkrConfig } from "../config/config.js";
|
|
3
|
+
export interface RunEnvironment {
|
|
4
|
+
tickmarkrVersion: string;
|
|
5
|
+
configHash: string;
|
|
6
|
+
adapterVersions: Record<string, string>;
|
|
7
|
+
}
|
|
8
|
+
export declare const UNKNOWN_ADAPTER_VERSION = "unknown";
|
|
9
|
+
export declare function tickmarkrVersion(): string;
|
|
10
|
+
export declare function configHash(cfg: TickmarkrConfig): string;
|
|
11
|
+
export declare function adapterVersions(channels: BillingChannel[], health: Record<string, AuthHealth>): Record<string, string>;
|
|
12
|
+
export declare function runEnvironment(cfg: TickmarkrConfig, channels: BillingChannel[], health: Record<string, AuthHealth>): RunEnvironment;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
// An adapter whose version probe failed is recorded, not dropped — "unknown", never a fabricated string.
|
|
6
|
+
export const UNKNOWN_ADAPTER_VERSION = "unknown";
|
|
7
|
+
// Same package.json read as src/cli/commands/version.ts (one resolution pattern, two consumers).
|
|
8
|
+
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "../../package.json");
|
|
9
|
+
export function tickmarkrVersion() {
|
|
10
|
+
const { version: v } = JSON.parse(readFileSync(pkgPath, "utf8"));
|
|
11
|
+
return v;
|
|
12
|
+
}
|
|
13
|
+
// Key order in a parsed config is an accident of the schema/merge layers, so the hash canonicalizes
|
|
14
|
+
// first: object keys sorted recursively, undefined dropped (JSON semantics), array order preserved.
|
|
15
|
+
function stableStringify(v) {
|
|
16
|
+
if (v === null || typeof v !== "object")
|
|
17
|
+
return JSON.stringify(v);
|
|
18
|
+
if (Array.isArray(v))
|
|
19
|
+
return `[${v.map(stableStringify).join(",")}]`;
|
|
20
|
+
const entries = Object.entries(v)
|
|
21
|
+
.filter(([, val]) => val !== undefined)
|
|
22
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
23
|
+
return `{${entries.map(([k, val]) => `${JSON.stringify(k)}:${stableStringify(val)}`).join(",")}}`;
|
|
24
|
+
}
|
|
25
|
+
// sha256 truncated to 16 hex — the graphDefinitionHash convention (stable, grep-friendly).
|
|
26
|
+
export function configHash(cfg) {
|
|
27
|
+
return createHash("sha256").update(stableStringify(cfg)).digest("hex").slice(0, 16);
|
|
28
|
+
}
|
|
29
|
+
// One entry per adapter with a channel in the run (not per channel). The version is whatever the
|
|
30
|
+
// adapter's own probe recorded in health; a missing/undefined probe result becomes "unknown".
|
|
31
|
+
export function adapterVersions(channels, health) {
|
|
32
|
+
const out = {};
|
|
33
|
+
for (const c of channels) {
|
|
34
|
+
if (!(c.adapter in out))
|
|
35
|
+
out[c.adapter] = health[c.adapter]?.version ?? UNKNOWN_ADAPTER_VERSION;
|
|
36
|
+
}
|
|
37
|
+
return out;
|
|
38
|
+
}
|
|
39
|
+
export function runEnvironment(cfg, channels, health) {
|
|
40
|
+
return { tickmarkrVersion: tickmarkrVersion(), configHash: configHash(cfg), adapterVersions: adapterVersions(channels, health) };
|
|
41
|
+
}
|
|
@@ -6,9 +6,17 @@ export async function runInteractiveSeed(opts) {
|
|
|
6
6
|
const seed = opts.adapter.interactiveSeed;
|
|
7
7
|
await opts.driver.run(opts.slot, seed.launch(opts.assignment.model));
|
|
8
8
|
const ready = await opts.driver.waitOutput(opts.slot, seed.readinessMatch, opts.taskTimeoutMinutes * 60_000);
|
|
9
|
+
const banner = await opts.driver.read(opts.slot, 1000);
|
|
9
10
|
if (!ready) {
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
return { output: banner, seedFailed: true, seedError: `readiness pattern not seen: ${seed.readinessMatch}` };
|
|
12
|
+
}
|
|
13
|
+
let sessionId;
|
|
14
|
+
if (seed.confirmBanner) {
|
|
15
|
+
const confirm = seed.confirmBanner(banner, opts.assignment.model);
|
|
16
|
+
if (!confirm.ok) {
|
|
17
|
+
return { output: banner, seedFailed: true, seedError: confirm.error };
|
|
18
|
+
}
|
|
19
|
+
sessionId = confirm.sessionId;
|
|
12
20
|
}
|
|
13
21
|
const seedText = seed.seedLine(opts.promptFile);
|
|
14
22
|
await opts.driver.run(opts.slot, seedText);
|
|
@@ -18,8 +26,8 @@ export async function runInteractiveSeed(opts) {
|
|
|
18
26
|
output = await opts.driver.read(opts.slot, 1000);
|
|
19
27
|
const bottom = output.trimEnd().split("\n").pop() ?? "";
|
|
20
28
|
if (!bottom.includes(seedText)) {
|
|
21
|
-
return { output, seedFailed: false };
|
|
29
|
+
return { output, seedFailed: false, ...(sessionId ? { sessionId } : {}) };
|
|
22
30
|
}
|
|
23
31
|
}
|
|
24
|
-
return { output, seedFailed: true, seedError: "seed line never left the input box" };
|
|
32
|
+
return { output, seedFailed: true, seedError: "seed line never left the input box", ...(sessionId ? { sessionId } : {}) };
|
|
25
33
|
}
|