zelari-code 1.27.2 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/kraken/executor.js +750 -52
- package/dist/cli/kraken/executor.js.map +1 -1
- package/dist/cli/kraken/graphMemory.js +133 -0
- package/dist/cli/kraken/graphMemory.js.map +1 -0
- package/dist/cli/kraken/graphStatus.js +59 -0
- package/dist/cli/kraken/graphStatus.js.map +1 -1
- package/dist/cli/kraken/planner.js +222 -21
- package/dist/cli/kraken/planner.js.map +1 -1
- package/dist/cli/main.bundled.js +1213 -292
- package/dist/cli/main.bundled.js.map +4 -4
- package/dist/cli/runHeadless.js +41 -3
- package/dist/cli/runHeadless.js.map +1 -1
- package/dist/cli/slashHandlers/krakenGraph.js +22 -4
- package/dist/cli/slashHandlers/krakenGraph.js.map +1 -1
- package/dist/cli/tools/taskTool.js +48 -3
- package/dist/cli/tools/taskTool.js.map +1 -1
- package/package.json +2 -2
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
* (`runTentacle`) to execute a validated `TaskGraph`:
|
|
7
7
|
* - repeatedly forms a parallel-safe "wave" of ready nodes (bounded by
|
|
8
8
|
* ZELARI_KRAKEN_MAX_PARALLEL) and runs it concurrently;
|
|
9
|
+
* - hands each node the conclusions of its completed dependencies
|
|
10
|
+
* (`buildUpstreamContext`), so a dep edge carries information and not just
|
|
11
|
+
* ordering;
|
|
9
12
|
* - retries a failed node up to `node.maxRetries`, then spawns a `fix`
|
|
10
13
|
* node (bounded by a global fix budget) that inherits the failed
|
|
11
14
|
* node's deps/scope/acceptance; a node that still fails is left
|
|
@@ -31,7 +34,7 @@
|
|
|
31
34
|
*
|
|
32
35
|
* @since v0.10.x — Kraken graph engine (F3)
|
|
33
36
|
*/
|
|
34
|
-
import { getReadyNodes, isSettled, isConverged, failedNodeIds, countByStatus,
|
|
37
|
+
import { getReadyNodes, isSettled, isConverged, failedNodeIds, countByStatus, canRunParallel, parseVerifyVerdict, } from '@zelari/core';
|
|
35
38
|
import { runTentacle, } from './tentacle.js';
|
|
36
39
|
import { mergeKrakenWorktree, } from '../tools/krakenWorktree.js';
|
|
37
40
|
import { appendKrakenRadio } from '../tools/krakenRadio.js';
|
|
@@ -39,6 +42,7 @@ import { runBacktest } from '../workspace/worldModel.js';
|
|
|
39
42
|
import { startKrakenGraphLive, updateKrakenGraphLive, endKrakenGraphLive, } from './graphStatus.js';
|
|
40
43
|
import { existsSync } from 'node:fs';
|
|
41
44
|
import path from 'node:path';
|
|
45
|
+
import { saveGraphSnapshot, toGraphSnapshot } from './graphMemory.js';
|
|
42
46
|
/** Default cap on concurrently-running tentacles across the whole graph. */
|
|
43
47
|
export const DEFAULT_MAX_PARALLEL = 12;
|
|
44
48
|
/** Default number of `fix` nodes the executor may spawn across one graph run. */
|
|
@@ -50,12 +54,58 @@ export function resolveMaxParallel(env = process.env) {
|
|
|
50
54
|
const n = Number.parseInt(raw, 10);
|
|
51
55
|
return Number.isFinite(n) && n > 0 ? n : DEFAULT_MAX_PARALLEL;
|
|
52
56
|
}
|
|
53
|
-
|
|
57
|
+
/**
|
|
58
|
+
* How many `fix` nodes this run may spawn.
|
|
59
|
+
*
|
|
60
|
+
* A flat budget of 3 is a graph-size-blind number: on a 20-node graph it is
|
|
61
|
+
* spent on the first three failures and every later failure goes terminal,
|
|
62
|
+
* cascade-skipping its dependents — the larger the graph, the less repair it
|
|
63
|
+
* gets, which is backwards. Scale with the node count, keeping 3 as the floor
|
|
64
|
+
* for small graphs. An explicit env value still wins outright.
|
|
65
|
+
*/
|
|
66
|
+
export function resolveFixBudget(env = process.env, nodeCount = 0) {
|
|
54
67
|
const raw = env.ZELARI_KRAKEN_FIX_BUDGET;
|
|
68
|
+
if (raw === undefined || raw === '') {
|
|
69
|
+
return Math.max(DEFAULT_FIX_BUDGET, Math.ceil(nodeCount / 2));
|
|
70
|
+
}
|
|
71
|
+
const n = Number.parseInt(raw, 10);
|
|
72
|
+
return Number.isFinite(n) && n >= 0
|
|
73
|
+
? n
|
|
74
|
+
: Math.max(DEFAULT_FIX_BUDGET, Math.ceil(nodeCount / 2));
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Default rework rounds per writer when its `verify` returns FAIL.
|
|
78
|
+
*
|
|
79
|
+
* One. A rework round is a full re-run of a writer plus a fresh verification,
|
|
80
|
+
* so the cost is roughly a doubling of that branch; and the second opinion of
|
|
81
|
+
* a model that just judged its own sibling's work has sharply diminishing
|
|
82
|
+
* value. Raise it deliberately (and with a wall-clock budget set) rather than
|
|
83
|
+
* by default.
|
|
84
|
+
*/
|
|
85
|
+
export const DEFAULT_MAX_REVIEW_ROUNDS = 1;
|
|
86
|
+
export function resolveMaxReviewRounds(env = process.env) {
|
|
87
|
+
const raw = env.ZELARI_KRAKEN_MAX_REVIEW_ROUNDS;
|
|
55
88
|
if (raw === undefined || raw === '')
|
|
56
|
-
return
|
|
89
|
+
return DEFAULT_MAX_REVIEW_ROUNDS;
|
|
57
90
|
const n = Number.parseInt(raw, 10);
|
|
58
|
-
return Number.isFinite(n) && n >= 0 ? n :
|
|
91
|
+
return Number.isFinite(n) && n >= 0 ? n : DEFAULT_MAX_REVIEW_ROUNDS;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Wall-clock bound on the WHOLE graph run (ms); 0 (the default) disables it.
|
|
95
|
+
*
|
|
96
|
+
* Per-node timeouts bound each tentacle but say nothing about the total: a
|
|
97
|
+
* wide graph, plus retries, plus fix nodes, plus rework rounds, can run far
|
|
98
|
+
* longer than any single node's budget without anything noticing. On expiry
|
|
99
|
+
* the run takes the ordinary cancellation path, so it still settles and still
|
|
100
|
+
* prints its digest.
|
|
101
|
+
*/
|
|
102
|
+
export const DEFAULT_GRAPH_TIMEOUT_MS = 0;
|
|
103
|
+
export function resolveGraphTimeoutMs(env = process.env) {
|
|
104
|
+
const raw = env.ZELARI_KRAKEN_GRAPH_TIMEOUT_MS;
|
|
105
|
+
if (raw === undefined || raw === '')
|
|
106
|
+
return DEFAULT_GRAPH_TIMEOUT_MS;
|
|
107
|
+
const n = Number.parseInt(raw, 10);
|
|
108
|
+
return Number.isFinite(n) && n >= 0 ? n : DEFAULT_GRAPH_TIMEOUT_MS;
|
|
59
109
|
}
|
|
60
110
|
/**
|
|
61
111
|
* Default wall-clock bound per tentacle run (ms). The graph executor calls
|
|
@@ -82,6 +132,20 @@ export const DEFAULT_NODE_TIMEOUT_MS = 300_000;
|
|
|
82
132
|
* cascade-skipped dependents with them.
|
|
83
133
|
*/
|
|
84
134
|
export const DEFAULT_WRITER_NODE_TIMEOUT_MS = 900_000;
|
|
135
|
+
/**
|
|
136
|
+
* How long to wait for a cancelled tentacle to actually unwind before
|
|
137
|
+
* declaring it unstoppable. Cancellation lands at the sub-agent's next event
|
|
138
|
+
* boundary, so this only needs to cover one in-flight provider chunk or tool
|
|
139
|
+
* call, not a whole turn.
|
|
140
|
+
*/
|
|
141
|
+
export const DEFAULT_CANCEL_GRACE_MS = 30_000;
|
|
142
|
+
export function resolveCancelGraceMs(env = process.env) {
|
|
143
|
+
const raw = env.ZELARI_KRAKEN_CANCEL_GRACE_MS;
|
|
144
|
+
if (raw === undefined || raw === '')
|
|
145
|
+
return DEFAULT_CANCEL_GRACE_MS;
|
|
146
|
+
const n = Number.parseInt(raw, 10);
|
|
147
|
+
return Number.isFinite(n) && n >= 0 ? n : DEFAULT_CANCEL_GRACE_MS;
|
|
148
|
+
}
|
|
85
149
|
/**
|
|
86
150
|
* Wall-clock bound for one tentacle. `ZELARI_KRAKEN_NODE_TIMEOUT_MS` overrides
|
|
87
151
|
* every kind (single knob, unchanged semantics: `0` disables);
|
|
@@ -128,39 +192,204 @@ function defaultChecksExists(cwd) {
|
|
|
128
192
|
return false;
|
|
129
193
|
}
|
|
130
194
|
}
|
|
195
|
+
/**
|
|
196
|
+
* Per-dependency cap on injected upstream text. Mirrors `MAX_PRIOR_CHARS` in
|
|
197
|
+
* the council path (`packages/core/src/agents/councilApi.ts`), for the same
|
|
198
|
+
* reason: one verbose tentacle must not be able to saturate every downstream
|
|
199
|
+
* one.
|
|
200
|
+
*/
|
|
201
|
+
export const MAX_UPSTREAM_CHARS_PER_DEP = 2800;
|
|
202
|
+
/** Total cap across all deps, so a wide fan-in cannot saturate a sub-agent. */
|
|
203
|
+
export const MAX_UPSTREAM_CHARS_TOTAL = 8000;
|
|
204
|
+
/**
|
|
205
|
+
* Render the conclusions of a node's already-completed dependencies as a
|
|
206
|
+
* prompt section for the sub-agent about to run.
|
|
207
|
+
*
|
|
208
|
+
* Without this the dependency edges were pure ordering constraints: a tentacle
|
|
209
|
+
* received only the prompt the PLANNER wrote before anything had executed, so
|
|
210
|
+
* an `explore` node's findings were computed, stored on the node, and then
|
|
211
|
+
* thrown away — the `general` nodes it "fed" started from zero and re-derived
|
|
212
|
+
* the same context (or guessed). The auto-injected `verify` node was worse
|
|
213
|
+
* still: it knew only the label of the work it was supposed to check.
|
|
214
|
+
*
|
|
215
|
+
* Direct dependencies only, deliberately — the transitive closure of a wide
|
|
216
|
+
* DAG would blow the sub-agent's context, and each hop's own conclusion is
|
|
217
|
+
* expected to carry forward what mattered.
|
|
218
|
+
*/
|
|
219
|
+
export function buildUpstreamContext(graph, node) {
|
|
220
|
+
const parts = [];
|
|
221
|
+
const omitted = [];
|
|
222
|
+
let budget = MAX_UPSTREAM_CHARS_TOTAL;
|
|
223
|
+
for (const depId of node.deps) {
|
|
224
|
+
const dep = graph.nodes.get(depId);
|
|
225
|
+
if (!dep || dep.status !== 'done')
|
|
226
|
+
continue;
|
|
227
|
+
const raw = (dep.result ?? '').trim();
|
|
228
|
+
if (!raw)
|
|
229
|
+
continue;
|
|
230
|
+
const cap = Math.min(MAX_UPSTREAM_CHARS_PER_DEP, budget);
|
|
231
|
+
if (cap <= 0) {
|
|
232
|
+
omitted.push(dep.label);
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const body = raw.length > cap
|
|
236
|
+
? `${raw.slice(0, cap)}\n… [truncated ${raw.length}→${cap} chars]`
|
|
237
|
+
: raw;
|
|
238
|
+
budget -= Math.min(raw.length, cap);
|
|
239
|
+
const scope = dep.scope && dep.scope.length > 0 ? `, scope: ${dep.scope.join(', ')}` : '';
|
|
240
|
+
parts.push(`### ${dep.label} (${dep.kind}${scope})\n${body}`);
|
|
241
|
+
}
|
|
242
|
+
if (parts.length === 0)
|
|
243
|
+
return '';
|
|
244
|
+
const lines = [
|
|
245
|
+
'',
|
|
246
|
+
'## Context from completed upstream tasks',
|
|
247
|
+
'Results reported by the tasks this one depends on. Treat them as ' +
|
|
248
|
+
'hypotheses — prefer the actual files on disk where they conflict.',
|
|
249
|
+
'',
|
|
250
|
+
...parts,
|
|
251
|
+
];
|
|
252
|
+
if (omitted.length > 0) {
|
|
253
|
+
lines.push('', `(omitted for context budget: ${omitted.join(', ')})`);
|
|
254
|
+
}
|
|
255
|
+
return lines.join('\n');
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Tool budget per node kind. Writers do real, multi-file work and were being
|
|
259
|
+
* run at the same `medium` budget as a read-only lookup; readers stay tight
|
|
260
|
+
* because a wide budget just invites them to dump the repository.
|
|
261
|
+
*/
|
|
262
|
+
export function thoroughnessForKind(kind) {
|
|
263
|
+
return kind === 'general' || kind === 'fix' ? 'deep' : 'medium';
|
|
264
|
+
}
|
|
265
|
+
/** First non-empty line of a block of text, capped — for one-line markers. */
|
|
266
|
+
function firstLine(text, maxChars = 160) {
|
|
267
|
+
const line = text.trim().split('\n').find((l) => l.trim() !== '')?.trim() ?? '';
|
|
268
|
+
return line.length > maxChars ? `${line.slice(0, maxChars)}…` : line;
|
|
269
|
+
}
|
|
270
|
+
/** The tentacle kind a node runs as (`fix`/`merge` are driven as `general`). */
|
|
271
|
+
function agentForNode(node) {
|
|
272
|
+
return node.kind === 'explore' || node.kind === 'verify' ? node.kind : 'general';
|
|
273
|
+
}
|
|
131
274
|
export class KrakenGraphExecutor {
|
|
132
275
|
deps;
|
|
133
276
|
parentCwd;
|
|
134
277
|
sessionId;
|
|
278
|
+
goal;
|
|
135
279
|
maxParallel;
|
|
136
280
|
/** Explicit all-kinds override; when undefined the budget is per-kind. */
|
|
137
281
|
nodeTimeoutMs;
|
|
282
|
+
cancelGraceMs;
|
|
283
|
+
/** Explicit override; when undefined the budget scales with the graph size. */
|
|
284
|
+
fixBudgetOption;
|
|
138
285
|
fixBudgetRemaining;
|
|
286
|
+
maxReviewRounds;
|
|
287
|
+
graphTimeoutMs;
|
|
139
288
|
worldModelGateOverride;
|
|
140
289
|
runTentacleFn;
|
|
141
290
|
mergeFn;
|
|
142
291
|
backtestFn;
|
|
143
292
|
nodeRunState = new Map();
|
|
293
|
+
/** fix node id → id of the failed node it was spawned to repair. */
|
|
294
|
+
repairs = new Map();
|
|
295
|
+
/**
|
|
296
|
+
* rework node id → id of the writer whose work it is redoing. A rework is a
|
|
297
|
+
* `fix` node, but unlike a repair it must NOT create a worktree of its own:
|
|
298
|
+
* it edits the writer's existing one (see {@link spawnReworkPair}).
|
|
299
|
+
*/
|
|
300
|
+
reworks = new Map();
|
|
301
|
+
/** lineage root writer id → rework rounds already spent on that lineage. */
|
|
302
|
+
reviewRounds = new Map();
|
|
303
|
+
/**
|
|
304
|
+
* rework node id → the ORIGINAL writer its lineage started from.
|
|
305
|
+
*
|
|
306
|
+
* The budget has to be per lineage, not per node: a rework is itself a
|
|
307
|
+
* writer, so counting rounds against the node being reworked reset the
|
|
308
|
+
* counter every round and the graph chained rework → verify → rework
|
|
309
|
+
* forever, terminating only on the scheduler's iteration cap.
|
|
310
|
+
*/
|
|
311
|
+
reviewLineage = new Map();
|
|
312
|
+
/** Verify verdicts left unresolved when the run ends. */
|
|
313
|
+
unresolved = [];
|
|
314
|
+
/** Live cancellation handles for the tentacles currently running. */
|
|
315
|
+
nodeControllers = new Map();
|
|
316
|
+
/** Wall-clock duration of each node's last run, by node id. */
|
|
317
|
+
durationsMs = new Map();
|
|
318
|
+
signal;
|
|
319
|
+
/** Set once the run has been cancelled: stops admission, retries and fixes. */
|
|
320
|
+
aborted = false;
|
|
144
321
|
fixCounter = 0;
|
|
145
322
|
constructor(opts) {
|
|
146
323
|
this.deps = opts.taskToolDeps;
|
|
147
324
|
this.parentCwd = opts.parentCwd;
|
|
148
325
|
this.sessionId = opts.sessionId;
|
|
326
|
+
this.goal = opts.goal;
|
|
149
327
|
this.maxParallel = opts.maxParallel ?? resolveMaxParallel();
|
|
150
328
|
this.nodeTimeoutMs = opts.nodeTimeoutMs;
|
|
329
|
+
this.cancelGraceMs = opts.cancelGraceMs;
|
|
330
|
+
this.fixBudgetOption = opts.fixBudget;
|
|
331
|
+
// Provisional: `execute()` re-resolves it once the graph size is known.
|
|
151
332
|
this.fixBudgetRemaining = opts.fixBudget ?? resolveFixBudget();
|
|
333
|
+
this.maxReviewRounds = opts.maxReviewRounds ?? resolveMaxReviewRounds();
|
|
334
|
+
this.graphTimeoutMs = opts.graphTimeoutMs ?? resolveGraphTimeoutMs();
|
|
152
335
|
this.worldModelGateOverride = opts.worldModelGate;
|
|
336
|
+
this.signal = opts.signal;
|
|
153
337
|
this.runTentacleFn = opts.runTentacleFn ?? runTentacle;
|
|
154
338
|
this.mergeFn = opts.mergeFn ?? mergeKrakenWorktree;
|
|
155
339
|
this.backtestFn = opts.backtestFn ?? runBacktest;
|
|
156
340
|
}
|
|
157
341
|
/** Execute the graph in place (mutates node statuses) until it settles. */
|
|
158
342
|
async execute(graph) {
|
|
343
|
+
// Now that the graph is known, size the repair budget to it (unless the
|
|
344
|
+
// caller pinned one). Done here rather than in the constructor because the
|
|
345
|
+
// node count is the whole input to the decision.
|
|
346
|
+
if (this.fixBudgetOption === undefined) {
|
|
347
|
+
this.fixBudgetRemaining = resolveFixBudget(process.env, graph.nodes.size);
|
|
348
|
+
}
|
|
349
|
+
// Cancel eagerly rather than at the next loop turn: without a listener the
|
|
350
|
+
// scheduler would sit in `Promise.race` until some node finished on its
|
|
351
|
+
// own, which for a stuck writer is the whole point of cancelling.
|
|
352
|
+
const onAbort = () => this.cancelRun();
|
|
353
|
+
if (this.signal) {
|
|
354
|
+
if (this.signal.aborted)
|
|
355
|
+
this.aborted = true;
|
|
356
|
+
else
|
|
357
|
+
this.signal.addEventListener('abort', onAbort, { once: true });
|
|
358
|
+
}
|
|
359
|
+
// Whole-run wall-clock bound. Reuses the cancellation path rather than
|
|
360
|
+
// inventing a second way to stop, so the run still settles and still
|
|
361
|
+
// reports.
|
|
362
|
+
let graphTimer;
|
|
363
|
+
if (this.graphTimeoutMs > 0) {
|
|
364
|
+
graphTimer = setTimeout(() => {
|
|
365
|
+
this.radio('graph_failed', {
|
|
366
|
+
description: 'graph executor',
|
|
367
|
+
detail: `graph exceeded its ${this.graphTimeoutMs}ms wall-clock budget — cancelling`,
|
|
368
|
+
ok: false,
|
|
369
|
+
});
|
|
370
|
+
this.cancelRun();
|
|
371
|
+
}, this.graphTimeoutMs);
|
|
372
|
+
// Do not hold the process open just to fire a cancellation.
|
|
373
|
+
graphTimer.unref?.();
|
|
374
|
+
}
|
|
375
|
+
try {
|
|
376
|
+
return await this.schedule(graph);
|
|
377
|
+
}
|
|
378
|
+
finally {
|
|
379
|
+
if (graphTimer)
|
|
380
|
+
clearTimeout(graphTimer);
|
|
381
|
+
this.signal?.removeEventListener('abort', onAbort);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
/** The scheduling loop proper. See {@link execute} for the cancellation wrapper. */
|
|
385
|
+
async schedule(graph) {
|
|
159
386
|
// Anti-explosion / self-inflicted-hang guard: cap total loop iterations
|
|
160
387
|
// at a generous multiple of the node count instead of trusting the
|
|
161
388
|
// graph to always shrink monotonically (fix-node spawns grow it).
|
|
162
389
|
const maxIterations = Math.max(64, graph.nodes.size * 8);
|
|
163
390
|
let iterations = 0;
|
|
391
|
+
/** Node id → its running tentacle. One entry per in-flight node. */
|
|
392
|
+
const inFlight = new Map();
|
|
164
393
|
startKrakenGraphLive(graph);
|
|
165
394
|
while (!isSettled(graph)) {
|
|
166
395
|
iterations += 1;
|
|
@@ -172,30 +401,66 @@ export class KrakenGraphExecutor {
|
|
|
172
401
|
});
|
|
173
402
|
break;
|
|
174
403
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
404
|
+
if (this.aborted && inFlight.size === 0)
|
|
405
|
+
break;
|
|
406
|
+
// Admit everything that can start alongside what is already running,
|
|
407
|
+
// up to the concurrency cap. A cancelled run admits nothing.
|
|
408
|
+
const admitted = this.aborted ? [] : this.admit(graph, inFlight);
|
|
409
|
+
if (admitted.length > 0) {
|
|
410
|
+
// Mark and publish the whole admission BEFORE starting any of it: an
|
|
411
|
+
// async function body runs synchronously up to its first await, so
|
|
412
|
+
// starting a tentacle first would let it observe stale counts. (The
|
|
413
|
+
// StatusBar's "n↑" never appeared at all while this was sampled only
|
|
414
|
+
// after the work had already settled.)
|
|
415
|
+
for (const node of admitted)
|
|
416
|
+
node.status = 'running';
|
|
417
|
+
updateKrakenGraphLive(graph);
|
|
418
|
+
for (const node of admitted)
|
|
419
|
+
inFlight.set(node.id, this.runNodeSafely(node, graph));
|
|
420
|
+
}
|
|
421
|
+
if (inFlight.size === 0) {
|
|
422
|
+
// Nothing running and nothing admissible: the remaining pending nodes
|
|
423
|
+
// are permanently blocked by a failed/skipped dependency (or by the
|
|
424
|
+
// run being cancelled). Cascade skip them so the loop can terminate.
|
|
425
|
+
if (!this.skipBlockedNodes(graph)) {
|
|
426
|
+
// Nothing ready, nothing running, nothing to skip — should not
|
|
427
|
+
// happen for a validated DAG, but bail out rather than spin forever.
|
|
184
428
|
break;
|
|
185
429
|
}
|
|
186
430
|
continue;
|
|
187
431
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
432
|
+
// Settle ONE node at a time. Waiting for a whole wave meant a 15-minute
|
|
433
|
+
// writer held back every explore that became ready a second later, and
|
|
434
|
+
// left the concurrency budget idle for the duration.
|
|
435
|
+
const { id, res } = await Promise.race(inFlight.values());
|
|
436
|
+
inFlight.delete(id);
|
|
437
|
+
const settledNode = graph.nodes.get(id);
|
|
438
|
+
if (settledNode)
|
|
439
|
+
this.applyResult(graph, settledNode, res);
|
|
195
440
|
updateKrakenGraphLive(graph);
|
|
196
441
|
}
|
|
442
|
+
// The loop can break with work still in flight (iteration cap, or a
|
|
443
|
+
// cancellation whose tentacles have not unwound yet). Let those settle
|
|
444
|
+
// rather than returning a summary while they are still writing, and
|
|
445
|
+
// record what they produced.
|
|
446
|
+
if (inFlight.size > 0) {
|
|
447
|
+
for (const { id, res } of await Promise.all(inFlight.values())) {
|
|
448
|
+
const node = graph.nodes.get(id);
|
|
449
|
+
if (node)
|
|
450
|
+
this.applyResult(graph, node, res);
|
|
451
|
+
}
|
|
452
|
+
inFlight.clear();
|
|
453
|
+
}
|
|
454
|
+
// A cancelled run leaves nodes that never started: `skipped` says exactly
|
|
455
|
+
// that, and lets the graph settle so a summary can be returned.
|
|
456
|
+
if (this.aborted) {
|
|
457
|
+
for (const n of graph.nodes.values()) {
|
|
458
|
+
if (n.status === 'pending')
|
|
459
|
+
n.status = 'skipped';
|
|
460
|
+
}
|
|
461
|
+
}
|
|
197
462
|
let backtest;
|
|
198
|
-
const converged = isConverged(graph);
|
|
463
|
+
const converged = !this.aborted && isConverged(graph);
|
|
199
464
|
if (converged) {
|
|
200
465
|
const gateOn = this.worldModelGateOverride ?? isWorldModelGateEnabled(this.parentCwd);
|
|
201
466
|
if (gateOn) {
|
|
@@ -210,45 +475,153 @@ export class KrakenGraphExecutor {
|
|
|
210
475
|
else {
|
|
211
476
|
this.radio('graph_failed', {
|
|
212
477
|
description: 'graph executor',
|
|
213
|
-
detail:
|
|
478
|
+
detail: this.aborted
|
|
479
|
+
? 'cancelled by caller'
|
|
480
|
+
: `failed nodes: ${failedNodeIds(graph).join(', ') || 'none'}`,
|
|
214
481
|
ok: false,
|
|
215
482
|
});
|
|
216
483
|
}
|
|
217
484
|
endKrakenGraphLive(graph, converged);
|
|
485
|
+
// Cross-run memory: let the next planning pass see where this one stopped
|
|
486
|
+
// instead of replanning the whole goal blind.
|
|
487
|
+
await saveGraphSnapshot(this.parentCwd, toGraphSnapshot(graph, {
|
|
488
|
+
goal: this.goal ?? graph.id,
|
|
489
|
+
converged,
|
|
490
|
+
unresolvedFindings: this.unresolved,
|
|
491
|
+
}));
|
|
218
492
|
return {
|
|
219
493
|
graph,
|
|
220
494
|
converged,
|
|
221
495
|
failedNodeIds: failedNodeIds(graph),
|
|
222
496
|
counts: countByStatus(graph),
|
|
497
|
+
durationsMs: Object.fromEntries(this.durationsMs),
|
|
498
|
+
cancelled: this.aborted,
|
|
499
|
+
unresolvedFindings: [...this.unresolved],
|
|
223
500
|
...(backtest ? { backtest } : {}),
|
|
224
501
|
};
|
|
225
502
|
}
|
|
503
|
+
/**
|
|
504
|
+
* Stop the run: no further admissions, and every tentacle currently running
|
|
505
|
+
* is told to unwind. Each node's own timeout/grace machinery then resolves
|
|
506
|
+
* it, so `execute()` settles instead of leaving orphans behind.
|
|
507
|
+
*/
|
|
508
|
+
cancelRun() {
|
|
509
|
+
if (this.aborted)
|
|
510
|
+
return;
|
|
511
|
+
this.aborted = true;
|
|
512
|
+
this.radio('graph_failed', {
|
|
513
|
+
description: 'graph executor',
|
|
514
|
+
detail: `cancelling ${this.nodeControllers.size} running tentacle(s)`,
|
|
515
|
+
ok: false,
|
|
516
|
+
});
|
|
517
|
+
for (const controller of this.nodeControllers.values())
|
|
518
|
+
controller.abort();
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Pick the ready nodes that may start right now: parallel-safe against every
|
|
522
|
+
* node already running AND against each other, within the concurrency cap.
|
|
523
|
+
*
|
|
524
|
+
* Unlike a wave-at-a-time scheduler this is called on every completion, so a
|
|
525
|
+
* node becomes eligible the moment its blocker settles instead of waiting
|
|
526
|
+
* for the slowest member of some earlier batch.
|
|
527
|
+
*/
|
|
528
|
+
admit(graph, inFlight) {
|
|
529
|
+
const capacity = this.maxParallel - inFlight.size;
|
|
530
|
+
if (capacity <= 0)
|
|
531
|
+
return [];
|
|
532
|
+
const running = [];
|
|
533
|
+
for (const id of inFlight.keys()) {
|
|
534
|
+
const n = graph.nodes.get(id);
|
|
535
|
+
if (n)
|
|
536
|
+
running.push(n);
|
|
537
|
+
}
|
|
538
|
+
const admitted = [];
|
|
539
|
+
for (const node of getReadyNodes(graph)) {
|
|
540
|
+
if (admitted.length >= capacity)
|
|
541
|
+
break;
|
|
542
|
+
const safe = running.every((r) => canRunParallel(r, node)) &&
|
|
543
|
+
admitted.every((a) => canRunParallel(a, node));
|
|
544
|
+
if (safe)
|
|
545
|
+
admitted.push(node);
|
|
546
|
+
}
|
|
547
|
+
return admitted;
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Run one node, tagging the result with its id and converting an unexpected
|
|
551
|
+
* throw into a node failure. The scheduler races these promises, so a
|
|
552
|
+
* rejection would abandon every other in-flight tentacle mid-write; one
|
|
553
|
+
* failed node that the retry/fix machinery can reason about is strictly
|
|
554
|
+
* better than an aborted graph.
|
|
555
|
+
*/
|
|
556
|
+
runNodeSafely(node, graph) {
|
|
557
|
+
return this.runNode(node, graph).then((res) => ({ id: node.id, res }), (err) => ({
|
|
558
|
+
id: node.id,
|
|
559
|
+
res: {
|
|
560
|
+
ok: false,
|
|
561
|
+
agent: agentForNode(node),
|
|
562
|
+
error: `tentacle threw: ${err instanceof Error ? err.message : String(err)}`,
|
|
563
|
+
cancelled: true,
|
|
564
|
+
},
|
|
565
|
+
}));
|
|
566
|
+
}
|
|
226
567
|
/** Run one node: dispatch to the merge handler for `merge` nodes, else a tentacle. */
|
|
227
568
|
async runNode(node, graph) {
|
|
569
|
+
const startedAt = Date.now();
|
|
570
|
+
try {
|
|
571
|
+
return await this.runNodeInner(node, graph);
|
|
572
|
+
}
|
|
573
|
+
finally {
|
|
574
|
+
this.durationsMs.set(node.id, Date.now() - startedAt);
|
|
575
|
+
this.nodeControllers.delete(node.id);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
async runNodeInner(node, graph) {
|
|
228
579
|
this.radio('node_start', { description: node.label, agent: node.kind });
|
|
229
580
|
if (node.kind === 'merge') {
|
|
230
581
|
return this.runMergeNode(node, graph);
|
|
231
582
|
}
|
|
232
|
-
|
|
583
|
+
// A rework edits the worktree its writer already produced (see
|
|
584
|
+
// spawnReworkPair) — it must not open a second one on the same scope.
|
|
585
|
+
const isRework = this.reworks.has(node.id);
|
|
586
|
+
const usesWorktree = (node.kind === 'general' || node.kind === 'fix') && !isRework;
|
|
233
587
|
const agent = node.kind === 'fix' ? 'general' : node.kind;
|
|
588
|
+
const controller = new AbortController();
|
|
589
|
+
// Registered so a cancelled run can reach in and stop this tentacle
|
|
590
|
+
// instead of waiting for it to finish on its own.
|
|
591
|
+
this.nodeControllers.set(node.id, controller);
|
|
592
|
+
if (this.aborted)
|
|
593
|
+
controller.abort();
|
|
594
|
+
// Hand the sub-agent what its dependencies actually concluded — the whole
|
|
595
|
+
// point of the dep edge (see buildUpstreamContext).
|
|
596
|
+
const upstream = buildUpstreamContext(graph, node);
|
|
597
|
+
// A verify inspects the tree its writer wrote in; a rework edits that same
|
|
598
|
+
// tree. Both resolve to the worktree recorded behind their deps.
|
|
599
|
+
const inheritedCwd = node.kind === 'verify' || isRework
|
|
600
|
+
? this.inheritedWorktreeCwdFor(node, graph)
|
|
601
|
+
: undefined;
|
|
234
602
|
const res = await this.withNodeTimeout(this.runTentacleFn({
|
|
235
|
-
|
|
603
|
+
// `allowWorktree: false` is what actually stops a rework from opening
|
|
604
|
+
// its own worktree: creation is driven by the agent kind ('general')
|
|
605
|
+
// inside runTentacle, not by anything the executor passes per-call.
|
|
606
|
+
deps: isRework ? { ...this.deps, allowWorktree: false } : this.deps,
|
|
236
607
|
args: {
|
|
237
608
|
description: node.label,
|
|
238
|
-
prompt: node.prompt,
|
|
609
|
+
prompt: upstream ? `${node.prompt}\n${upstream}` : node.prompt,
|
|
239
610
|
scope: node.scope,
|
|
240
611
|
acceptance: node.acceptance,
|
|
241
612
|
},
|
|
242
613
|
agent,
|
|
243
|
-
thoroughness:
|
|
614
|
+
thoroughness: thoroughnessForKind(node.kind),
|
|
244
615
|
parentCwd: this.parentCwd,
|
|
616
|
+
...(inheritedCwd ? { cwdOverride: inheritedCwd } : {}),
|
|
245
617
|
sessionId: this.sessionId,
|
|
246
618
|
// Defer merge for writers so the executor controls merge ordering
|
|
247
|
-
// (Correction 4); explore/verify never create a worktree.
|
|
619
|
+
// (Correction 4); explore/verify/rework never create a worktree.
|
|
248
620
|
deferMerge: usesWorktree,
|
|
249
621
|
graphId: graph.id,
|
|
250
622
|
nodeId: node.id,
|
|
251
|
-
|
|
623
|
+
signal: controller.signal,
|
|
624
|
+
}), agent, controller);
|
|
252
625
|
if (res.ok && usesWorktree) {
|
|
253
626
|
this.nodeRunState.set(node.id, { worktreeHandle: res.worktreeHandle });
|
|
254
627
|
}
|
|
@@ -256,44 +629,136 @@ export class KrakenGraphExecutor {
|
|
|
256
629
|
}
|
|
257
630
|
/**
|
|
258
631
|
* Bound a tentacle run to its wall-clock budget (explicit `nodeTimeoutMs`
|
|
259
|
-
* option, else per-kind — writers get more than readers). On timeout
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
632
|
+
* option, else per-kind — writers get more than readers). On timeout the
|
|
633
|
+
* run is CANCELLED via its AbortSignal, then given `cancelGraceMs` to
|
|
634
|
+
* unwind. Cancellation lands at the sub-agent's next event boundary (an
|
|
635
|
+
* async generator only observes `.return()` when it next yields), so a run
|
|
636
|
+
* blocked on a slow provider call can outlive the grace period — that case
|
|
637
|
+
* resolves with `cancelled: false` and `applyResult` refuses to re-spawn
|
|
638
|
+
* the node, since a tentacle that may still be writing must not be joined
|
|
639
|
+
* by a second one on the same scope.
|
|
640
|
+
*
|
|
641
|
+
* Either way `execute()` always settles, so the caller's process can exit
|
|
642
|
+
* instead of hanging forever on one stuck node.
|
|
265
643
|
*/
|
|
266
|
-
withNodeTimeout(promise, agent) {
|
|
644
|
+
async withNodeTimeout(promise, agent, controller) {
|
|
267
645
|
const ms = this.nodeTimeoutMs ?? resolveNodeTimeoutMs(process.env, agent);
|
|
268
646
|
if (ms <= 0)
|
|
269
647
|
return promise;
|
|
270
648
|
let timer;
|
|
649
|
+
const TIMED_OUT = Symbol('timeout');
|
|
271
650
|
const timeout = new Promise((resolve) => {
|
|
272
|
-
timer = setTimeout(() =>
|
|
273
|
-
resolve({ ok: false, agent, error: `tentacle timed out after ${ms}ms` });
|
|
274
|
-
}, ms);
|
|
651
|
+
timer = setTimeout(() => resolve(TIMED_OUT), ms);
|
|
275
652
|
});
|
|
276
|
-
|
|
653
|
+
const raced = await Promise.race([promise, timeout]).finally(() => {
|
|
277
654
|
if (timer)
|
|
278
655
|
clearTimeout(timer);
|
|
279
656
|
});
|
|
657
|
+
if (raced !== TIMED_OUT)
|
|
658
|
+
return raced;
|
|
659
|
+
// Budget blown. Tell the tentacle to stop, then give it a bounded grace
|
|
660
|
+
// period to actually unwind — a run that has already been told to stop is
|
|
661
|
+
// safe to re-run, one that is still going is not (two agents writing the
|
|
662
|
+
// same scope is exactly the corruption this guards against).
|
|
663
|
+
controller?.abort();
|
|
664
|
+
const graceMs = this.cancelGraceMs ?? resolveCancelGraceMs();
|
|
665
|
+
let graceTimer;
|
|
666
|
+
const grace = new Promise((resolve) => {
|
|
667
|
+
graceTimer = setTimeout(() => resolve(TIMED_OUT), graceMs);
|
|
668
|
+
});
|
|
669
|
+
const settled = await Promise.race([promise, grace]).finally(() => {
|
|
670
|
+
if (graceTimer)
|
|
671
|
+
clearTimeout(graceTimer);
|
|
672
|
+
});
|
|
673
|
+
if (settled !== TIMED_OUT) {
|
|
674
|
+
// The run is over, so the scope is free and a retry/fix would be safe.
|
|
675
|
+
// If it actually SUCCEEDED — finishing in the window between the
|
|
676
|
+
// deadline and the abort landing — keep that result rather than
|
|
677
|
+
// discarding completed (and already written) work over a few ms.
|
|
678
|
+
if (settled.ok)
|
|
679
|
+
return settled;
|
|
680
|
+
return { ...settled, cancelled: true };
|
|
681
|
+
}
|
|
682
|
+
// Did not stop in time. Report it as uncancelled so `applyResult` refuses
|
|
683
|
+
// to re-spawn this node — better one failed node than two concurrent
|
|
684
|
+
// writers on one directory.
|
|
685
|
+
return {
|
|
686
|
+
ok: false,
|
|
687
|
+
agent,
|
|
688
|
+
error: `tentacle timed out after ${ms}ms and did not stop within ${graceMs}ms of being cancelled; ` +
|
|
689
|
+
`not retrying to avoid two tentacles writing the same scope`,
|
|
690
|
+
cancelled: false,
|
|
691
|
+
};
|
|
280
692
|
}
|
|
281
693
|
/**
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
*
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
694
|
+
* The worktree a node should run in, inherited from the writer behind it.
|
|
695
|
+
*
|
|
696
|
+
* For a `verify`: verification happens BEFORE the merge node, so when its
|
|
697
|
+
* writer worked in an isolated worktree the changes are not in the parent
|
|
698
|
+
* tree yet — a verify tentacle pointed at `parentCwd` was inspecting a tree
|
|
699
|
+
* that provably did not contain the work it was asked to check, and reported
|
|
700
|
+
* it missing.
|
|
701
|
+
*
|
|
702
|
+
* For a rework: the same tree, for the stronger reason that writing anywhere
|
|
703
|
+
* else would strand the round on a second branch.
|
|
704
|
+
*
|
|
705
|
+
* Returns undefined when there is no single tree: no worktrees (isolation
|
|
706
|
+
* disabled — the writers edited the parent tree directly), or several
|
|
707
|
+
* distinct ones, in which case no single cwd is correct and the parent tree
|
|
708
|
+
* is the honest default.
|
|
709
|
+
*/
|
|
710
|
+
inheritedWorktreeCwdFor(node, graph) {
|
|
711
|
+
const paths = new Set(this.collectWorktreeSources(node, graph).map((s) => s.handle.path));
|
|
712
|
+
return paths.size === 1 ? [...paths][0] : undefined;
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Resolve the deferred worktrees produced behind a node's dependencies, in
|
|
716
|
+
* ancestors-first order. Used to decide what a `merge` node must merge, and
|
|
717
|
+
* which tree a `verify` node should actually inspect.
|
|
718
|
+
*
|
|
719
|
+
* A merge node's direct deps are NOT the writers: `buildGraphFromPlan`
|
|
720
|
+
* injects a `verify` node after every `general` node and points the merge at
|
|
721
|
+
* those verifies, while worktree handles are recorded against the writer
|
|
722
|
+
* node ids. Looking only at direct deps therefore found nothing to merge and
|
|
723
|
+
* silently reported success while every tentacle's work stayed stranded on
|
|
724
|
+
* its branch. Walk up through non-writer deps until the writers are found.
|
|
725
|
+
*
|
|
726
|
+
* Post-order so a writer that depends on another writer merges after it (the
|
|
727
|
+
* later branch was cut from a HEAD that already contained the earlier work).
|
|
728
|
+
* `merge` nodes terminate the walk: another merge already owns its subtree.
|
|
729
|
+
*/
|
|
730
|
+
collectWorktreeSources(node, graph) {
|
|
731
|
+
const out = [];
|
|
732
|
+
const seen = new Set();
|
|
733
|
+
const visit = (id) => {
|
|
734
|
+
if (seen.has(id))
|
|
735
|
+
return;
|
|
736
|
+
seen.add(id);
|
|
737
|
+
const n = graph.nodes.get(id);
|
|
738
|
+
if (!n || n.kind === 'merge')
|
|
739
|
+
return;
|
|
740
|
+
for (const dep of n.deps)
|
|
741
|
+
visit(dep);
|
|
742
|
+
const handle = this.nodeRunState.get(id)?.worktreeHandle;
|
|
743
|
+
if (handle)
|
|
744
|
+
out.push({ id, handle });
|
|
745
|
+
};
|
|
746
|
+
for (const depId of node.deps)
|
|
747
|
+
visit(depId);
|
|
748
|
+
return out;
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Sequentially merge every deferred worktree this node covers (in
|
|
752
|
+
* ancestors-first order) into parent HEAD. Nodes without a recorded worktree
|
|
753
|
+
* handle (worktree isolation disabled, or a read-only node) are a no-op. On
|
|
754
|
+
* conflict the branch is kept and the conflict is surfaced in the merge
|
|
755
|
+
* node's error — remaining sources still attempt to merge (independent
|
|
756
|
+
* branches shouldn't be blocked by one conflict).
|
|
288
757
|
*/
|
|
289
758
|
async runMergeNode(node, graph) {
|
|
290
759
|
const conflicts = [];
|
|
291
760
|
const merged = [];
|
|
292
|
-
for (const depId of node
|
|
293
|
-
const state = this.nodeRunState.get(depId);
|
|
294
|
-
const handle = state?.worktreeHandle;
|
|
295
|
-
if (!handle)
|
|
296
|
-
continue; // nothing to merge for this dep
|
|
761
|
+
for (const { id: depId, handle } of this.collectWorktreeSources(node, graph)) {
|
|
297
762
|
let result;
|
|
298
763
|
try {
|
|
299
764
|
result = await this.mergeFn(handle, {
|
|
@@ -313,6 +778,10 @@ export class KrakenGraphExecutor {
|
|
|
313
778
|
conflicts.push(`${depId}: ${result.message}`);
|
|
314
779
|
}
|
|
315
780
|
else {
|
|
781
|
+
// The branch is merged (and, unless KEEP is set, the worktree is gone).
|
|
782
|
+
// Drop the handle so a second merge node covering the same ancestor
|
|
783
|
+
// cannot try to merge a branch that no longer exists.
|
|
784
|
+
this.nodeRunState.set(depId, { worktreeHandle: null });
|
|
316
785
|
merged.push(depId);
|
|
317
786
|
}
|
|
318
787
|
}
|
|
@@ -340,9 +809,40 @@ export class KrakenGraphExecutor {
|
|
|
340
809
|
node.status = 'done';
|
|
341
810
|
node.result = res.result;
|
|
342
811
|
this.radio('node_end', { description: node.label, agent: node.kind, ok: true });
|
|
812
|
+
this.reconcileRepairedNode(graph, node);
|
|
813
|
+
// A verify that RAN successfully is not the same thing as work that
|
|
814
|
+
// PASSED. Read what it actually concluded.
|
|
815
|
+
if (node.kind === 'verify')
|
|
816
|
+
this.applyVerifyVerdict(graph, node);
|
|
343
817
|
return;
|
|
344
818
|
}
|
|
345
819
|
node.error = res.error;
|
|
820
|
+
// A cancelled run must not retry or spawn repairs: the caller asked for
|
|
821
|
+
// the graph to stop, and re-spawning work is the opposite of that.
|
|
822
|
+
if (this.aborted) {
|
|
823
|
+
node.status = 'error';
|
|
824
|
+
this.radio('node_end', {
|
|
825
|
+
description: node.label,
|
|
826
|
+
agent: node.kind,
|
|
827
|
+
detail: res.error,
|
|
828
|
+
ok: false,
|
|
829
|
+
});
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
// A run we could not confirm has stopped may still be writing this node's
|
|
833
|
+
// scope. Re-spawning would put two tentacles in the same directory — the
|
|
834
|
+
// failure mode that produced duplicate parallel implementations of the
|
|
835
|
+
// same modules. Fail terminally instead; dependents cascade-skip.
|
|
836
|
+
if (res.cancelled === false) {
|
|
837
|
+
node.status = 'error';
|
|
838
|
+
this.radio('node_end', {
|
|
839
|
+
description: node.label,
|
|
840
|
+
agent: node.kind,
|
|
841
|
+
detail: res.error,
|
|
842
|
+
ok: false,
|
|
843
|
+
});
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
346
846
|
if (node.retryCount < node.maxRetries) {
|
|
347
847
|
node.retryCount += 1;
|
|
348
848
|
node.status = 'pending';
|
|
@@ -378,6 +878,201 @@ export class KrakenGraphExecutor {
|
|
|
378
878
|
ok: false,
|
|
379
879
|
});
|
|
380
880
|
}
|
|
881
|
+
/**
|
|
882
|
+
* A `fix` node just completed the work its failed predecessor could not.
|
|
883
|
+
* That unit of work IS done — but the predecessor was left terminally
|
|
884
|
+
* `error`, and since `isConverged` requires every node to be `done`/
|
|
885
|
+
* `skipped`, a fully repaired graph reported "did not converge" and listed
|
|
886
|
+
* the repaired node under `failedNodeIds`. The cross-run snapshot then told
|
|
887
|
+
* the next planner to redo work the fix had already completed.
|
|
888
|
+
*
|
|
889
|
+
* Marking it `done` has no scheduling effect (dependents were re-pointed at
|
|
890
|
+
* the fix when it was spawned) — it is purely how the run is reported. The
|
|
891
|
+
* original failure stays visible as the separate `fix: …` node and in the
|
|
892
|
+
* repaired node's result line.
|
|
893
|
+
*/
|
|
894
|
+
reconcileRepairedNode(graph, fixNode) {
|
|
895
|
+
const failedId = this.repairs.get(fixNode.id);
|
|
896
|
+
if (!failedId)
|
|
897
|
+
return;
|
|
898
|
+
const failed = graph.nodes.get(failedId);
|
|
899
|
+
if (!failed || failed.status !== 'error')
|
|
900
|
+
return;
|
|
901
|
+
const original = failed.error ? ` (original failure: ${failed.error})` : '';
|
|
902
|
+
failed.status = 'done';
|
|
903
|
+
failed.result = `repaired by "${fixNode.label}"${original}${fixNode.result ? `: ${fixNode.result}` : ''}`;
|
|
904
|
+
// Clear the error: the snapshot lists a `done` node under "Already
|
|
905
|
+
// completed — do NOT redo this work", and a trailing error message there
|
|
906
|
+
// reads as if it still needs repair.
|
|
907
|
+
failed.error = undefined;
|
|
908
|
+
this.radio('node_end', {
|
|
909
|
+
description: failed.label,
|
|
910
|
+
agent: failed.kind,
|
|
911
|
+
detail: `repaired by ${fixNode.id}`,
|
|
912
|
+
ok: true,
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
/**
|
|
916
|
+
* Act on what a completed `verify` node concluded.
|
|
917
|
+
*
|
|
918
|
+
* The verify itself stays `done` either way — it did its job, and doing it
|
|
919
|
+
* well means being free to say "no". A FAIL instead sends the WRITER back
|
|
920
|
+
* through a bounded rework round.
|
|
921
|
+
*
|
|
922
|
+
* Without this the verdict text was never read: a verify that reported the
|
|
923
|
+
* work as wrong was recorded exactly like one that reported it correct, the
|
|
924
|
+
* graph converged over the defect, and the only iteration the engine could
|
|
925
|
+
* do was on execution failure. An `unknown` verdict (no parseable trailer)
|
|
926
|
+
* is deliberately non-blocking — a prompt drift must not be able to wedge
|
|
927
|
+
* every graph — but it is recorded, because a gate that has silently stopped
|
|
928
|
+
* working is worse than no gate.
|
|
929
|
+
*/
|
|
930
|
+
applyVerifyVerdict(graph, verify) {
|
|
931
|
+
const { verdict, findings } = parseVerifyVerdict(verify.result);
|
|
932
|
+
if (verdict === 'pass')
|
|
933
|
+
return;
|
|
934
|
+
const writer = this.writerBehind(verify, graph);
|
|
935
|
+
if (!writer)
|
|
936
|
+
return; // nothing to send back to
|
|
937
|
+
if (verdict === 'unknown') {
|
|
938
|
+
this.unresolved.push({
|
|
939
|
+
nodeId: writer.id,
|
|
940
|
+
label: writer.label,
|
|
941
|
+
reason: 'unknown',
|
|
942
|
+
findings: findings || '(verify produced no parseable VERDICT line)',
|
|
943
|
+
});
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
// A cancelled run must not spawn new work, and a lineage that has spent
|
|
947
|
+
// its rounds is done being reworked.
|
|
948
|
+
const root = this.reviewLineage.get(writer.id) ?? writer.id;
|
|
949
|
+
const spent = this.reviewRounds.get(root) ?? 0;
|
|
950
|
+
if (this.aborted || spent >= this.maxReviewRounds) {
|
|
951
|
+
this.unresolved.push({
|
|
952
|
+
nodeId: writer.id,
|
|
953
|
+
label: writer.label,
|
|
954
|
+
reason: 'fail',
|
|
955
|
+
findings,
|
|
956
|
+
});
|
|
957
|
+
// Keep the unresolved verdict attached to the node too, so the digest,
|
|
958
|
+
// the snapshot and any reader of the graph see it in the same place the
|
|
959
|
+
// repaired-node marker lives.
|
|
960
|
+
writer.result =
|
|
961
|
+
`${writer.result ?? ''}\n\n[accepted with unresolved verify findings from ` +
|
|
962
|
+
`"${verify.label}"]${findings ? `: ${firstLine(findings)}` : ''}`.trim();
|
|
963
|
+
this.radio('node_end', {
|
|
964
|
+
description: writer.label,
|
|
965
|
+
agent: writer.kind,
|
|
966
|
+
detail: this.aborted
|
|
967
|
+
? 'verify FAIL left unresolved (run cancelled)'
|
|
968
|
+
: `verify FAIL left unresolved (rework budget ${this.maxReviewRounds} spent)`,
|
|
969
|
+
ok: false,
|
|
970
|
+
});
|
|
971
|
+
return;
|
|
972
|
+
}
|
|
973
|
+
this.reviewRounds.set(root, spent + 1);
|
|
974
|
+
this.spawnReworkPair(graph, writer, verify, findings, root, spent + 1);
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* The writer whose work a `verify` node judged.
|
|
978
|
+
*
|
|
979
|
+
* Walks up through non-writer deps, the same shape `collectWorktreeSources`
|
|
980
|
+
* relies on: a verify's dep is normally its writer directly, but after a
|
|
981
|
+
* rework round the chain is writer → verify → rework → verify, and the
|
|
982
|
+
* rework (a `fix` node) is itself the writer to send back.
|
|
983
|
+
*/
|
|
984
|
+
writerBehind(verify, graph) {
|
|
985
|
+
const seen = new Set();
|
|
986
|
+
const visit = (id) => {
|
|
987
|
+
if (seen.has(id))
|
|
988
|
+
return undefined;
|
|
989
|
+
seen.add(id);
|
|
990
|
+
const n = graph.nodes.get(id);
|
|
991
|
+
if (!n || n.kind === 'merge')
|
|
992
|
+
return undefined;
|
|
993
|
+
if (n.kind === 'general' || n.kind === 'fix')
|
|
994
|
+
return n;
|
|
995
|
+
for (const dep of n.deps) {
|
|
996
|
+
const found = visit(dep);
|
|
997
|
+
if (found)
|
|
998
|
+
return found;
|
|
999
|
+
}
|
|
1000
|
+
return undefined;
|
|
1001
|
+
};
|
|
1002
|
+
for (const depId of verify.deps) {
|
|
1003
|
+
const found = visit(depId);
|
|
1004
|
+
if (found)
|
|
1005
|
+
return found;
|
|
1006
|
+
}
|
|
1007
|
+
return undefined;
|
|
1008
|
+
}
|
|
1009
|
+
/**
|
|
1010
|
+
* Send a writer's work back for one more round: a rework node carrying the
|
|
1011
|
+
* verify's findings, plus a fresh verify to judge the result.
|
|
1012
|
+
*
|
|
1013
|
+
* The rework runs INSIDE the writer's worktree instead of creating one of
|
|
1014
|
+
* its own. Two worktrees for one scope means two branches, and the merge
|
|
1015
|
+
* node walks up to the writer — so a rework on its own branch would be
|
|
1016
|
+
* merged never or twice, exactly the stranded-work failure the merge fix
|
|
1017
|
+
* addressed. `allowWorktree: false` on this node's deps suppresses creation,
|
|
1018
|
+
* and `cwdOverride` (resolved via {@link inheritedWorktreeCwdFor}) points it
|
|
1019
|
+
* at the existing tree; the handle stays registered against the writer.
|
|
1020
|
+
*
|
|
1021
|
+
* Acyclicity is preserved by construction: both new nodes point only at
|
|
1022
|
+
* nodes that already exist, and the rewiring moves an existing edge forward
|
|
1023
|
+
* along the chain rather than back into it.
|
|
1024
|
+
*/
|
|
1025
|
+
spawnReworkPair(graph, writer, verify, findings, root, round) {
|
|
1026
|
+
// Named after the lineage root, so round 2 of g1 is `rework-g1-2` rather
|
|
1027
|
+
// than a nested `rework-rework-g1-1-1`.
|
|
1028
|
+
const reworkId = `rework-${root}-${round}`;
|
|
1029
|
+
const reworkNode = {
|
|
1030
|
+
id: reworkId,
|
|
1031
|
+
kind: 'fix',
|
|
1032
|
+
label: `rework: ${writer.label}`,
|
|
1033
|
+
prompt: `A reviewer inspected this work on disk and REJECTED it. Address every finding below, ` +
|
|
1034
|
+
`then leave the work in a state that satisfies the original task.\n\n` +
|
|
1035
|
+
`## Original task\n${writer.prompt}\n\n` +
|
|
1036
|
+
`## Reviewer findings (these are what must change)\n${findings || '(the reviewer reported FAIL without detail)'}`,
|
|
1037
|
+
...(writer.scope ? { scope: writer.scope } : {}),
|
|
1038
|
+
...(writer.acceptance ? { acceptance: writer.acceptance } : {}),
|
|
1039
|
+
// The verify is already `done`, so the rework is immediately ready.
|
|
1040
|
+
deps: [verify.id],
|
|
1041
|
+
status: 'pending',
|
|
1042
|
+
retryCount: 0,
|
|
1043
|
+
maxRetries: 0,
|
|
1044
|
+
};
|
|
1045
|
+
graph.nodes.set(reworkId, reworkNode);
|
|
1046
|
+
this.reworks.set(reworkId, writer.id);
|
|
1047
|
+
this.reviewLineage.set(reworkId, root);
|
|
1048
|
+
const reVerifyId = `verify-${reworkId}`;
|
|
1049
|
+
const reVerifyNode = {
|
|
1050
|
+
id: reVerifyId,
|
|
1051
|
+
kind: 'verify',
|
|
1052
|
+
label: `verify: ${writer.label} (rework ${round})`,
|
|
1053
|
+
prompt: verify.prompt,
|
|
1054
|
+
deps: [reworkId],
|
|
1055
|
+
status: 'pending',
|
|
1056
|
+
retryCount: 0,
|
|
1057
|
+
maxRetries: verify.maxRetries,
|
|
1058
|
+
};
|
|
1059
|
+
graph.nodes.set(reVerifyId, reVerifyNode);
|
|
1060
|
+
// Whatever waited on the old verify (typically the merge node) must now
|
|
1061
|
+
// wait on the new one, or it would merge the branch mid-rework.
|
|
1062
|
+
for (const other of graph.nodes.values()) {
|
|
1063
|
+
if (other.id === reworkId || other.id === reVerifyId)
|
|
1064
|
+
continue;
|
|
1065
|
+
if (other.deps.includes(verify.id)) {
|
|
1066
|
+
other.deps = other.deps.map((d) => (d === verify.id ? reVerifyId : d));
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
this.radio('node_fix', {
|
|
1070
|
+
description: reworkNode.label,
|
|
1071
|
+
agent: 'fix',
|
|
1072
|
+
detail: `verify FAIL on "${writer.label}" — rework round ${round}/${this.maxReviewRounds}`,
|
|
1073
|
+
ok: false,
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
381
1076
|
/**
|
|
382
1077
|
* Create a `fix` node that attempts to redo the failed node's work, wired
|
|
383
1078
|
* so downstream dependents of the failed node also wait on the fix.
|
|
@@ -400,9 +1095,12 @@ export class KrakenGraphExecutor {
|
|
|
400
1095
|
// no further retries — one fix attempt per failed node in v1
|
|
401
1096
|
};
|
|
402
1097
|
graph.nodes.set(fixId, fixNode);
|
|
1098
|
+
this.repairs.set(fixId, failed.id);
|
|
403
1099
|
// Downstream dependents wait on the fix attempt INSTEAD of the failed
|
|
404
|
-
// node (not in addition to it)
|
|
405
|
-
// so leaving it in `deps` would strand
|
|
1100
|
+
// node (not in addition to it): the failed node is terminally `error` and
|
|
1101
|
+
// will never run again, so leaving it in `deps` would strand them forever.
|
|
1102
|
+
// (If the fix succeeds, `reconcileRepairedNode` marks the failed node
|
|
1103
|
+
// `done` for reporting — after this rewiring, and with no effect on it.)
|
|
406
1104
|
for (const other of graph.nodes.values()) {
|
|
407
1105
|
if (other.id === fixId)
|
|
408
1106
|
continue;
|