zelari-code 1.27.2 → 1.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,9 @@
6
6
  * (`runTentacle`) to execute a validated `TaskGraph`:
7
7
  * - repeatedly forms a parallel-safe "wave" of ready nodes (bounded by
8
8
  * ZELARI_KRAKEN_MAX_PARALLEL) and runs it concurrently;
9
+ * - hands each node the conclusions of its completed dependencies
10
+ * (`buildUpstreamContext`), so a dep edge carries information and not just
11
+ * ordering;
9
12
  * - retries a failed node up to `node.maxRetries`, then spawns a `fix`
10
13
  * node (bounded by a global fix budget) that inherits the failed
11
14
  * node's deps/scope/acceptance; a node that still fails is left
@@ -31,7 +34,7 @@
31
34
  *
32
35
  * @since v0.10.x — Kraken graph engine (F3)
33
36
  */
34
- import { getReadyNodes, isSettled, isConverged, failedNodeIds, countByStatus, selectParallelWave, } from '@zelari/core';
37
+ import { getReadyNodes, isSettled, isConverged, failedNodeIds, countByStatus, canRunParallel, parseVerifyVerdict, } from '@zelari/core';
35
38
  import { runTentacle, } from './tentacle.js';
36
39
  import { mergeKrakenWorktree, } from '../tools/krakenWorktree.js';
37
40
  import { appendKrakenRadio } from '../tools/krakenRadio.js';
@@ -39,6 +42,7 @@ import { runBacktest } from '../workspace/worldModel.js';
39
42
  import { startKrakenGraphLive, updateKrakenGraphLive, endKrakenGraphLive, } from './graphStatus.js';
40
43
  import { existsSync } from 'node:fs';
41
44
  import path from 'node:path';
45
+ import { saveGraphSnapshot, toGraphSnapshot } from './graphMemory.js';
42
46
  /** Default cap on concurrently-running tentacles across the whole graph. */
43
47
  export const DEFAULT_MAX_PARALLEL = 12;
44
48
  /** Default number of `fix` nodes the executor may spawn across one graph run. */
@@ -50,12 +54,58 @@ export function resolveMaxParallel(env = process.env) {
50
54
  const n = Number.parseInt(raw, 10);
51
55
  return Number.isFinite(n) && n > 0 ? n : DEFAULT_MAX_PARALLEL;
52
56
  }
53
- export function resolveFixBudget(env = process.env) {
57
+ /**
58
+ * How many `fix` nodes this run may spawn.
59
+ *
60
+ * A flat budget of 3 is a graph-size-blind number: on a 20-node graph it is
61
+ * spent on the first three failures and every later failure goes terminal,
62
+ * cascade-skipping its dependents — the larger the graph, the less repair it
63
+ * gets, which is backwards. Scale with the node count, keeping 3 as the floor
64
+ * for small graphs. An explicit env value still wins outright.
65
+ */
66
+ export function resolveFixBudget(env = process.env, nodeCount = 0) {
54
67
  const raw = env.ZELARI_KRAKEN_FIX_BUDGET;
68
+ if (raw === undefined || raw === '') {
69
+ return Math.max(DEFAULT_FIX_BUDGET, Math.ceil(nodeCount / 2));
70
+ }
71
+ const n = Number.parseInt(raw, 10);
72
+ return Number.isFinite(n) && n >= 0
73
+ ? n
74
+ : Math.max(DEFAULT_FIX_BUDGET, Math.ceil(nodeCount / 2));
75
+ }
76
+ /**
77
+ * Default rework rounds per writer when its `verify` returns FAIL.
78
+ *
79
+ * One. A rework round is a full re-run of a writer plus a fresh verification,
80
+ * so the cost is roughly a doubling of that branch; and the second opinion of
81
+ * a model that just judged its own sibling's work has sharply diminishing
82
+ * value. Raise it deliberately (and with a wall-clock budget set) rather than
83
+ * by default.
84
+ */
85
+ export const DEFAULT_MAX_REVIEW_ROUNDS = 1;
86
+ export function resolveMaxReviewRounds(env = process.env) {
87
+ const raw = env.ZELARI_KRAKEN_MAX_REVIEW_ROUNDS;
55
88
  if (raw === undefined || raw === '')
56
- return DEFAULT_FIX_BUDGET;
89
+ return DEFAULT_MAX_REVIEW_ROUNDS;
57
90
  const n = Number.parseInt(raw, 10);
58
- return Number.isFinite(n) && n >= 0 ? n : DEFAULT_FIX_BUDGET;
91
+ return Number.isFinite(n) && n >= 0 ? n : DEFAULT_MAX_REVIEW_ROUNDS;
92
+ }
93
+ /**
94
+ * Wall-clock bound on the WHOLE graph run (ms); 0 (the default) disables it.
95
+ *
96
+ * Per-node timeouts bound each tentacle but say nothing about the total: a
97
+ * wide graph, plus retries, plus fix nodes, plus rework rounds, can run far
98
+ * longer than any single node's budget without anything noticing. On expiry
99
+ * the run takes the ordinary cancellation path, so it still settles and still
100
+ * prints its digest.
101
+ */
102
+ export const DEFAULT_GRAPH_TIMEOUT_MS = 0;
103
+ export function resolveGraphTimeoutMs(env = process.env) {
104
+ const raw = env.ZELARI_KRAKEN_GRAPH_TIMEOUT_MS;
105
+ if (raw === undefined || raw === '')
106
+ return DEFAULT_GRAPH_TIMEOUT_MS;
107
+ const n = Number.parseInt(raw, 10);
108
+ return Number.isFinite(n) && n >= 0 ? n : DEFAULT_GRAPH_TIMEOUT_MS;
59
109
  }
60
110
  /**
61
111
  * Default wall-clock bound per tentacle run (ms). The graph executor calls
@@ -82,6 +132,20 @@ export const DEFAULT_NODE_TIMEOUT_MS = 300_000;
82
132
  * cascade-skipped dependents with them.
83
133
  */
84
134
  export const DEFAULT_WRITER_NODE_TIMEOUT_MS = 900_000;
135
+ /**
136
+ * How long to wait for a cancelled tentacle to actually unwind before
137
+ * declaring it unstoppable. Cancellation lands at the sub-agent's next event
138
+ * boundary, so this only needs to cover one in-flight provider chunk or tool
139
+ * call, not a whole turn.
140
+ */
141
+ export const DEFAULT_CANCEL_GRACE_MS = 30_000;
142
+ export function resolveCancelGraceMs(env = process.env) {
143
+ const raw = env.ZELARI_KRAKEN_CANCEL_GRACE_MS;
144
+ if (raw === undefined || raw === '')
145
+ return DEFAULT_CANCEL_GRACE_MS;
146
+ const n = Number.parseInt(raw, 10);
147
+ return Number.isFinite(n) && n >= 0 ? n : DEFAULT_CANCEL_GRACE_MS;
148
+ }
85
149
  /**
86
150
  * Wall-clock bound for one tentacle. `ZELARI_KRAKEN_NODE_TIMEOUT_MS` overrides
87
151
  * every kind (single knob, unchanged semantics: `0` disables);
@@ -128,39 +192,204 @@ function defaultChecksExists(cwd) {
128
192
  return false;
129
193
  }
130
194
  }
195
+ /**
196
+ * Per-dependency cap on injected upstream text. Mirrors `MAX_PRIOR_CHARS` in
197
+ * the council path (`packages/core/src/agents/councilApi.ts`), for the same
198
+ * reason: one verbose tentacle must not be able to saturate every downstream
199
+ * one.
200
+ */
201
+ export const MAX_UPSTREAM_CHARS_PER_DEP = 2800;
202
+ /** Total cap across all deps, so a wide fan-in cannot saturate a sub-agent. */
203
+ export const MAX_UPSTREAM_CHARS_TOTAL = 8000;
204
+ /**
205
+ * Render the conclusions of a node's already-completed dependencies as a
206
+ * prompt section for the sub-agent about to run.
207
+ *
208
+ * Without this the dependency edges were pure ordering constraints: a tentacle
209
+ * received only the prompt the PLANNER wrote before anything had executed, so
210
+ * an `explore` node's findings were computed, stored on the node, and then
211
+ * thrown away — the `general` nodes it "fed" started from zero and re-derived
212
+ * the same context (or guessed). The auto-injected `verify` node was worse
213
+ * still: it knew only the label of the work it was supposed to check.
214
+ *
215
+ * Direct dependencies only, deliberately — the transitive closure of a wide
216
+ * DAG would blow the sub-agent's context, and each hop's own conclusion is
217
+ * expected to carry forward what mattered.
218
+ */
219
+ export function buildUpstreamContext(graph, node) {
220
+ const parts = [];
221
+ const omitted = [];
222
+ let budget = MAX_UPSTREAM_CHARS_TOTAL;
223
+ for (const depId of node.deps) {
224
+ const dep = graph.nodes.get(depId);
225
+ if (!dep || dep.status !== 'done')
226
+ continue;
227
+ const raw = (dep.result ?? '').trim();
228
+ if (!raw)
229
+ continue;
230
+ const cap = Math.min(MAX_UPSTREAM_CHARS_PER_DEP, budget);
231
+ if (cap <= 0) {
232
+ omitted.push(dep.label);
233
+ continue;
234
+ }
235
+ const body = raw.length > cap
236
+ ? `${raw.slice(0, cap)}\n… [truncated ${raw.length}→${cap} chars]`
237
+ : raw;
238
+ budget -= Math.min(raw.length, cap);
239
+ const scope = dep.scope && dep.scope.length > 0 ? `, scope: ${dep.scope.join(', ')}` : '';
240
+ parts.push(`### ${dep.label} (${dep.kind}${scope})\n${body}`);
241
+ }
242
+ if (parts.length === 0)
243
+ return '';
244
+ const lines = [
245
+ '',
246
+ '## Context from completed upstream tasks',
247
+ 'Results reported by the tasks this one depends on. Treat them as ' +
248
+ 'hypotheses — prefer the actual files on disk where they conflict.',
249
+ '',
250
+ ...parts,
251
+ ];
252
+ if (omitted.length > 0) {
253
+ lines.push('', `(omitted for context budget: ${omitted.join(', ')})`);
254
+ }
255
+ return lines.join('\n');
256
+ }
257
+ /**
258
+ * Tool budget per node kind. Writers do real, multi-file work and were being
259
+ * run at the same `medium` budget as a read-only lookup; readers stay tight
260
+ * because a wide budget just invites them to dump the repository.
261
+ */
262
+ export function thoroughnessForKind(kind) {
263
+ return kind === 'general' || kind === 'fix' ? 'deep' : 'medium';
264
+ }
265
+ /** First non-empty line of a block of text, capped — for one-line markers. */
266
+ function firstLine(text, maxChars = 160) {
267
+ const line = text.trim().split('\n').find((l) => l.trim() !== '')?.trim() ?? '';
268
+ return line.length > maxChars ? `${line.slice(0, maxChars)}…` : line;
269
+ }
270
+ /** The tentacle kind a node runs as (`fix`/`merge` are driven as `general`). */
271
+ function agentForNode(node) {
272
+ return node.kind === 'explore' || node.kind === 'verify' ? node.kind : 'general';
273
+ }
131
274
  export class KrakenGraphExecutor {
132
275
  deps;
133
276
  parentCwd;
134
277
  sessionId;
278
+ goal;
135
279
  maxParallel;
136
280
  /** Explicit all-kinds override; when undefined the budget is per-kind. */
137
281
  nodeTimeoutMs;
282
+ cancelGraceMs;
283
+ /** Explicit override; when undefined the budget scales with the graph size. */
284
+ fixBudgetOption;
138
285
  fixBudgetRemaining;
286
+ maxReviewRounds;
287
+ graphTimeoutMs;
139
288
  worldModelGateOverride;
140
289
  runTentacleFn;
141
290
  mergeFn;
142
291
  backtestFn;
143
292
  nodeRunState = new Map();
293
+ /** fix node id → id of the failed node it was spawned to repair. */
294
+ repairs = new Map();
295
+ /**
296
+ * rework node id → id of the writer whose work it is redoing. A rework is a
297
+ * `fix` node, but unlike a repair it must NOT create a worktree of its own:
298
+ * it edits the writer's existing one (see {@link spawnReworkPair}).
299
+ */
300
+ reworks = new Map();
301
+ /** lineage root writer id → rework rounds already spent on that lineage. */
302
+ reviewRounds = new Map();
303
+ /**
304
+ * rework node id → the ORIGINAL writer its lineage started from.
305
+ *
306
+ * The budget has to be per lineage, not per node: a rework is itself a
307
+ * writer, so counting rounds against the node being reworked reset the
308
+ * counter every round and the graph chained rework → verify → rework
309
+ * forever, terminating only on the scheduler's iteration cap.
310
+ */
311
+ reviewLineage = new Map();
312
+ /** Verify verdicts left unresolved when the run ends. */
313
+ unresolved = [];
314
+ /** Live cancellation handles for the tentacles currently running. */
315
+ nodeControllers = new Map();
316
+ /** Wall-clock duration of each node's last run, by node id. */
317
+ durationsMs = new Map();
318
+ signal;
319
+ /** Set once the run has been cancelled: stops admission, retries and fixes. */
320
+ aborted = false;
144
321
  fixCounter = 0;
145
322
  constructor(opts) {
146
323
  this.deps = opts.taskToolDeps;
147
324
  this.parentCwd = opts.parentCwd;
148
325
  this.sessionId = opts.sessionId;
326
+ this.goal = opts.goal;
149
327
  this.maxParallel = opts.maxParallel ?? resolveMaxParallel();
150
328
  this.nodeTimeoutMs = opts.nodeTimeoutMs;
329
+ this.cancelGraceMs = opts.cancelGraceMs;
330
+ this.fixBudgetOption = opts.fixBudget;
331
+ // Provisional: `execute()` re-resolves it once the graph size is known.
151
332
  this.fixBudgetRemaining = opts.fixBudget ?? resolveFixBudget();
333
+ this.maxReviewRounds = opts.maxReviewRounds ?? resolveMaxReviewRounds();
334
+ this.graphTimeoutMs = opts.graphTimeoutMs ?? resolveGraphTimeoutMs();
152
335
  this.worldModelGateOverride = opts.worldModelGate;
336
+ this.signal = opts.signal;
153
337
  this.runTentacleFn = opts.runTentacleFn ?? runTentacle;
154
338
  this.mergeFn = opts.mergeFn ?? mergeKrakenWorktree;
155
339
  this.backtestFn = opts.backtestFn ?? runBacktest;
156
340
  }
157
341
  /** Execute the graph in place (mutates node statuses) until it settles. */
158
342
  async execute(graph) {
343
+ // Now that the graph is known, size the repair budget to it (unless the
344
+ // caller pinned one). Done here rather than in the constructor because the
345
+ // node count is the whole input to the decision.
346
+ if (this.fixBudgetOption === undefined) {
347
+ this.fixBudgetRemaining = resolveFixBudget(process.env, graph.nodes.size);
348
+ }
349
+ // Cancel eagerly rather than at the next loop turn: without a listener the
350
+ // scheduler would sit in `Promise.race` until some node finished on its
351
+ // own, which for a stuck writer is the whole point of cancelling.
352
+ const onAbort = () => this.cancelRun();
353
+ if (this.signal) {
354
+ if (this.signal.aborted)
355
+ this.aborted = true;
356
+ else
357
+ this.signal.addEventListener('abort', onAbort, { once: true });
358
+ }
359
+ // Whole-run wall-clock bound. Reuses the cancellation path rather than
360
+ // inventing a second way to stop, so the run still settles and still
361
+ // reports.
362
+ let graphTimer;
363
+ if (this.graphTimeoutMs > 0) {
364
+ graphTimer = setTimeout(() => {
365
+ this.radio('graph_failed', {
366
+ description: 'graph executor',
367
+ detail: `graph exceeded its ${this.graphTimeoutMs}ms wall-clock budget — cancelling`,
368
+ ok: false,
369
+ });
370
+ this.cancelRun();
371
+ }, this.graphTimeoutMs);
372
+ // Do not hold the process open just to fire a cancellation.
373
+ graphTimer.unref?.();
374
+ }
375
+ try {
376
+ return await this.schedule(graph);
377
+ }
378
+ finally {
379
+ if (graphTimer)
380
+ clearTimeout(graphTimer);
381
+ this.signal?.removeEventListener('abort', onAbort);
382
+ }
383
+ }
384
+ /** The scheduling loop proper. See {@link execute} for the cancellation wrapper. */
385
+ async schedule(graph) {
159
386
  // Anti-explosion / self-inflicted-hang guard: cap total loop iterations
160
387
  // at a generous multiple of the node count instead of trusting the
161
388
  // graph to always shrink monotonically (fix-node spawns grow it).
162
389
  const maxIterations = Math.max(64, graph.nodes.size * 8);
163
390
  let iterations = 0;
391
+ /** Node id → its running tentacle. One entry per in-flight node. */
392
+ const inFlight = new Map();
164
393
  startKrakenGraphLive(graph);
165
394
  while (!isSettled(graph)) {
166
395
  iterations += 1;
@@ -172,30 +401,66 @@ export class KrakenGraphExecutor {
172
401
  });
173
402
  break;
174
403
  }
175
- const ready = getReadyNodes(graph);
176
- if (ready.length === 0) {
177
- // Nothing is ready but the graph hasn't settled: some pending nodes
178
- // are permanently blocked by a failed/skipped dependency. Cascade
179
- // skip them so the loop can terminate.
180
- const skippedAny = this.skipBlockedNodes(graph);
181
- if (!skippedAny) {
182
- // Nothing ready and nothing to skip should not happen for a
183
- // validated DAG, but bail out rather than spin forever.
404
+ if (this.aborted && inFlight.size === 0)
405
+ break;
406
+ // Admit everything that can start alongside what is already running,
407
+ // up to the concurrency cap. A cancelled run admits nothing.
408
+ const admitted = this.aborted ? [] : this.admit(graph, inFlight);
409
+ if (admitted.length > 0) {
410
+ // Mark and publish the whole admission BEFORE starting any of it: an
411
+ // async function body runs synchronously up to its first await, so
412
+ // starting a tentacle first would let it observe stale counts. (The
413
+ // StatusBar's "n↑" never appeared at all while this was sampled only
414
+ // after the work had already settled.)
415
+ for (const node of admitted)
416
+ node.status = 'running';
417
+ updateKrakenGraphLive(graph);
418
+ for (const node of admitted)
419
+ inFlight.set(node.id, this.runNodeSafely(node, graph));
420
+ }
421
+ if (inFlight.size === 0) {
422
+ // Nothing running and nothing admissible: the remaining pending nodes
423
+ // are permanently blocked by a failed/skipped dependency (or by the
424
+ // run being cancelled). Cascade skip them so the loop can terminate.
425
+ if (!this.skipBlockedNodes(graph)) {
426
+ // Nothing ready, nothing running, nothing to skip — should not
427
+ // happen for a validated DAG, but bail out rather than spin forever.
184
428
  break;
185
429
  }
186
430
  continue;
187
431
  }
188
- const wave = selectParallelWave(ready).slice(0, this.maxParallel);
189
- for (const node of wave)
190
- node.status = 'running';
191
- const results = await Promise.all(wave.map((node) => this.runNode(node, graph)));
192
- for (let i = 0; i < wave.length; i++) {
193
- this.applyResult(graph, wave[i], results[i]);
194
- }
432
+ // Settle ONE node at a time. Waiting for a whole wave meant a 15-minute
433
+ // writer held back every explore that became ready a second later, and
434
+ // left the concurrency budget idle for the duration.
435
+ const { id, res } = await Promise.race(inFlight.values());
436
+ inFlight.delete(id);
437
+ const settledNode = graph.nodes.get(id);
438
+ if (settledNode)
439
+ this.applyResult(graph, settledNode, res);
195
440
  updateKrakenGraphLive(graph);
196
441
  }
442
+ // The loop can break with work still in flight (iteration cap, or a
443
+ // cancellation whose tentacles have not unwound yet). Let those settle
444
+ // rather than returning a summary while they are still writing, and
445
+ // record what they produced.
446
+ if (inFlight.size > 0) {
447
+ for (const { id, res } of await Promise.all(inFlight.values())) {
448
+ const node = graph.nodes.get(id);
449
+ if (node)
450
+ this.applyResult(graph, node, res);
451
+ }
452
+ inFlight.clear();
453
+ }
454
+ // A cancelled run leaves nodes that never started: `skipped` says exactly
455
+ // that, and lets the graph settle so a summary can be returned.
456
+ if (this.aborted) {
457
+ for (const n of graph.nodes.values()) {
458
+ if (n.status === 'pending')
459
+ n.status = 'skipped';
460
+ }
461
+ }
197
462
  let backtest;
198
- const converged = isConverged(graph);
463
+ const converged = !this.aborted && isConverged(graph);
199
464
  if (converged) {
200
465
  const gateOn = this.worldModelGateOverride ?? isWorldModelGateEnabled(this.parentCwd);
201
466
  if (gateOn) {
@@ -210,45 +475,153 @@ export class KrakenGraphExecutor {
210
475
  else {
211
476
  this.radio('graph_failed', {
212
477
  description: 'graph executor',
213
- detail: `failed nodes: ${failedNodeIds(graph).join(', ') || 'none'}`,
478
+ detail: this.aborted
479
+ ? 'cancelled by caller'
480
+ : `failed nodes: ${failedNodeIds(graph).join(', ') || 'none'}`,
214
481
  ok: false,
215
482
  });
216
483
  }
217
484
  endKrakenGraphLive(graph, converged);
485
+ // Cross-run memory: let the next planning pass see where this one stopped
486
+ // instead of replanning the whole goal blind.
487
+ await saveGraphSnapshot(this.parentCwd, toGraphSnapshot(graph, {
488
+ goal: this.goal ?? graph.id,
489
+ converged,
490
+ unresolvedFindings: this.unresolved,
491
+ }));
218
492
  return {
219
493
  graph,
220
494
  converged,
221
495
  failedNodeIds: failedNodeIds(graph),
222
496
  counts: countByStatus(graph),
497
+ durationsMs: Object.fromEntries(this.durationsMs),
498
+ cancelled: this.aborted,
499
+ unresolvedFindings: [...this.unresolved],
223
500
  ...(backtest ? { backtest } : {}),
224
501
  };
225
502
  }
503
+ /**
504
+ * Stop the run: no further admissions, and every tentacle currently running
505
+ * is told to unwind. Each node's own timeout/grace machinery then resolves
506
+ * it, so `execute()` settles instead of leaving orphans behind.
507
+ */
508
+ cancelRun() {
509
+ if (this.aborted)
510
+ return;
511
+ this.aborted = true;
512
+ this.radio('graph_failed', {
513
+ description: 'graph executor',
514
+ detail: `cancelling ${this.nodeControllers.size} running tentacle(s)`,
515
+ ok: false,
516
+ });
517
+ for (const controller of this.nodeControllers.values())
518
+ controller.abort();
519
+ }
520
+ /**
521
+ * Pick the ready nodes that may start right now: parallel-safe against every
522
+ * node already running AND against each other, within the concurrency cap.
523
+ *
524
+ * Unlike a wave-at-a-time scheduler this is called on every completion, so a
525
+ * node becomes eligible the moment its blocker settles instead of waiting
526
+ * for the slowest member of some earlier batch.
527
+ */
528
+ admit(graph, inFlight) {
529
+ const capacity = this.maxParallel - inFlight.size;
530
+ if (capacity <= 0)
531
+ return [];
532
+ const running = [];
533
+ for (const id of inFlight.keys()) {
534
+ const n = graph.nodes.get(id);
535
+ if (n)
536
+ running.push(n);
537
+ }
538
+ const admitted = [];
539
+ for (const node of getReadyNodes(graph)) {
540
+ if (admitted.length >= capacity)
541
+ break;
542
+ const safe = running.every((r) => canRunParallel(r, node)) &&
543
+ admitted.every((a) => canRunParallel(a, node));
544
+ if (safe)
545
+ admitted.push(node);
546
+ }
547
+ return admitted;
548
+ }
549
+ /**
550
+ * Run one node, tagging the result with its id and converting an unexpected
551
+ * throw into a node failure. The scheduler races these promises, so a
552
+ * rejection would abandon every other in-flight tentacle mid-write; one
553
+ * failed node that the retry/fix machinery can reason about is strictly
554
+ * better than an aborted graph.
555
+ */
556
+ runNodeSafely(node, graph) {
557
+ return this.runNode(node, graph).then((res) => ({ id: node.id, res }), (err) => ({
558
+ id: node.id,
559
+ res: {
560
+ ok: false,
561
+ agent: agentForNode(node),
562
+ error: `tentacle threw: ${err instanceof Error ? err.message : String(err)}`,
563
+ cancelled: true,
564
+ },
565
+ }));
566
+ }
226
567
  /** Run one node: dispatch to the merge handler for `merge` nodes, else a tentacle. */
227
568
  async runNode(node, graph) {
569
+ const startedAt = Date.now();
570
+ try {
571
+ return await this.runNodeInner(node, graph);
572
+ }
573
+ finally {
574
+ this.durationsMs.set(node.id, Date.now() - startedAt);
575
+ this.nodeControllers.delete(node.id);
576
+ }
577
+ }
578
+ async runNodeInner(node, graph) {
228
579
  this.radio('node_start', { description: node.label, agent: node.kind });
229
580
  if (node.kind === 'merge') {
230
581
  return this.runMergeNode(node, graph);
231
582
  }
232
- const usesWorktree = node.kind === 'general' || node.kind === 'fix';
583
+ // A rework edits the worktree its writer already produced (see
584
+ // spawnReworkPair) — it must not open a second one on the same scope.
585
+ const isRework = this.reworks.has(node.id);
586
+ const usesWorktree = (node.kind === 'general' || node.kind === 'fix') && !isRework;
233
587
  const agent = node.kind === 'fix' ? 'general' : node.kind;
588
+ const controller = new AbortController();
589
+ // Registered so a cancelled run can reach in and stop this tentacle
590
+ // instead of waiting for it to finish on its own.
591
+ this.nodeControllers.set(node.id, controller);
592
+ if (this.aborted)
593
+ controller.abort();
594
+ // Hand the sub-agent what its dependencies actually concluded — the whole
595
+ // point of the dep edge (see buildUpstreamContext).
596
+ const upstream = buildUpstreamContext(graph, node);
597
+ // A verify inspects the tree its writer wrote in; a rework edits that same
598
+ // tree. Both resolve to the worktree recorded behind their deps.
599
+ const inheritedCwd = node.kind === 'verify' || isRework
600
+ ? this.inheritedWorktreeCwdFor(node, graph)
601
+ : undefined;
234
602
  const res = await this.withNodeTimeout(this.runTentacleFn({
235
- deps: this.deps,
603
+ // `allowWorktree: false` is what actually stops a rework from opening
604
+ // its own worktree: creation is driven by the agent kind ('general')
605
+ // inside runTentacle, not by anything the executor passes per-call.
606
+ deps: isRework ? { ...this.deps, allowWorktree: false } : this.deps,
236
607
  args: {
237
608
  description: node.label,
238
- prompt: node.prompt,
609
+ prompt: upstream ? `${node.prompt}\n${upstream}` : node.prompt,
239
610
  scope: node.scope,
240
611
  acceptance: node.acceptance,
241
612
  },
242
613
  agent,
243
- thoroughness: 'medium',
614
+ thoroughness: thoroughnessForKind(node.kind),
244
615
  parentCwd: this.parentCwd,
616
+ ...(inheritedCwd ? { cwdOverride: inheritedCwd } : {}),
245
617
  sessionId: this.sessionId,
246
618
  // Defer merge for writers so the executor controls merge ordering
247
- // (Correction 4); explore/verify never create a worktree.
619
+ // (Correction 4); explore/verify/rework never create a worktree.
248
620
  deferMerge: usesWorktree,
249
621
  graphId: graph.id,
250
622
  nodeId: node.id,
251
- }), agent);
623
+ signal: controller.signal,
624
+ }), agent, controller);
252
625
  if (res.ok && usesWorktree) {
253
626
  this.nodeRunState.set(node.id, { worktreeHandle: res.worktreeHandle });
254
627
  }
@@ -256,44 +629,136 @@ export class KrakenGraphExecutor {
256
629
  }
257
630
  /**
258
631
  * Bound a tentacle run to its wall-clock budget (explicit `nodeTimeoutMs`
259
- * option, else per-kind — writers get more than readers). On timeout, resolves
260
- * to a synthetic `TentacleFailure` so the normal retry/fix/cascade-skip
261
- * path handles it this only bounds how long the EXECUTOR waits, it does
262
- * not (cannot) forcibly cancel the underlying sub-agent run; the point is
263
- * to guarantee `execute()` itself always settles so the caller's process
264
- * can exit instead of hanging forever on one stuck node.
632
+ * option, else per-kind — writers get more than readers). On timeout the
633
+ * run is CANCELLED via its AbortSignal, then given `cancelGraceMs` to
634
+ * unwind. Cancellation lands at the sub-agent's next event boundary (an
635
+ * async generator only observes `.return()` when it next yields), so a run
636
+ * blocked on a slow provider call can outlive the grace period — that case
637
+ * resolves with `cancelled: false` and `applyResult` refuses to re-spawn
638
+ * the node, since a tentacle that may still be writing must not be joined
639
+ * by a second one on the same scope.
640
+ *
641
+ * Either way `execute()` always settles, so the caller's process can exit
642
+ * instead of hanging forever on one stuck node.
265
643
  */
266
- withNodeTimeout(promise, agent) {
644
+ async withNodeTimeout(promise, agent, controller) {
267
645
  const ms = this.nodeTimeoutMs ?? resolveNodeTimeoutMs(process.env, agent);
268
646
  if (ms <= 0)
269
647
  return promise;
270
648
  let timer;
649
+ const TIMED_OUT = Symbol('timeout');
271
650
  const timeout = new Promise((resolve) => {
272
- timer = setTimeout(() => {
273
- resolve({ ok: false, agent, error: `tentacle timed out after ${ms}ms` });
274
- }, ms);
651
+ timer = setTimeout(() => resolve(TIMED_OUT), ms);
275
652
  });
276
- return Promise.race([promise, timeout]).finally(() => {
653
+ const raced = await Promise.race([promise, timeout]).finally(() => {
277
654
  if (timer)
278
655
  clearTimeout(timer);
279
656
  });
657
+ if (raced !== TIMED_OUT)
658
+ return raced;
659
+ // Budget blown. Tell the tentacle to stop, then give it a bounded grace
660
+ // period to actually unwind — a run that has already been told to stop is
661
+ // safe to re-run, one that is still going is not (two agents writing the
662
+ // same scope is exactly the corruption this guards against).
663
+ controller?.abort();
664
+ const graceMs = this.cancelGraceMs ?? resolveCancelGraceMs();
665
+ let graceTimer;
666
+ const grace = new Promise((resolve) => {
667
+ graceTimer = setTimeout(() => resolve(TIMED_OUT), graceMs);
668
+ });
669
+ const settled = await Promise.race([promise, grace]).finally(() => {
670
+ if (graceTimer)
671
+ clearTimeout(graceTimer);
672
+ });
673
+ if (settled !== TIMED_OUT) {
674
+ // The run is over, so the scope is free and a retry/fix would be safe.
675
+ // If it actually SUCCEEDED — finishing in the window between the
676
+ // deadline and the abort landing — keep that result rather than
677
+ // discarding completed (and already written) work over a few ms.
678
+ if (settled.ok)
679
+ return settled;
680
+ return { ...settled, cancelled: true };
681
+ }
682
+ // Did not stop in time. Report it as uncancelled so `applyResult` refuses
683
+ // to re-spawn this node — better one failed node than two concurrent
684
+ // writers on one directory.
685
+ return {
686
+ ok: false,
687
+ agent,
688
+ error: `tentacle timed out after ${ms}ms and did not stop within ${graceMs}ms of being cancelled; ` +
689
+ `not retrying to avoid two tentacles writing the same scope`,
690
+ cancelled: false,
691
+ };
280
692
  }
281
693
  /**
282
- * Sequentially merge every dep node's deferred worktree (in dep order) into
283
- * parent HEAD. Deps without a recorded worktree handle (worktree isolation
284
- * disabled, or a read-only node) are a no-op. On conflict the branch is
285
- * kept and the conflict is surfaced in the merge node's error remaining
286
- * deps still attempt to merge (independent branches shouldn't be blocked
287
- * by one conflict).
694
+ * The worktree a node should run in, inherited from the writer behind it.
695
+ *
696
+ * For a `verify`: verification happens BEFORE the merge node, so when its
697
+ * writer worked in an isolated worktree the changes are not in the parent
698
+ * tree yet a verify tentacle pointed at `parentCwd` was inspecting a tree
699
+ * that provably did not contain the work it was asked to check, and reported
700
+ * it missing.
701
+ *
702
+ * For a rework: the same tree, for the stronger reason that writing anywhere
703
+ * else would strand the round on a second branch.
704
+ *
705
+ * Returns undefined when there is no single tree: no worktrees (isolation
706
+ * disabled — the writers edited the parent tree directly), or several
707
+ * distinct ones, in which case no single cwd is correct and the parent tree
708
+ * is the honest default.
709
+ */
710
+ inheritedWorktreeCwdFor(node, graph) {
711
+ const paths = new Set(this.collectWorktreeSources(node, graph).map((s) => s.handle.path));
712
+ return paths.size === 1 ? [...paths][0] : undefined;
713
+ }
714
+ /**
715
+ * Resolve the deferred worktrees produced behind a node's dependencies, in
716
+ * ancestors-first order. Used to decide what a `merge` node must merge, and
717
+ * which tree a `verify` node should actually inspect.
718
+ *
719
+ * A merge node's direct deps are NOT the writers: `buildGraphFromPlan`
720
+ * injects a `verify` node after every `general` node and points the merge at
721
+ * those verifies, while worktree handles are recorded against the writer
722
+ * node ids. Looking only at direct deps therefore found nothing to merge and
723
+ * silently reported success while every tentacle's work stayed stranded on
724
+ * its branch. Walk up through non-writer deps until the writers are found.
725
+ *
726
+ * Post-order so a writer that depends on another writer merges after it (the
727
+ * later branch was cut from a HEAD that already contained the earlier work).
728
+ * `merge` nodes terminate the walk: another merge already owns its subtree.
729
+ */
730
+ collectWorktreeSources(node, graph) {
731
+ const out = [];
732
+ const seen = new Set();
733
+ const visit = (id) => {
734
+ if (seen.has(id))
735
+ return;
736
+ seen.add(id);
737
+ const n = graph.nodes.get(id);
738
+ if (!n || n.kind === 'merge')
739
+ return;
740
+ for (const dep of n.deps)
741
+ visit(dep);
742
+ const handle = this.nodeRunState.get(id)?.worktreeHandle;
743
+ if (handle)
744
+ out.push({ id, handle });
745
+ };
746
+ for (const depId of node.deps)
747
+ visit(depId);
748
+ return out;
749
+ }
750
+ /**
751
+ * Sequentially merge every deferred worktree this node covers (in
752
+ * ancestors-first order) into parent HEAD. Nodes without a recorded worktree
753
+ * handle (worktree isolation disabled, or a read-only node) are a no-op. On
754
+ * conflict the branch is kept and the conflict is surfaced in the merge
755
+ * node's error — remaining sources still attempt to merge (independent
756
+ * branches shouldn't be blocked by one conflict).
288
757
  */
289
758
  async runMergeNode(node, graph) {
290
759
  const conflicts = [];
291
760
  const merged = [];
292
- for (const depId of node.deps) {
293
- const state = this.nodeRunState.get(depId);
294
- const handle = state?.worktreeHandle;
295
- if (!handle)
296
- continue; // nothing to merge for this dep
761
+ for (const { id: depId, handle } of this.collectWorktreeSources(node, graph)) {
297
762
  let result;
298
763
  try {
299
764
  result = await this.mergeFn(handle, {
@@ -313,6 +778,10 @@ export class KrakenGraphExecutor {
313
778
  conflicts.push(`${depId}: ${result.message}`);
314
779
  }
315
780
  else {
781
+ // The branch is merged (and, unless KEEP is set, the worktree is gone).
782
+ // Drop the handle so a second merge node covering the same ancestor
783
+ // cannot try to merge a branch that no longer exists.
784
+ this.nodeRunState.set(depId, { worktreeHandle: null });
316
785
  merged.push(depId);
317
786
  }
318
787
  }
@@ -340,9 +809,40 @@ export class KrakenGraphExecutor {
340
809
  node.status = 'done';
341
810
  node.result = res.result;
342
811
  this.radio('node_end', { description: node.label, agent: node.kind, ok: true });
812
+ this.reconcileRepairedNode(graph, node);
813
+ // A verify that RAN successfully is not the same thing as work that
814
+ // PASSED. Read what it actually concluded.
815
+ if (node.kind === 'verify')
816
+ this.applyVerifyVerdict(graph, node);
343
817
  return;
344
818
  }
345
819
  node.error = res.error;
820
+ // A cancelled run must not retry or spawn repairs: the caller asked for
821
+ // the graph to stop, and re-spawning work is the opposite of that.
822
+ if (this.aborted) {
823
+ node.status = 'error';
824
+ this.radio('node_end', {
825
+ description: node.label,
826
+ agent: node.kind,
827
+ detail: res.error,
828
+ ok: false,
829
+ });
830
+ return;
831
+ }
832
+ // A run we could not confirm has stopped may still be writing this node's
833
+ // scope. Re-spawning would put two tentacles in the same directory — the
834
+ // failure mode that produced duplicate parallel implementations of the
835
+ // same modules. Fail terminally instead; dependents cascade-skip.
836
+ if (res.cancelled === false) {
837
+ node.status = 'error';
838
+ this.radio('node_end', {
839
+ description: node.label,
840
+ agent: node.kind,
841
+ detail: res.error,
842
+ ok: false,
843
+ });
844
+ return;
845
+ }
346
846
  if (node.retryCount < node.maxRetries) {
347
847
  node.retryCount += 1;
348
848
  node.status = 'pending';
@@ -378,6 +878,201 @@ export class KrakenGraphExecutor {
378
878
  ok: false,
379
879
  });
380
880
  }
881
+ /**
882
+ * A `fix` node just completed the work its failed predecessor could not.
883
+ * That unit of work IS done — but the predecessor was left terminally
884
+ * `error`, and since `isConverged` requires every node to be `done`/
885
+ * `skipped`, a fully repaired graph reported "did not converge" and listed
886
+ * the repaired node under `failedNodeIds`. The cross-run snapshot then told
887
+ * the next planner to redo work the fix had already completed.
888
+ *
889
+ * Marking it `done` has no scheduling effect (dependents were re-pointed at
890
+ * the fix when it was spawned) — it is purely how the run is reported. The
891
+ * original failure stays visible as the separate `fix: …` node and in the
892
+ * repaired node's result line.
893
+ */
894
+ reconcileRepairedNode(graph, fixNode) {
895
+ const failedId = this.repairs.get(fixNode.id);
896
+ if (!failedId)
897
+ return;
898
+ const failed = graph.nodes.get(failedId);
899
+ if (!failed || failed.status !== 'error')
900
+ return;
901
+ const original = failed.error ? ` (original failure: ${failed.error})` : '';
902
+ failed.status = 'done';
903
+ failed.result = `repaired by "${fixNode.label}"${original}${fixNode.result ? `: ${fixNode.result}` : ''}`;
904
+ // Clear the error: the snapshot lists a `done` node under "Already
905
+ // completed — do NOT redo this work", and a trailing error message there
906
+ // reads as if it still needs repair.
907
+ failed.error = undefined;
908
+ this.radio('node_end', {
909
+ description: failed.label,
910
+ agent: failed.kind,
911
+ detail: `repaired by ${fixNode.id}`,
912
+ ok: true,
913
+ });
914
+ }
915
+ /**
916
+ * Act on what a completed `verify` node concluded.
917
+ *
918
+ * The verify itself stays `done` either way — it did its job, and doing it
919
+ * well means being free to say "no". A FAIL instead sends the WRITER back
920
+ * through a bounded rework round.
921
+ *
922
+ * Without this the verdict text was never read: a verify that reported the
923
+ * work as wrong was recorded exactly like one that reported it correct, the
924
+ * graph converged over the defect, and the only iteration the engine could
925
+ * do was on execution failure. An `unknown` verdict (no parseable trailer)
926
+ * is deliberately non-blocking — a prompt drift must not be able to wedge
927
+ * every graph — but it is recorded, because a gate that has silently stopped
928
+ * working is worse than no gate.
929
+ */
930
+ applyVerifyVerdict(graph, verify) {
931
+ const { verdict, findings } = parseVerifyVerdict(verify.result);
932
+ if (verdict === 'pass')
933
+ return;
934
+ const writer = this.writerBehind(verify, graph);
935
+ if (!writer)
936
+ return; // nothing to send back to
937
+ if (verdict === 'unknown') {
938
+ this.unresolved.push({
939
+ nodeId: writer.id,
940
+ label: writer.label,
941
+ reason: 'unknown',
942
+ findings: findings || '(verify produced no parseable VERDICT line)',
943
+ });
944
+ return;
945
+ }
946
+ // A cancelled run must not spawn new work, and a lineage that has spent
947
+ // its rounds is done being reworked.
948
+ const root = this.reviewLineage.get(writer.id) ?? writer.id;
949
+ const spent = this.reviewRounds.get(root) ?? 0;
950
+ if (this.aborted || spent >= this.maxReviewRounds) {
951
+ this.unresolved.push({
952
+ nodeId: writer.id,
953
+ label: writer.label,
954
+ reason: 'fail',
955
+ findings,
956
+ });
957
+ // Keep the unresolved verdict attached to the node too, so the digest,
958
+ // the snapshot and any reader of the graph see it in the same place the
959
+ // repaired-node marker lives.
960
+ writer.result =
961
+ `${writer.result ?? ''}\n\n[accepted with unresolved verify findings from ` +
962
+ `"${verify.label}"]${findings ? `: ${firstLine(findings)}` : ''}`.trim();
963
+ this.radio('node_end', {
964
+ description: writer.label,
965
+ agent: writer.kind,
966
+ detail: this.aborted
967
+ ? 'verify FAIL left unresolved (run cancelled)'
968
+ : `verify FAIL left unresolved (rework budget ${this.maxReviewRounds} spent)`,
969
+ ok: false,
970
+ });
971
+ return;
972
+ }
973
+ this.reviewRounds.set(root, spent + 1);
974
+ this.spawnReworkPair(graph, writer, verify, findings, root, spent + 1);
975
+ }
976
+ /**
977
+ * The writer whose work a `verify` node judged.
978
+ *
979
+ * Walks up through non-writer deps, the same shape `collectWorktreeSources`
980
+ * relies on: a verify's dep is normally its writer directly, but after a
981
+ * rework round the chain is writer → verify → rework → verify, and the
982
+ * rework (a `fix` node) is itself the writer to send back.
983
+ */
984
+ writerBehind(verify, graph) {
985
+ const seen = new Set();
986
+ const visit = (id) => {
987
+ if (seen.has(id))
988
+ return undefined;
989
+ seen.add(id);
990
+ const n = graph.nodes.get(id);
991
+ if (!n || n.kind === 'merge')
992
+ return undefined;
993
+ if (n.kind === 'general' || n.kind === 'fix')
994
+ return n;
995
+ for (const dep of n.deps) {
996
+ const found = visit(dep);
997
+ if (found)
998
+ return found;
999
+ }
1000
+ return undefined;
1001
+ };
1002
+ for (const depId of verify.deps) {
1003
+ const found = visit(depId);
1004
+ if (found)
1005
+ return found;
1006
+ }
1007
+ return undefined;
1008
+ }
1009
+ /**
1010
+ * Send a writer's work back for one more round: a rework node carrying the
1011
+ * verify's findings, plus a fresh verify to judge the result.
1012
+ *
1013
+ * The rework runs INSIDE the writer's worktree instead of creating one of
1014
+ * its own. Two worktrees for one scope means two branches, and the merge
1015
+ * node walks up to the writer — so a rework on its own branch would be
1016
+ * merged never or twice, exactly the stranded-work failure the merge fix
1017
+ * addressed. `allowWorktree: false` on this node's deps suppresses creation,
1018
+ * and `cwdOverride` (resolved via {@link inheritedWorktreeCwdFor}) points it
1019
+ * at the existing tree; the handle stays registered against the writer.
1020
+ *
1021
+ * Acyclicity is preserved by construction: both new nodes point only at
1022
+ * nodes that already exist, and the rewiring moves an existing edge forward
1023
+ * along the chain rather than back into it.
1024
+ */
1025
+ spawnReworkPair(graph, writer, verify, findings, root, round) {
1026
+ // Named after the lineage root, so round 2 of g1 is `rework-g1-2` rather
1027
+ // than a nested `rework-rework-g1-1-1`.
1028
+ const reworkId = `rework-${root}-${round}`;
1029
+ const reworkNode = {
1030
+ id: reworkId,
1031
+ kind: 'fix',
1032
+ label: `rework: ${writer.label}`,
1033
+ prompt: `A reviewer inspected this work on disk and REJECTED it. Address every finding below, ` +
1034
+ `then leave the work in a state that satisfies the original task.\n\n` +
1035
+ `## Original task\n${writer.prompt}\n\n` +
1036
+ `## Reviewer findings (these are what must change)\n${findings || '(the reviewer reported FAIL without detail)'}`,
1037
+ ...(writer.scope ? { scope: writer.scope } : {}),
1038
+ ...(writer.acceptance ? { acceptance: writer.acceptance } : {}),
1039
+ // The verify is already `done`, so the rework is immediately ready.
1040
+ deps: [verify.id],
1041
+ status: 'pending',
1042
+ retryCount: 0,
1043
+ maxRetries: 0,
1044
+ };
1045
+ graph.nodes.set(reworkId, reworkNode);
1046
+ this.reworks.set(reworkId, writer.id);
1047
+ this.reviewLineage.set(reworkId, root);
1048
+ const reVerifyId = `verify-${reworkId}`;
1049
+ const reVerifyNode = {
1050
+ id: reVerifyId,
1051
+ kind: 'verify',
1052
+ label: `verify: ${writer.label} (rework ${round})`,
1053
+ prompt: verify.prompt,
1054
+ deps: [reworkId],
1055
+ status: 'pending',
1056
+ retryCount: 0,
1057
+ maxRetries: verify.maxRetries,
1058
+ };
1059
+ graph.nodes.set(reVerifyId, reVerifyNode);
1060
+ // Whatever waited on the old verify (typically the merge node) must now
1061
+ // wait on the new one, or it would merge the branch mid-rework.
1062
+ for (const other of graph.nodes.values()) {
1063
+ if (other.id === reworkId || other.id === reVerifyId)
1064
+ continue;
1065
+ if (other.deps.includes(verify.id)) {
1066
+ other.deps = other.deps.map((d) => (d === verify.id ? reVerifyId : d));
1067
+ }
1068
+ }
1069
+ this.radio('node_fix', {
1070
+ description: reworkNode.label,
1071
+ agent: 'fix',
1072
+ detail: `verify FAIL on "${writer.label}" — rework round ${round}/${this.maxReviewRounds}`,
1073
+ ok: false,
1074
+ });
1075
+ }
381
1076
  /**
382
1077
  * Create a `fix` node that attempts to redo the failed node's work, wired
383
1078
  * so downstream dependents of the failed node also wait on the fix.
@@ -400,9 +1095,12 @@ export class KrakenGraphExecutor {
400
1095
  // no further retries — one fix attempt per failed node in v1
401
1096
  };
402
1097
  graph.nodes.set(fixId, fixNode);
1098
+ this.repairs.set(fixId, failed.id);
403
1099
  // Downstream dependents wait on the fix attempt INSTEAD of the failed
404
- // node (not in addition to it) the failed node never becomes `done`,
405
- // so leaving it in `deps` would strand dependents forever.
1100
+ // node (not in addition to it): the failed node is terminally `error` and
1101
+ // will never run again, so leaving it in `deps` would strand them forever.
1102
+ // (If the fix succeeds, `reconcileRepairedNode` marks the failed node
1103
+ // `done` for reporting — after this rewiring, and with no effect on it.)
406
1104
  for (const other of graph.nodes.values()) {
407
1105
  if (other.id === fixId)
408
1106
  continue;