pi-agent-fleet 0.6.0 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-agent-fleet",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "DAG-of-agents fleets for pi — parallel workers with contracts, reviewer-gated iteration loops, and live progress",
   "type": "module",
   "scripts": {

package/src/controller.ts

@@ -70,7 +70,37 @@ export async function statusText(fleet: ActiveFleet): Promise<string> {
     : failed ? `next: fleet_relaunch ${failed[0]}`
     : state.status === "completed" ? `next: read report ${reportPath}`
     : `next: inspect ${join(fleet.fleetRoot, "state.json")}`;
-  return `${renderDag(fleet.spec, state)}\n\nreport: ${reportPath}\n${next}`;
+  let crashWarning = "";
+  if (state.status === "running") {
+    const heartbeat = state.heartbeat_at;
+    const stale = !heartbeat || (Date.now() - new Date(heartbeat).getTime() > 60000);
+    if (stale && typeof state.pid === "number") {
+      let dead = false;
+      try {
+        process.kill(state.pid, 0);
+      } catch (e) {
+        if ((e as NodeJS.ErrnoException).code === "ESRCH") dead = true;
+      }
+      if (dead) {
+        crashWarning = `\n\nwarning: fleet appears crashed (stale heartbeat/dead pid ${state.pid}); run fleet_continue to recover`;
+      }
+    }
+  }
+  return `${renderDag(fleet.spec, state)}\n\nreport: ${reportPath}\n${next}${crashWarning}`;
+}
+
+export function checkCostLimits(fleet: ActiveFleet, ctx: ExtensionContext): void {
+  const warn = fleet.spec.config.warn_cost_usd;
+  const cap = fleet.spec.config.max_cost_usd;
+  const cost = fleet.state.cost_usd_estimate;
+  if (warn && !fleet.costWarned && cost >= warn) {
+    fleet.costWarned = true;
+    if (ctx.hasUI) ctx.ui.notify(`fleet cost warning: $${cost.toFixed(4)} >= $${warn}`, "warning");
+  }
+  if (cap && cost >= cap && !fleet.killSwitch.killed) {
+    fleet.killSwitch.killed = true;
+    if (ctx.hasUI) ctx.ui.notify(`fleet cost cap reached: $${cost.toFixed(4)} >= $${cap.toFixed(4)}`, "error");
+  }
 }
 
 export async function dagPreview(spec: FleetSpec, state: FleetState | undefined, fleetRoot: string): Promise<string> {
@@ -180,17 +210,40 @@ export async function startLoop(fleet: ActiveFleet, ctx: ExtensionContext, resum
 
   fleet.running = true;
   fleet.costWarned = false;
-  fleet.state = { ...fleet.state, status: "running" };
+  fleet.state = { ...fleet.state, status: "running", pid: process.pid, heartbeat_at: new Date().toISOString() };
+  await writeState(fleet.fleetRoot, fleet.state);
   const stopSpinner = startSpinner(ctx, fleet);
   updateWidget(ctx, fleet);
 
-  const checkCostWarning = () => {
-    const warn = fleet.spec.config.warn_cost_usd;
-    if (!warn || fleet.costWarned) return;
-    const cost = fleet.state.cost_usd_estimate;
-    if (cost >= warn) {
-      fleet.costWarned = true;
-      if (ctx.hasUI) ctx.ui.notify(`fleet cost warning: $${cost.toFixed(4)} >= $${warn}`, "warning");
+  let lastHeartbeatWrite = Date.now();
+  const heartbeatInterval = setInterval(async () => {
+    const now = Date.now();
+    if (now - lastHeartbeatWrite < 5000) return;
+    fleet.state = { ...fleet.state, heartbeat_at: new Date().toISOString() };
+    lastHeartbeatWrite = now;
+    try {
+      await writeState(fleet.fleetRoot, fleet.state);
+    } catch {
+      // ignore heartbeat write failures
+    }
+  }, 5000);
+  if (typeof heartbeatInterval.unref === "function") heartbeatInterval.unref();
+
+  let cleanedUp = false;
+  let finalStateWritten = false;
+  const cleanup = () => {
+    if (cleanedUp) return;
+    cleanedUp = true;
+    stopSpinner();
+    clearInterval(heartbeatInterval);
+  };
+  const writeFinalState = async () => {
+    if (finalStateWritten) return;
+    finalStateWritten = true;
+    try {
+      await writeState(fleet.fleetRoot, fleet.state);
+    } catch {
+      // best-effort final persistence
     }
   };
 
@@ -229,6 +282,7 @@ export async function startLoop(fleet: ActiveFleet, ctx: ExtensionContext, resum
           repoCwd: worktreeCwd,
           sessionDir,
           thinkingLevel: effort,
+          extensionAllowlist: fleet.spec.config.worker_extensions,
           sessionFactory: resolvedModel ? sessionFactoryForModel(resolvedModel) : undefined,
           onSession: (s) => registerNodeSession(fleet, nodeId, s),
           onEvent: (e) => {
@@ -236,7 +290,7 @@ export async function startLoop(fleet: ActiveFleet, ctx: ExtensionContext, resum
             if (e.type === "tokens") fleet.state = patchNode(fleet.fleetRoot, fleet.state, nodeId, { tokens: e.tokens });
             if (e.type === "cost") {
               fleet.state = patchNode(fleet.fleetRoot, fleet.state, nodeId, { cost_usd_estimate: e.cost });
-              checkCostWarning();
+              checkCostLimits(fleet, ctx);
             }
             if (e.type === "error") fleet.state = patchNode(fleet.fleetRoot, fleet.state, nodeId, { status_note: e.message });
             updateWidget(ctx, fleet);
@@ -271,7 +325,7 @@ export async function startLoop(fleet: ActiveFleet, ctx: ExtensionContext, resum
         fleet.state = fleet.state.nodes[nodeId]
           ? patchNode(fleet.fleetRoot, fleet.state, nodeId, nodeState)
           : { ...fleet.state, nodes: { ...fleet.state.nodes, [nodeId]: nodeState } };
-        checkCostWarning();
+        checkCostLimits(fleet, ctx);
         updateWidget(ctx, fleet);
       },
       onNodeAdded: (w) => {
@@ -298,9 +352,10 @@ export async function startLoop(fleet: ActiveFleet, ctx: ExtensionContext, resum
     });
     fleet.state = state;
     fleet.running = false;
-    stopSpinner();
+    cleanup();
     updateWidget(ctx, fleet); // keep final per-node stats visible (todo #12)
-    const report = await writeReport({ spec: fleet.spec, state, fleetRoot: fleet.fleetRoot, repoCwd: ctx.cwd });
+    await writeReport({ spec: fleet.spec, state, fleetRoot: fleet.fleetRoot, repoCwd: ctx.cwd });
+    await writeFinalState();
     const last = state.iterations[state.iterations.length - 1];
     if (state.status === "paused" && last?.verdict === "escalate") {
       if (ctx.hasUI) ctx.ui.notify(`fleet paused: reviewer escalated; report: ${join(fleet.fleetRoot, "report.md")}`, "warning");
@@ -309,8 +364,9 @@ export async function startLoop(fleet: ActiveFleet, ctx: ExtensionContext, resum
     }
   } catch (err: unknown) {
     fleet.running = false;
-    stopSpinner();
+    cleanup();
     updateWidget(ctx, fleet);
+    await writeFinalState();
     const error = err instanceof Error ? err.message : String(err);
     if (ctx.hasUI) ctx.ui.notify(`fleet failed: ${error}`, "error");
   }

package/src/dag.ts

@@ -107,6 +107,10 @@ export function validateFleetSpec(
     errors.push(`config.effort must be one of ${THINKING_LEVELS.join(", ")}`);
   }
   const warnCost = typeof cfg.warn_cost_usd === "number" ? cfg.warn_cost_usd : undefined;
+  const maxCost = typeof cfg.max_cost_usd === "number" ? cfg.max_cost_usd : undefined;
+  const workerExtensions = Array.isArray(cfg.worker_extensions)
+    ? (cfg.worker_extensions as unknown[]).filter((e): e is string => typeof e === "string")
+    : undefined;
 
   const rawWorkers = Array.isArray(r?.workers) ? (r.workers as Record<string, unknown>[]) : [];
   if (rawWorkers.length === 0) errors.push("at least one worker required");
@@ -266,7 +270,7 @@ export function validateFleetSpec(
   const spec: FleetSpec = {
     fleet_name: String(r?.fleet_name ?? ""),
     type: "dag",
-    config: { max_concurrent: maxConcurrent, model, effort, warn_cost_usd: warnCost, loop: loopConfig },
+    config: { max_concurrent: maxConcurrent, model, effort, warn_cost_usd: warnCost, max_cost_usd: maxCost, worker_extensions: workerExtensions, loop: loopConfig },
     workers,
   };
 

package/src/edits.ts

@@ -8,7 +8,7 @@ import type { NodeStatus, ThinkingLevelName } from "./types.js";
 import { THINKING_LEVELS } from "./types.js";
 
 export type NodeEditKey = "model" | "effort" | "task";
-export type ConfigEditKey = "max_concurrent" | "warn_cost_usd" | "model" | "effort";
+export type ConfigEditKey = "max_concurrent" | "warn_cost_usd" | "max_cost_usd" | "worker_extensions" | "model" | "effort";
 
 export interface EditResult {
   ok: boolean;
@@ -85,6 +85,25 @@ export async function editConfig(
       fleet.spec.config.warn_cost_usd = n;
       break;
     }
+    case "max_cost_usd": {
+      const n = Number(value);
+      if (!Number.isFinite(n) || n < 0) return { ok: false, message: "max_cost_usd must be a number >= 0" };
+      fleet.spec.config.max_cost_usd = n;
+      break;
+    }
+    case "worker_extensions": {
+      // accept JSON array or comma-separated string
+      let list: string[];
+      try {
+        const parsed: unknown = JSON.parse(value);
+        list = Array.isArray(parsed) ? parsed.map(String) : [value];
+      } catch {
+        list = value.split(",").map((s) => s.trim()).filter(Boolean);
+      }
+      if (list.some((s) => s.length === 0)) return { ok: false, message: "worker_extensions must be non-empty strings" };
+      fleet.spec.config.worker_extensions = list;
+      break;
+    }
     case "model": {
       const r = resolveModelReference(registry, value);
       if (!r.ok) return { ok: false, message: r.error };
@@ -99,7 +118,7 @@ export async function editConfig(
       break;
     }
     default:
-      return { ok: false, message: `unknown config key "${String(key)}" (keys: max_concurrent, warn_cost_usd, model, effort)` };
+      return { ok: false, message: `unknown config key "${String(key)}" (keys: max_concurrent, warn_cost_usd, max_cost_usd, worker_extensions, model, effort)` };
   }
   await persistFleetJson(fleet);
   return { ok: true, message: `config.${key} updated` };

package/src/runner.ts

@@ -1,4 +1,4 @@
-import { createAgentSession, SessionManager, type CreateAgentSessionOptions } from "@earendil-works/pi-coding-agent";
+import { createAgentSession, SessionManager, DefaultResourceLoader, type CreateAgentSessionOptions } from "@earendil-works/pi-coding-agent";
 import type { Api, Model } from "@earendil-works/pi-ai";
 import type { ThinkingLevelName, WorkerSpec } from "./types.js";
 import { WORKER_TYPE_TOOLS } from "./types.js";
@@ -25,16 +25,52 @@ export interface SessionOpts {
   tools: string[];
   model?: string;
   thinkingLevel?: ThinkingLevelName;
+  extensionAllowlist?: string[];
 }
 
 export type SessionFactory = (opts: SessionOpts) => Promise<AgentSessionLike>;
 
+// Fleet workers must not load user extensions/skills: reading global skill files
+// burns tokens mid-task, and extensions like pi-web-access run background async work
+// that can crash the pi process after the session is disposed.
+// Exception: some model providers are registered BY extensions (e.g. opencode-pi
+// registers the opencode-cli provider). worker_extensions allowlists exactly those.
+export function filterExtensionsByAllowlist<T extends { path: string }>(
+  extensions: T[],
+  allowlist: string[] | undefined,
+): T[] {
+  if (!allowlist || allowlist.length === 0) return extensions;
+  return extensions.filter((e) => allowlist.some((a) => e.path.includes(a)));
+}
+
+async function createLeanResourceLoader(cwd: string, extensionAllowlist?: string[]): Promise<DefaultResourceLoader> {
+  const allow = extensionAllowlist && extensionAllowlist.length > 0;
+  const loader = new DefaultResourceLoader({
+    cwd,
+    agentDir: cwd,
+    noExtensions: !allow,
+    extensionsOverride: allow
+      ? (base) => ({ ...base, extensions: filterExtensionsByAllowlist(base.extensions, extensionAllowlist) })
+      : undefined,
+    noSkills: true,
+    noPromptTemplates: true,
+    noThemes: true,
+    noContextFiles: true,
+    systemPromptOverride: () => undefined,
+    appendSystemPromptOverride: () => [],
+  });
+  await loader.reload();
+  return loader;
+}
+
 export const defaultSessionFactory: SessionFactory = async (opts) => {
+  const resourceLoader = await createLeanResourceLoader(opts.cwd, opts.extensionAllowlist);
   const { session } = await createAgentSession({
     cwd: opts.cwd,
     tools: opts.tools,
     sessionManager: SessionManager.create(opts.cwd, opts.sessionDir),
     thinkingLevel: opts.thinkingLevel as ThinkingLevelOption,
+    resourceLoader,
   });
   return session as unknown as AgentSessionLike;
 };
@@ -49,6 +85,7 @@ export interface RunWorkerOpts {
   onSession?: (session: AgentSessionLike) => void;
   sessionFactory?: SessionFactory;
   thinkingLevel?: ThinkingLevelName;
+  extensionAllowlist?: string[];
 }
 
 export interface RunWorkerResult {
@@ -69,6 +106,7 @@ export async function runWorker(opts: RunWorkerOpts): Promise<RunWorkerResult> {
       tools: WORKER_TYPE_TOOLS[opts.worker.type],
       model: opts.worker.model,
       thinkingLevel: opts.thinkingLevel,
+      extensionAllowlist: opts.extensionAllowlist,
     });
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
@@ -79,13 +117,29 @@ export async function runWorker(opts: RunWorkerOpts): Promise<RunWorkerResult> {
   let turns = 0;
   let tokens = 0;
   let cost = 0;
+  // pi resolves session.prompt() normally even when the model run ends in an error
+  // (e.g. provider usage limit) — the failure is only visible on the final assistant
+  // message's stopReason/errorMessage. Track the last assistant message so a resolved
+  // prompt that actually errored is reported as ok:false instead of running the output
+  // contract against files that were never written.
+  let lastStopReason: string | undefined;
+  let lastErrorMessage: string | undefined;
   const unsub = session.subscribe((e) => {
     if (e.type === "turn_end") {
       turns++;
       opts.onEvent({ type: "turn", nodeId: opts.nodeId, turns });
     }
     if (e.type === "message_end") {
-      const msg = e.message as { role?: string; usage?: { totalTokens?: number; cost?: { total?: number } } } | undefined;
+      const msg = e.message as {
+        role?: string;
+        stopReason?: string;
+        errorMessage?: string;
+        usage?: { totalTokens?: number; cost?: { total?: number } };
+      } | undefined;
+      if (msg?.role === "assistant") {
+        lastStopReason = msg.stopReason;
+        lastErrorMessage = msg.errorMessage;
+      }
       if (msg?.role === "assistant" && msg.usage?.totalTokens) {
         tokens += msg.usage.totalTokens;
         opts.onEvent({ type: "tokens", nodeId: opts.nodeId, tokens });
@@ -98,6 +152,11 @@ export async function runWorker(opts: RunWorkerOpts): Promise<RunWorkerResult> {
   });
   try {
     await session.prompt(opts.prompt);
+    if (lastStopReason === "error") {
+      const message = lastErrorMessage ?? "worker model run ended with an error";
+      opts.onEvent({ type: "error", nodeId: opts.nodeId, message });
+      return { ok: false, turns, tokens, cost, error: message };
+    }
     opts.onEvent({ type: "done", nodeId: opts.nodeId });
     return { ok: true, turns, tokens, cost };
   } catch (err) {
@@ -112,12 +171,14 @@ export async function runWorker(opts: RunWorkerOpts): Promise<RunWorkerResult> {
 
 export function sessionFactoryForModel(model: Model<Api>): SessionFactory {
   return async (opts) => {
+    const resourceLoader = await createLeanResourceLoader(opts.cwd, opts.extensionAllowlist);
     const { session } = await createAgentSession({
       cwd: opts.cwd,
       tools: opts.tools,
       sessionManager: SessionManager.create(opts.cwd, opts.sessionDir),
       model,
       thinkingLevel: opts.thinkingLevel as ThinkingLevelOption,
+      resourceLoader,
     });
     return session as unknown as AgentSessionLike;
   };

package/src/scheduler.ts

@@ -25,6 +25,15 @@ export interface RunFleetOpts {
   continuePass?: boolean;
   onIterationEnd?: (snap: IterationSnapshot) => void;
   prepareIteration?: (n: number, state: FleetState) => Promise<void>;
+  retryDelayMs?: (attempt: number) => number;
+}
+
+const MAX_RETRY_ATTEMPTS = 3;
+const DEFAULT_RETRY_DELAY_MS = (attempt: number): number => 1000 * 2 ** attempt;
+const RETRYABLE_ERROR_RE = /usage limit|rate.?limit|429|overloaded|502|503|timed? ?out|timeout|econnreset|etimedout|econnrefused|fetch failed/i;
+
+export function isRetryableError(message: string): boolean {
+  return RETRYABLE_ERROR_RE.test(message);
 }
 
 const FAILED: ReadonlySet<string> = new Set(["failed", "contract_failed", "killed", "blocked"]);
@@ -52,6 +61,19 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
   let state: FleetState;
   if (opts.resumeFrom) {
     state = { ...opts.resumeFrom, paused: false, status: "running" };
+    // Crash recovery: nodes left "running" on disk by a dead process have no live
+    // session in this runFleet call — reset them to pending so they get dispatched.
+    // Without this, the scheduler can neither dispatch nor terminate them and spins
+    // forever (nothing runnable, not all nodes terminal).
+    for (const w of spec.workers) {
+      if (state.nodes[w.id]?.status === "running") {
+        state = patchNode(fleetRoot, state, w.id, {
+          status: "pending",
+          started_at: undefined,
+          status_note: "recovered stale running state from a previous crashed/killed process",
+        });
+      }
+    }
     await writeState(fleetRoot, state);
     if (allNodesTerminal(state, spec) && !opts.continuePass) {
       state = resetForIteration(state, spec);
@@ -71,6 +93,15 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
   };
 
   const running = new Set<Promise<void>>();
+  const circuitErrors = new Map<string, Set<string>>();
+  let circuitOpen = false;
+  function recordCircuitError(nodeId: string, error: string | undefined) {
+    if (!error) return;
+    const set = circuitErrors.get(error) ?? new Set<string>();
+    set.add(nodeId);
+    circuitErrors.set(error, set);
+    if (set.size >= 2) circuitOpen = true;
+  }
 
   const repoCwdFor = (nodeId: string): string =>
     typeof opts.repoCwd === "function" ? opts.repoCwd(nodeId) : opts.repoCwd;
@@ -179,6 +210,23 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
           await patch(w.id, { status: "killed", ended_at: new Date().toISOString() });
         }
       }
+      // circuit breaker: two distinct nodes failed with the identical error
+      if (circuitOpen) {
+        const [tripError, tripSet] = [...circuitErrors.entries()].find(([_, ids]) => ids.size >= 2) ?? ["", new Set<string>()];
+        const tripCount = tripSet.size;
+        const note = `circuit breaker: ${tripCount} nodes failed with identical error: ${tripError.slice(0, 80)}`;
+        for (const w of spec.workers) {
+          const n = state.nodes[w.id];
+          if (!n || (n.status !== "pending" && n.status !== "ready")) continue;
+          await patch(w.id, { status: "blocked", ended_at: new Date().toISOString(), status_note: note });
+        }
+        if (running.size > 0) {
+          await Promise.race(running);
+          continue;
+        }
+        break;
+      }
+
       // dispatch ready
       const activeCount = running.size;
       let slots = spec.config.max_concurrent - activeCount;
@@ -251,16 +299,55 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
 
         const dispatchMs = Date.now();
         await patch(w.id, { status: "running", started_at: new Date().toISOString() });
-        const p = opts.spawn(w.id).then(async (res) => {
+        const p = (async () => {
+          let totals = { turns: 0, tokens: 0, cost: 0 };
+          let lastRes: Awaited<ReturnType<typeof opts.spawn>> | undefined;
+          let finalError: string | undefined;
+
+          for (let attempt = 0; attempt < MAX_RETRY_ATTEMPTS; attempt++) {
+            if (opts.killSwitch?.killed) return;
+            if (opts.nodeKills?.has(w.id)) {
+              await patch(w.id, { status: "killed", ended_at: new Date().toISOString(), turns: totals.turns, tokens: totals.tokens, cost_usd_estimate: totals.cost });
+              return;
+            }
+            const res = await opts.spawn(w.id);
+            lastRes = res;
+            totals.turns += res.turns;
+            totals.tokens += res.tokens;
+            totals.cost += res.cost ?? 0;
+            if (opts.killSwitch?.killed) return;
+            if (opts.nodeKills?.has(w.id)) {
+              await patch(w.id, { status: "killed", ended_at: new Date().toISOString(), turns: totals.turns, tokens: totals.tokens, cost_usd_estimate: totals.cost });
+              return;
+            }
+            if (res.ok) {
+              finalError = undefined;
+              break;
+            }
+            finalError = res.error;
+            if (!res.error || !isRetryableError(res.error) || attempt === MAX_RETRY_ATTEMPTS - 1) {
+              break;
+            }
+            await patch(w.id, { status_note: `retry ${attempt + 1}/${MAX_RETRY_ATTEMPTS - 1} after: ${res.error}` });
+            const delayMs = opts.retryDelayMs ? opts.retryDelayMs(attempt) : DEFAULT_RETRY_DELAY_MS(attempt);
+            await new Promise((r) => setTimeout(r, delayMs));
+          }
+
+          const res = lastRes;
+          if (!res) return;
+
           if (opts.killSwitch?.killed) return;
           if (opts.nodeKills?.has(w.id)) {
-            await patch(w.id, { status: "killed", ended_at: new Date().toISOString(), turns: res.turns, tokens: res.tokens, cost_usd_estimate: res.cost ?? 0 });
+            await patch(w.id, { status: "killed", ended_at: new Date().toISOString(), turns: totals.turns, tokens: totals.tokens, cost_usd_estimate: totals.cost });
             return;
           }
+
           if (!res.ok) {
-            await patch(w.id, { status: "failed", ended_at: new Date().toISOString(), turns: res.turns, tokens: res.tokens, cost_usd_estimate: res.cost ?? 0 });
+            await patch(w.id, { status: "failed", ended_at: new Date().toISOString(), turns: totals.turns, tokens: totals.tokens, cost_usd_estimate: totals.cost });
+            recordCircuitError(w.id, finalError ?? res.error);
             return;
           }
+
           if (w.worktree) {
             try {
               await commitWorktree({
@@ -274,9 +361,9 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
               await patch(w.id, {
                 status: "failed",
                 ended_at: new Date().toISOString(),
-                turns: res.turns,
-                tokens: res.tokens,
-                cost_usd_estimate: res.cost ?? 0,
+                turns: totals.turns,
+                tokens: totals.tokens,
+                cost_usd_estimate: totals.cost,
                 status_note: `commit failed: ${msg}`,
               });
               return;
@@ -293,9 +380,9 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
           await patch(w.id, {
             status: contract.ok ? "completed" : "contract_failed",
             ended_at: new Date().toISOString(),
-            turns: res.turns,
-            tokens: res.tokens,
-            cost_usd_estimate: res.cost ?? 0,
+            turns: totals.turns,
+            tokens: totals.tokens,
+            cost_usd_estimate: totals.cost,
             contract_result: contract,
             produced_outputs: contract.checks.filter((c) => c.ok || c.actualPath).map((c) => c.path),
             status_note: contract.ok ? costNote : [contractFailureNote(contract.checks), costNote].filter(Boolean).join(" · "),
@@ -304,7 +391,7 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
             const note = await opts.onNodeCompleted?.(w.id);
             if (note) await patch(w.id, { status_note: note });
           }
-        }).finally(() => running.delete(p));
+        })().finally(() => running.delete(p));
         running.add(p);
       }
       if (running.size > 0) {
@@ -314,6 +401,33 @@ export async function runFleet(opts: RunFleetOpts): Promise<FleetState> {
         return !!n && TERMINAL_NODE_STATUSES.has(n.status);
       })) {
         break;
+      } else {
+        // Defensive: nothing in flight, not all terminal, and nothing dispatchable
+        // would mean a tight infinite loop (busy-spin, event-loop starvation).
+        // Fail the stuck nodes instead of hanging.
+        const dispatchable = spec.workers.some((w) => {
+          const n = state.nodes[w.id];
+          if (!n || (n.status !== "pending" && n.status !== "ready")) return false;
+          if (opts.nodeKills?.has(w.id)) return true; // will be killed next pass
+          return w.depends_on.every((d) => state.nodes[d]?.status === "completed");
+        });
+        const blockingDepsFailed = spec.workers.some((w) => {
+          const n = state.nodes[w.id];
+          if (!n || (n.status !== "pending" && n.status !== "ready")) return false;
+          return w.depends_on.some((d) => FAILED.has(state.nodes[d]?.status ?? ""));
+        });
+        if (!dispatchable && !blockingDepsFailed) {
+          for (const w of spec.workers) {
+            const n = state.nodes[w.id];
+            if (!n || TERMINAL_NODE_STATUSES.has(n.status)) continue;
+            await patch(w.id, {
+              status: "failed",
+              ended_at: new Date().toISOString(),
+              status_note: `stuck in non-terminal status "${n.status}" with no dispatchable path; failed to avoid scheduler hang`,
+            });
+          }
+          break;
+        }
       }
     }
     await Promise.allSettled([...running]);

package/src/state.ts

@@ -32,6 +32,8 @@ export async function readState(fleetRoot: string): Promise<FleetState> {
     lgtm_streak: parsed.lgtm_streak ?? 0,
     paused: parsed.paused ?? false,
     iterations: parsed.iterations ?? [],
+    pid: parsed.pid,
+    heartbeat_at: parsed.heartbeat_at,
   };
 }
 

package/src/tools.ts

@@ -58,6 +58,8 @@ export function registerFleetTools(pi: ExtensionAPI): void {
       model: Type.Optional(Type.String({ description: "Fleet-wide default model" })),
       effort: Type.Optional(EffortSchema),
       warn_cost_usd: Type.Optional(Type.Number()),
+      max_cost_usd: Type.Optional(Type.Number()),
+      worker_extensions: Type.Optional(Type.Array(Type.String())),
       loop: Type.Optional(Type.Object({
         gate: Type.Union([Type.Literal("reviewer"), Type.Literal("none")]),
         max_iterations: Type.Number(),
@@ -335,11 +337,11 @@ export function registerFleetTools(pi: ExtensionAPI): void {
   pi.registerTool({
     name: "fleet_edit",
     label: "Fleet Edit",
-    description: "Edit the active fleet: a pending or relaunchable node's model, effort, or task — or fleet config (max_concurrent, warn_cost_usd, model, effort) when node_id is omitted. Changes persist to fleet.json and apply immediately. Edits to running or completed nodes are refused; pending, blocked, failed, contract_failed, and killed nodes can be edited (blocked nodes have not started — nothing to invalidate).",
+    description: "Edit the active fleet: a pending or relaunchable node's model, effort, or task — or fleet config (max_concurrent, warn_cost_usd, max_cost_usd, model, effort) when node_id is omitted. Changes persist to fleet.json and apply immediately. Edits to running or completed nodes are refused; pending, blocked, failed, contract_failed, and killed nodes can be edited (blocked nodes have not started — nothing to invalidate).",
     promptSnippet: "Edit a pending fleet node or fleet config.",
     parameters: Type.Object({
       node_id: Type.Optional(Type.String({ description: "Worker id to edit; omit for fleet config edits" })),
-      key: Type.String({ description: "Node keys: model, effort, task. Config keys: max_concurrent, warn_cost_usd, model, effort" }),
+      key: Type.String({ description: "Node keys: model, effort, task. Config keys: max_concurrent, warn_cost_usd, max_cost_usd, model, effort" }),
       value: Type.String({ description: "New value" }),
     }),
     async execute(_id, params, _signal, _onUpdate, ctx) {

package/src/types.ts

@@ -44,6 +44,11 @@ export interface FleetConfig {
   model?: string;
   effort?: ThinkingLevelName;
   warn_cost_usd?: number;
+  /** hard cap — fleet is killed when total estimated cost reaches it */
+  max_cost_usd?: number;
+  /** extension path fragments to load inside worker sessions (default: none).
+   *  Needed for extension-provided model providers, e.g. ["opencode-pi"]. */
+  worker_extensions?: string[];
   loop?: LoopConfig;
 }
 
@@ -112,6 +117,10 @@ export interface FleetState {
   lgtm_streak: number;
   paused: boolean;
   iterations: IterationSnapshot[];
+  /** liveness signal written by the running controller */
+  pid?: number;
+  /** ISO timestamp liveness signal written by the running controller */
+  heartbeat_at?: string;
 }
 
 export const WORKER_TYPE_TOOLS: Record<WorkerType, string[]> = {