pi-orchestrator-config 4.1.5

package/extensions/orchestrator/async-capability.ts

@@ -0,0 +1,129 @@
+/**
+ * Async LLM capability — whether the parent session can host detached LLM async agents.
+ *
+ * acpx providers are registered by `extensions/acpx-provider` as `acpx-${agent}` for each
+ * entry in `acpx_agents` settings. Child pi processes skip acpx registration
+ * (PI_SUBAGENT_CHILD=1), so async children cannot use those parent models.
+ *
+ * Capability gate: any provider id starting with `acpx-` → supportsAsyncLlm false.
+ * Registration list (`isAcpxProvider`) still comes from `acpx_agents` settings.
+ *
+ * See: https://github.com/myk-org/pi-config/issues/647
+ */
+
+import { asStringArray, getSetting } from "./project-settings.js";
+
+export interface AsyncLlmSidecar {
+  provider: string;
+  model: string;
+}
+
+/**
+ * Provider ids we register for configured acpx agents (`acpx-cursor`, …).
+ * Source of truth: `acpx_agents` in project/global settings (same list acpx-provider uses).
+ */
+export function getRegisteredAcpxProviders(cwd: string): string[] {
+  // Defensive: stale getSetting may return non-array (issue #651).
+  return asStringArray(getSetting(cwd, "acpx_agents")).map(
+    (agent) => `acpx-${agent}`,
+  );
+}
+
+/** True when provider is one we registered via acpx-provider for this project. */
+export function isAcpxProvider(
+  provider: string | null | undefined,
+  cwd: string,
+): boolean {
+  if (typeof provider !== "string" || !provider) return false;
+  return getRegisteredAcpxProviders(cwd).includes(provider);
+}
+
+/** True when provider id uses the acpx-* namespace (registered or not). */
+export function isAcpxProviderId(
+  provider: string | null | undefined,
+): boolean {
+  return typeof provider === "string" && provider.startsWith("acpx-");
+}
+
+/**
+ * Whether detached LLM async agents can inherit the parent model/provider.
+ * Native / cli-* : true. Any `acpx-*` provider id: false (children skip acpx load).
+ * `cwd` kept for API stability / settings lookups elsewhere.
+ */
+export function supportsAsyncLlm(
+  provider: string | null | undefined,
+  _cwd: string,
+): boolean {
+  return !isAcpxProviderId(provider);
+}
+
+/**
+ * Sidecar model for must-async LLM work (dream, fireAndForget) when parent is acpx.
+ * Both provider and model must be set. Returns null if incomplete or sidecar is acpx-*.
+ */
+export function getAsyncLlmSidecar(cwd: string): AsyncLlmSidecar | null {
+  const provider = getSetting(cwd, "async_llm_provider");
+  const model = getSetting(cwd, "async_llm_model");
+  if (
+    typeof provider === "string" &&
+    provider.trim() &&
+    typeof model === "string" &&
+    model.trim()
+  ) {
+    const p = provider.trim();
+    if (isAcpxProviderId(p)) return null;
+    return { provider: p, model: model.trim() };
+  }
+  return null;
+}
+
+export type AsyncDispatchDecision =
+  | { action: "keep-async" }
+  | { action: "coerce-sync"; note: string }
+  | { action: "sidecar-async"; sidecar: AsyncLlmSidecar; note: string }
+  | { action: "skip"; note: string };
+
+/**
+ * Decide how to handle an async LLM request given parent provider + settings.
+ * Pure decision helper for subagent/dream/cron (code-enforced paths).
+ *
+ * Pass `parentSupportsAsyncLlm` when the caller already computed capability from the
+ * parent session cwd — do not re-derive it from a different (task) cwd.
+ * `cwd` is used only for sidecar settings lookup.
+ */
+export function decideAsyncLlmDispatch(opts: {
+  parentProvider?: string | null;
+  cwd: string;
+  /** fireAndForget / dream / cron — must stay detached when possible */
+  mustAsync?: boolean;
+  /** Explicit parent capability; when set, skips supportsAsyncLlm(parent, cwd). */
+  parentSupportsAsyncLlm?: boolean;
+}): AsyncDispatchDecision {
+  const asyncOk =
+    opts.parentSupportsAsyncLlm !== undefined
+      ? opts.parentSupportsAsyncLlm
+      : supportsAsyncLlm(opts.parentProvider, opts.cwd);
+  if (asyncOk) {
+    return { action: "keep-async" };
+  }
+  const sidecar = getAsyncLlmSidecar(opts.cwd);
+  if (opts.mustAsync) {
+    if (!sidecar) {
+      return {
+        action: "skip",
+        note:
+          "Skipped must-async LLM work: parent is acpx and " +
+          "async_llm_provider/async_llm_model are not set (or sidecar is acpx-*).",
+      };
+    }
+    return {
+      action: "sidecar-async",
+      sidecar,
+      note: `Using async_llm sidecar ${sidecar.provider}/${sidecar.model} (parent is acpx).`,
+    };
+  }
+  return {
+    action: "coerce-sync",
+    note: "Coerced async→sync: parent provider does not support async LLM (acpx).",
+  };
+}

package/extensions/orchestrator/async-runner.ts

@@ -0,0 +1,324 @@
+/**
+ * Async subagent runner — spawned as a detached process.
+ * Reads config from a JSON file, runs pi in print mode,
+ * writes status + result files for the parent to poll.
+ */
+
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { createRequire } from "node:module";
+import { getSetting } from "./project-settings.js";
+
+interface RunConfig {
+  id: string;
+  agent: string;
+  task: string;
+  cwd: string;
+  model?: string;
+  contextWindow?: number;
+  tools?: string[];
+  systemPrompt?: string;
+  resultPath: string;
+  workerDir: string;
+  sessionId?: string;
+  piCommand: string;
+  piArgs: string[];
+}
+
+interface StatusPayload {
+  runId: string;
+  state: "queued" | "running" | "complete" | "failed";
+  agent: string;
+  task: string;
+  startedAt: number;
+  endedAt?: number;
+  lastUpdate: number;
+  pid: number;
+  childPid?: number;
+  cwd: string;
+  exitCode?: number | null;
+  error?: string;
+  outputLines: number;
+}
+
+function writeJson(filePath: string, data: object): void {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  const tmp = `${filePath}.${process.pid}.tmp`;
+  try {
+    fs.writeFileSync(tmp, JSON.stringify(data, null, 2), "utf-8");
+    fs.renameSync(tmp, filePath);
+  } finally {
+    try { fs.unlinkSync(tmp); } catch {}
+  }
+}
+
+async function run(config: RunConfig): Promise<void> {
+  const statusPath = path.join(config.workerDir, "status.json");
+  const outputPath = path.join(config.workerDir, "output.log");
+  const startedAt = Date.now();
+
+  const status: StatusPayload = {
+    runId: config.id,
+    state: "running",
+    agent: config.agent,
+    task: config.task.slice(0, 200),
+    startedAt,
+    lastUpdate: startedAt,
+    pid: process.pid,
+    cwd: config.cwd,
+    outputLines: 0,
+  };
+  writeJson(statusPath, status);
+  let lastUsage: any = null;
+
+  const outputStream = fs.createWriteStream(outputPath, { flags: "w" });
+
+  // Connect to pidash server to stream events
+  const pidashPort = getSetting(config.cwd, "pidash_port");
+  let pidashWs: any = null;
+  const pidashLog = path.join(config.workerDir, "pidash-ws.log");
+  fs.writeFileSync(pidashLog, `START port=${pidashPort} id=${config.id}\n`);
+  try {
+    const _require = createRequire(import.meta.url);
+    fs.appendFileSync(pidashLog, `require ok, importing ws\n`);
+    const WebSocket = _require("ws");
+    fs.appendFileSync(pidashLog, `ws loaded, connecting...\n`);
+    pidashWs = new WebSocket(`ws://127.0.0.1:${pidashPort}/ws/async`);
+    pidashWs.on("open", () => {
+      fs.appendFileSync(pidashLog, `connected!\n`);
+      pidashWs.send(JSON.stringify({
+        type: "async_register",
+        id: config.id,
+        agent: config.agent,
+        task: config.task.slice(0, 200),
+        cwd: config.cwd,
+        sessionId: config.sessionId,
+      }));
+    });
+    pidashWs.on("error", (e: any) => { fs.appendFileSync(pidashLog, `error: ${e.message}\n`); });
+  } catch (e: any) {
+    fs.appendFileSync(pidashLog, `CATCH: ${e.message}\n${e.stack}\n`);
+  }
+
+  let lineCount = 0;
+  let finalOutput = "";
+  // Keep a small tail for fallback when no message_end is found
+  const MAX_TAIL = 4000;
+  let stdoutTail = "";
+  // Buffer for incomplete lines across chunk boundaries
+  let lineBuf = "";
+
+  const exitCode = await new Promise<number | null>((resolve) => {
+    const proc = spawn(config.piCommand, config.piArgs, {
+      cwd: config.cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    // Store child PID in status file so /async-kill can find it
+    if (proc.pid) {
+      status.childPid = proc.pid;
+      writeJson(statusPath, status);
+    }
+
+    proc.stdout.on("data", (chunk: Buffer) => {
+      const text = chunk.toString();
+      outputStream.write(text);
+
+      // Keep only the last MAX_TAIL chars for fallback
+      stdoutTail += text;
+      if (stdoutTail.length > MAX_TAIL * 2) {
+        stdoutTail = stdoutTail.slice(-MAX_TAIL);
+      }
+
+      // Buffer partial lines across chunk boundaries
+      lineBuf += text;
+      const lines = lineBuf.split("\n");
+      // Last element is incomplete (no trailing newline) — keep in buffer
+      lineBuf = lines.pop() || "";
+
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        lineCount++;
+
+        try {
+          const ev = JSON.parse(trimmed);
+
+          // Extract final output and usage from message_end events inline
+          if (ev.type === "message_end" && ev.message?.role === "assistant") {
+            for (const p of ev.message.content || []) {
+              if (p.type === "text") finalOutput = p.text;
+            }
+            if (ev.message.usage) lastUsage = ev.message.usage;
+          }
+
+          // Forward events to pidash
+          if (pidashWs?.readyState === 1) {
+            pidashWs.send(JSON.stringify({ type: "async_event", id: config.id, event: ev }));
+          }
+        } catch (e: any) { console.debug("[async-runner] event processing failed:", e?.message || e); }
+      }
+
+      // Update status periodically (every ~10 lines)
+      if (lineCount % 10 === 0) {
+        status.lastUpdate = Date.now();
+        status.outputLines = lineCount;
+        writeJson(statusPath, status);
+      }
+    });
+
+    proc.stderr.on("data", (chunk: Buffer) => {
+      outputStream.write(chunk.toString());
+    });
+
+    proc.on("close", (code) => {
+      resolve(code);
+    });
+
+    proc.on("error", (err) => {
+      status.error = err.message;
+      resolve(1);
+    });
+  });
+
+  const endedAt = Date.now();
+
+  // Process any remaining buffered line
+  if (lineBuf.trim()) {
+    try {
+      const ev = JSON.parse(lineBuf.trim());
+      if (ev.type === "message_end" && ev.message?.role === "assistant") {
+        for (const p of ev.message.content || []) {
+          if (p.type === "text") finalOutput = p.text;
+        }
+      }
+    } catch (e: any) { console.debug("[async-runner] final line parse failed:", e?.message || e); }
+  }
+
+  // Write final status
+  status.state = exitCode === 0 ? "complete" : "failed";
+  status.endedAt = endedAt;
+  status.lastUpdate = endedAt;
+  status.exitCode = exitCode;
+  status.outputLines = lineCount;
+  writeJson(statusPath, status);
+
+  // Notify pidash of completion and close
+  if (pidashWs?.readyState === 1) {
+    pidashWs.send(JSON.stringify({ type: "async_complete", id: config.id, success: exitCode === 0 }));
+    pidashWs.close();
+  }
+
+  // For code reviewers: validate output is JSON, retry if not
+  if (config.agent.startsWith("code-reviewer-") && exitCode === 0) {
+    let validJson = false;
+    let retryOutput = finalOutput || stdoutTail.slice(-2000);
+    let jsonRetryCount = 0;
+    while (!validJson) {
+      jsonRetryCount++;
+      try {
+        let cleaned = retryOutput.trim();
+        if (cleaned.startsWith("```")) {
+          cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
+        }
+        const parsed = JSON.parse(cleaned);
+        if (parsed && Array.isArray(parsed.findings)) {
+          validJson = true;
+          finalOutput = cleaned; // Use the cleaned JSON as final output
+        }
+      } catch { /* not valid JSON */ }
+
+      if (!validJson) {
+        // After 3 retries, stop and let the user decide
+        if (jsonRetryCount >= 3) {
+          finalOutput = JSON.stringify({
+            findings: [{
+              severity: "CRITICAL",
+              file: "<reviewer-output>",
+              line: 0,
+              description: `Reviewer ${config.agent} failed to return valid JSON after ${jsonRetryCount} attempts. Raw output included in this result. User action required.`,
+              suggestion: "Re-run the reviewer or check the agent prompt.",
+            }],
+            _jsonRetryExhausted: true,
+            _rawOutput: retryOutput.slice(0, 1000),
+          });
+          break;
+        }
+        // Re-run with same session, asking for JSON
+        const retryArgs = config.piArgs.slice(0, -1); // Remove the last arg (the task)
+        retryArgs.push("Your output is not valid JSON. Return ONLY a raw JSON object: {\"findings\": [...]}. No markdown fences, no text before or after.");
+        const retryCode = await new Promise<number | null>((resolve) => {
+          const retryProc = spawn(config.piCommand, retryArgs, {
+            cwd: config.cwd,
+            stdio: ["ignore", "pipe", "pipe"],
+          });
+          let retryBuf = "";
+          retryProc.stdout.on("data", (chunk: Buffer) => {
+            retryBuf += chunk.toString();
+            outputStream.write(chunk.toString());
+          });
+          retryProc.stderr.on("data", (chunk: Buffer) => {
+            outputStream.write(chunk.toString());
+          });
+          retryProc.on("close", (code) => {
+            // Extract final output from the retry
+            for (const line of retryBuf.split("\n")) {
+              try {
+                const ev = JSON.parse(line.trim());
+                if (ev.type === "message_end" && ev.message?.role === "assistant") {
+                  for (const p of ev.message.content || []) {
+                    if (p.type === "text") retryOutput = p.text;
+                  }
+                }
+              } catch { /* skip */ }
+            }
+            resolve(code);
+          });
+          retryProc.on("error", () => resolve(1));
+        });
+        if (retryCode !== 0) break; // pi itself failed, stop retrying
+      }
+    }
+  }
+
+  // End the output stream after all retries are done
+  outputStream.end();
+
+  // Write result for the watcher to pick up
+  writeJson(config.resultPath, {
+    id: config.id,
+    agent: config.agent,
+    task: config.task,
+    success: exitCode === 0,
+    output: finalOutput || stdoutTail.slice(-2000),
+    exitCode,
+    startedAt,
+    endedAt,
+    durationMs: endedAt - startedAt,
+    cwd: config.cwd,
+    sessionId: config.sessionId,
+    workerDir: config.workerDir,
+    lastUsage: lastUsage || null,
+    contextWindow: config.contextWindow || 0,
+  });
+}
+
+// Entry point
+const configPath = process.argv[2];
+if (!configPath) {
+  console.error("Usage: async-runner.ts <config.json>");
+  process.exit(1);
+}
+
+try {
+  const config = JSON.parse(fs.readFileSync(configPath, "utf-8")) as RunConfig;
+  try { fs.unlinkSync(configPath); } catch {}
+  run(config).catch((err) => {
+    console.error("Async runner error:", err);
+    process.exit(1);
+  });
+} catch (err) {
+  console.error("Failed to read config:", err);
+  process.exit(1);
+}

package/extensions/orchestrator/async-status-parse.ts

@@ -0,0 +1,31 @@
+/**
+ * Pure parsers for async agent output.log JSONL — no TUI deps (unit-testable).
+ */
+
+/** Parse one JSONL event line from async output.log into display text. */
+export function parseAsyncOutputLine(raw: string): string | null {
+  try {
+    const ev = JSON.parse(raw);
+    if (ev.type === "message_update" && ev.assistantMessageEvent) {
+      const ae = ev.assistantMessageEvent;
+      if (ae.type === "text_delta" && ae.delta) return ae.delta;
+      return null;
+    }
+    if (ev.type === "tool_execution_start") {
+      const name = ev.toolName || "tool";
+      const cmd = ev.args?.command
+        ? ` ${String(ev.args.command).slice(0, 80)}`
+        : "";
+      return `\n→ ${name}${cmd}`;
+    }
+    if (ev.type === "tool_execution_end") {
+      const text = ev.result?.content?.[0]?.text || "";
+      const prefix = ev.isError ? "✗" : "✓";
+      return `\n${prefix} ${String(text).slice(0, 200)}`;
+    }
+    if (ev.type === "agent_end") return "\n--- Agent finished ---";
+    return null;
+  } catch {
+    return null;
+  }
+}

package/extensions/orchestrator/async-status-ui.ts

@@ -0,0 +1,201 @@
+/**
+ * /async-status fullscreen overlay — list dashboard → live output detail.
+ */
+
+import type { ExtensionCommandContext, Theme } from "@earendil-works/pi-coding-agent";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { parseAsyncOutputLine } from "./async-status-parse.js";
+import {
+  openListDetailOverlay,
+  OverlayScrollDetail,
+} from "./overlay-dashboard.js";
+
+export { parseAsyncOutputLine } from "./async-status-parse.js";
+
+export interface AsyncStatusJobView {
+  id: string;
+  agent: string;
+  name?: string;
+  task: string;
+  status: string;
+  workerDir: string;
+  startedAt: number;
+  durationMs?: number;
+  model?: string;
+}
+
+export interface AsyncStatusUiDeps {
+  listJobs: () => AsyncStatusJobView[];
+  killJob: (id: string) => void;
+  formatDuration: (ms: number) => string;
+  /** Optional live status.json reader for detail header. */
+  readLiveStatus?: (workerDir: string) => { state?: string } | null;
+  title?: string;
+  emptyMessage?: string;
+  footerHints?: string;
+}
+
+function jobTitle(job: AsyncStatusJobView): string {
+  return job.name || job.agent;
+}
+
+function isActive(status: string): boolean {
+  return status === "running" || status === "queued";
+}
+
+function statusGlyph(status: string, theme: Theme): string {
+  switch (status) {
+    case "running":
+    case "queued":
+      return theme.fg("warning", "■");
+    case "complete":
+      return "\x1b[32m■\x1b[0m";
+    case "failed":
+      return theme.fg("error", "■");
+    default:
+      return theme.fg("muted", "■");
+  }
+}
+
+function statusWord(status: string, theme: Theme): string {
+  switch (status) {
+    case "running":
+      return theme.fg("warning", "running");
+    case "queued":
+      return theme.fg("warning", "queued");
+    case "complete":
+      return "\x1b[32mdone\x1b[0m";
+    case "failed":
+      return theme.fg("error", "failed");
+    default:
+      return theme.fg("muted", status);
+  }
+}
+
+function elapsedMs(job: AsyncStatusJobView): number {
+  if (job.durationMs != null && !isActive(job.status)) return job.durationMs;
+  return Date.now() - job.startedAt;
+}
+
+/**
+ * Open fullscreen async-status picker. Loops list → detail until user closes list.
+ */
+export async function openAsyncStatusOverlay(
+  ctx: ExtensionCommandContext,
+  deps: AsyncStatusUiDeps,
+): Promise<void> {
+  await openListDetailOverlay(ctx, {
+    emptyMessage:
+      deps.emptyMessage ?? "No async agents running or recently completed.",
+    listSpec: {
+      title: deps.title ?? "Async agents",
+      countLabel: (jobs) => `${jobs.length} job${jobs.length === 1 ? "" : "s"}`,
+      borderTitle: (jobs) => {
+        const active = jobs.filter((j) => isActive(j.status)).length;
+        return `jobs · ${active} active / ${jobs.length}`;
+      },
+      footerHints:
+        deps.footerHints ?? "↑↓/jk select · Enter view · x kill · Esc close",
+      listItems: () => deps.listJobs(),
+      rowParts: (job, theme) => {
+        const shortId = job.id.length > 8 ? job.id.slice(-8) : job.id;
+        const taskPreview =
+          job.task.length > 40 ? `${job.task.slice(0, 40)}…` : job.task;
+        return {
+          glyph: statusGlyph(job.status, theme),
+          title: jobTitle(job),
+          idLabel: theme.fg("dim", shortId),
+          rightParts: [
+            theme.fg("muted", job.agent),
+            ...(job.model ? [theme.fg("dim", job.model)] : []),
+            theme.fg("dim", taskPreview),
+            theme.fg("muted", deps.formatDuration(elapsedMs(job))),
+            statusWord(job.status, theme),
+          ],
+        };
+      },
+      onX: (job) => {
+        if (isActive(job.status)) deps.killJob(job.id);
+      },
+    },
+    createDetail: (job, tui, theme, done) => {
+      const lines: string[] = [];
+      let filePos = 0;
+      let textBuffer = "";
+      let lastLoggedError = "";
+
+      const readNewContent = (): boolean => {
+        try {
+          const content = fs.readFileSync(
+            path.join(job.workerDir, "output.log"),
+            "utf-8",
+          );
+          if (content.length <= filePos) return false;
+          const newContent = content.slice(filePos);
+          filePos = content.length;
+          textBuffer += newContent;
+
+          const parts = textBuffer.split("\n");
+          textBuffer = parts.pop() || "";
+          let changed = false;
+          for (const part of parts) {
+            if (!part.trim()) continue;
+            const parsed = parseAsyncOutputLine(part);
+            if (parsed === null) continue;
+            for (const l of parsed.split("\n")) lines.push(l);
+            changed = true;
+          }
+          return changed;
+        } catch (e: any) {
+          if (e?.code === "ENOENT") return false;
+          const msg = e?.message || String(e);
+          if (msg !== lastLoggedError) {
+            lastLoggedError = msg;
+            console.debug("[async-status-ui] live output read failed:", msg);
+          }
+          return false;
+        }
+      };
+
+      const jobId = job.id;
+      return new OverlayScrollDetail(
+        tui,
+        theme,
+        {
+          followTail: true,
+          emptyBody: "(no output yet)",
+          footerHints:
+            "↑↓/jk scroll · PgUp/PgDn · Home/End · x kill · Esc back",
+          pollMs: 500,
+          onPoll: readNewContent,
+          onX: () => {
+            const current = deps.listJobs().find((j) => j.id === jobId);
+            if (current && isActive(current.status)) {
+              deps.killJob(jobId);
+              return true;
+            }
+            return false;
+          },
+          getHeader: (t) => {
+            const live = deps.readLiveStatus?.(job.workerDir);
+            const current = deps.listJobs().find((j) => j.id === jobId) || job;
+            const state = live?.state || current.status;
+            const dur = deps.formatDuration(elapsedMs(current));
+            return (
+              `${statusGlyph(state, t)} ` +
+              t.fg(
+                "accent",
+                t.bold(`${jobTitle(current)} · ${current.id.slice(-8)}`),
+              ) +
+              t.fg("muted", ` · ${state} · ${dur}`) +
+              t.fg("dim", ` · ${current.task.slice(0, 40)}`)
+            );
+          },
+          getBodyLines: () => lines,
+        },
+        done,
+      );
+    },
+  });
+}