pi-workflow-engine 0.2.0 → 0.4.0

package/.pi/extensions/pi-workflow-engine/src/perf.ts

@@ -0,0 +1,139 @@
+import { performance } from "node:perf_hooks";
+
+export type PerfTags = Record<string, string | number>;
+
+export interface PerfSample {
+  readonly name: string;
+  readonly durationMs?: number;
+  readonly value?: number;
+  readonly tags?: PerfTags;
+}
+
+export interface PerfAggregate {
+  readonly name: string;
+  readonly count: number;
+  readonly total: number;
+  readonly min: number;
+  readonly max: number;
+  readonly mean: number;
+  readonly p50: number;
+  readonly p95: number;
+}
+
+export interface PerfSnapshot {
+  readonly enabled: boolean;
+  readonly startedAt: number;
+  readonly samples: readonly PerfSample[];
+  readonly aggregates: readonly PerfAggregate[];
+}
+
+export interface PerfSink {
+  time<T>(name: string, fn: () => Promise<T>, tags?: PerfTags): Promise<T>;
+  timeSync<T>(name: string, fn: () => T, tags?: PerfTags): T;
+  observe(name: string, value: number, tags?: PerfTags): void;
+  counter(name: string, delta?: number, tags?: PerfTags): void;
+  snapshot(): PerfSnapshot;
+}
+
+export class PerfRecorder implements PerfSink {
+  private readonly samples: PerfSample[] = [];
+
+  constructor(private readonly startedAt = Date.now()) {}
+
+  async time<T>(name: string, fn: () => Promise<T>, tags?: PerfTags): Promise<T> {
+    const start = performance.now();
+    try {
+      return await fn();
+    } finally {
+      this.observeDuration(name, performance.now() - start, tags);
+    }
+  }
+
+  timeSync<T>(name: string, fn: () => T, tags?: PerfTags): T {
+    const start = performance.now();
+    try {
+      return fn();
+    } finally {
+      this.observeDuration(name, performance.now() - start, tags);
+    }
+  }
+
+  observe(name: string, value: number, tags?: PerfTags): void {
+    if (!Number.isFinite(value)) return;
+    this.samples.push({ name, value, tags });
+  }
+
+  counter(name: string, delta = 1, tags?: PerfTags): void {
+    this.observe(name, delta, tags);
+  }
+
+  snapshot(): PerfSnapshot {
+    return {
+      enabled: true,
+      startedAt: this.startedAt,
+      samples: this.samples.map((sample) => ({ ...sample, tags: sample.tags ? { ...sample.tags } : undefined })),
+      aggregates: aggregateSamples(this.samples),
+    };
+  }
+
+  private observeDuration(name: string, durationMs: number, tags?: PerfTags): void {
+    if (!Number.isFinite(durationMs)) return;
+    this.samples.push({ name, durationMs, tags });
+  }
+}
+
+export class NoopPerfRecorder implements PerfSink {
+  constructor(private readonly startedAt = Date.now()) {}
+
+  async time<T>(_name: string, fn: () => Promise<T>, _tags?: PerfTags): Promise<T> {
+    return await fn();
+  }
+
+  timeSync<T>(_name: string, fn: () => T, _tags?: PerfTags): T {
+    return fn();
+  }
+
+  observe(_name: string, _value: number, _tags?: PerfTags): void {}
+
+  counter(_name: string, _delta?: number, _tags?: PerfTags): void {}
+
+  snapshot(): PerfSnapshot {
+    return { enabled: false, startedAt: this.startedAt, samples: [], aggregates: [] };
+  }
+}
+
+export function createPerfRecorder(enabled: boolean, startedAt?: number): PerfSink {
+  return enabled ? new PerfRecorder(startedAt) : new NoopPerfRecorder(startedAt);
+}
+
+function aggregateSamples(samples: readonly PerfSample[]): PerfAggregate[] {
+  const byName = new Map<string, number[]>();
+  for (const sample of samples) {
+    const value = sample.durationMs ?? sample.value;
+    if (value === undefined || !Number.isFinite(value)) continue;
+    const values = byName.get(sample.name) ?? [];
+    values.push(value);
+    byName.set(sample.name, values);
+  }
+
+  return [...byName.entries()].map(([name, rawValues]) => {
+    const values = [...rawValues].sort((a, b) => a - b);
+    const total = values.reduce((sum, value) => sum + value, 0);
+    return {
+      name,
+      count: values.length,
+      total,
+      min: values[0] ?? 0,
+      max: values[values.length - 1] ?? 0,
+      mean: values.length === 0 ? 0 : total / values.length,
+      p50: percentile(values, 0.5),
+      p95: percentile(values, 0.95),
+    };
+  });
+}
+
+function percentile(sortedValues: readonly number[], p: number): number {
+  if (sortedValues.length === 0) return 0;
+  const index = Math.min(sortedValues.length - 1, Math.max(0, Math.ceil(sortedValues.length * p) - 1));
+  return sortedValues[index] ?? 0;
+}

package/{src → .pi/extensions/pi-workflow-engine/src}/progress.ts

@@ -1,7 +1,7 @@
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import type { TUI } from "@earendil-works/pi-tui";
 import type { WorkflowLaneItemStatus, WorkflowProgressEvent } from "./types.ts";
-import { statusText } from "./ui/workflow-format.ts";
+import { statusTextFromCounts, type WorkflowStatusCounts } from "./ui/workflow-format.ts";
 import { createWorkflowWidget, type WorkflowWidget } from "./ui/workflow-widget.ts";
 
 export type AgentRowStatus = "queued" | "running" | "done" | "failed";
@@ -46,6 +46,7 @@ export interface WorkflowProgressSnapshot {
   readonly counters: readonly WorkflowCounterSnapshot[];
   readonly summary: readonly [string, string | number][];
   readonly lanes: readonly [string, readonly WorkflowLaneItemSnapshot[]][];
+  readonly laneOverflow: readonly [string, number][];
   readonly logs: readonly string[];
 }
 
@@ -81,6 +82,7 @@ interface WorkflowLaneItem {
 }
 
 const LOG_LIMIT = 24;
+export const DEFAULT_LANE_ITEM_LIMIT = 200;
 
 /**
  * Tracks live workflow state for widgets, footer/status text, result renderers,
@@ -92,7 +94,11 @@ export class ProgressTracker {
   private readonly counters = new Map<string, WorkflowCounter>();
   private readonly summary = new Map<string, string | number>();
   private readonly lanes = new Map<string, WorkflowLaneItem[]>();
+  private readonly laneOverflow = new Map<string, number>();
+  private readonly rowsById = new Map<number, AgentRow>();
+  private readonly agentCounts: Record<AgentRowStatus, number> = { queued: 0, running: 0, done: 0, failed: 0 };
   private readonly startedAt = Date.now();
+  private readonly laneItemLimit = laneItemLimitFromEnv();
   private doneAt: number | undefined;
   private currentPhase = "Workflow";
   private nextAgentId = 1;
@@ -101,6 +107,7 @@ export class ProgressTracker {
   private tui: Pick<TUI, "requestRender"> | undefined;
   private widgetInterval: ReturnType<typeof setInterval> | undefined;
   private lastStatusText: string | undefined;
+  private renderQueued = false;
 
   constructor(
     private readonly ctx: ExtensionContext,
@@ -156,6 +163,7 @@ export class ProgressTracker {
           details: event.details,
           createdAt: Date.now(),
         });
+        this.pruneLane(event.lane, lane);
         this.lanes.set(event.lane, lane);
         break;
       }
@@ -168,7 +176,10 @@ export class ProgressTracker {
 
   agentQueued(phase: string | undefined, label: string): number {
     const id = this.nextAgentId++;
-    this.ensurePhase(phase ?? this.currentPhase).agents.push({ label, id, status: "queued", toolUses: 0 });
+    const row = { label, id, status: "queued" as const, toolUses: 0 };
+    this.ensurePhase(phase ?? this.currentPhase).agents.push(row);
+    this.rowsById.set(id, row);
+    this.agentCounts.queued++;
     this.render();
     return id;
   }
@@ -176,17 +187,20 @@ export class ProgressTracker {
   agentStart(phase: string | undefined, label: string, id?: number): void {
     const row = id === undefined ? undefined : this.findRowById(id);
     if (row) {
-      row.status = "running";
+      this.transitionAgentStatus(row, "running");
       row.startedAt = Date.now();
       row.error = undefined;
     } else {
-      this.ensurePhase(phase ?? this.currentPhase).agents.push({
+      const nextRow = {
         label,
         id: this.nextAgentId++,
-        status: "running",
+        status: "running" as const,
         startedAt: Date.now(),
         toolUses: 0,
-      });
+      };
+      this.ensurePhase(phase ?? this.currentPhase).agents.push(nextRow);
+      this.rowsById.set(nextRow.id, nextRow);
+      this.agentCounts.running++;
     }
     this.render();
   }
@@ -203,7 +217,7 @@ export class ProgressTracker {
   agentDone(label: string, id?: number): void {
     const row = this.findRow(label, id);
     if (row && row.status !== "failed") {
-      row.status = "done";
+      this.transitionAgentStatus(row, "done");
       row.doneAt = Date.now();
     }
     this.render();
@@ -212,7 +226,7 @@ export class ProgressTracker {
   agentFailed(label: string, error: unknown, id?: number): void {
     const row = this.findRow(label, id);
     if (row) {
-      row.status = "failed";
+      this.transitionAgentStatus(row, "failed");
       row.doneAt = Date.now();
       row.error = errorMessage(error);
     }
@@ -232,10 +246,40 @@ export class ProgressTracker {
       counters: [...this.counters.values()].map((counter) => ({ ...counter })),
       summary: [...this.summary.entries()],
       lanes: [...this.lanes.entries()].map(([lane, items]) => [lane, items.map((item) => ({ ...item }))]),
+      laneOverflow: [...this.laneOverflow.entries()],
       logs: [...this.logs],
     };
   }
 
+  statusCounts(): WorkflowStatusCounts {
+    return this.statusCountsSnapshot();
+  }
+
+  private statusCountsSnapshot(): WorkflowStatusCounts {
+    return {
+      queued: this.agentCounts.queued,
+      running: this.agentCounts.running,
+      done: this.agentCounts.done,
+      failed: this.agentCounts.failed,
+      total: this.agentCounts.queued + this.agentCounts.running + this.agentCounts.done + this.agentCounts.failed,
+    };
+  }
+
+  private transitionAgentStatus(row: AgentRow, nextStatus: AgentRowStatus): void {
+    if (row.status === nextStatus) return;
+    this.agentCounts[row.status]--;
+    row.status = nextStatus;
+    this.agentCounts[nextStatus]++;
+  }
+
+  private pruneLane(laneName: string, lane: WorkflowLaneItem[]): void {
+    if (this.laneItemLimit <= 0) return;
+    while (lane.length > this.laneItemLimit) {
+      lane.shift();
+      this.laneOverflow.set(laneName, (this.laneOverflow.get(laneName) ?? 0) + 1);
+    }
+  }
+
   private findRow(label: string, id?: number): AgentRow | undefined {
     if (id !== undefined) return this.findRowById(id);
     for (let i = this.phases.length - 1; i >= 0; i--) {
@@ -245,48 +289,44 @@ export class ProgressTracker {
       if (running) return running;
     }
     for (let i = this.phases.length - 1; i >= 0; i--) {
-      const any = this.phases[i].agents.find((agent) => agent.label === label);
-      if (any) return any;
+      const matching = this.phases[i].agents.find((agent) => agent.label === label);
+      if (matching) return matching;
     }
     return undefined;
   }
 
   private findRowById(id: number): AgentRow | undefined {
-    for (const phase of this.phases) {
-      const row = phase.agents.find((agent) => agent.id === id);
-      if (row) return row;
-    }
-    return undefined;
-  }
-
-  private lines(): string[] {
-    const out: string[] = [`⚙ ${this.title}`];
-    for (const phase of this.phases) {
-      if (phase.agents.length === 0 && phase.title === "Workflow") continue;
-      out.push(`  ${phase.title}`);
-      for (const agent of phase.agents) {
-        const icon = agent.status === "done" ? "✓" : agent.status === "failed" ? "✗" : agent.status === "queued" ? "○" : "⏳";
-        const tool = agent.status === "running" && agent.lastTool ? ` · ${agent.lastTool}` : "";
-        const error = agent.status === "failed" && agent.error ? ` · ${agent.error}` : "";
-        out.push(`    ${icon} ${agent.label}${tool}${error}`);
-      }
-    }
-    const counterLine = [...this.counters.values()].map((counter) => `${counter.label}: ${counter.value}`).join(" · ");
-    if (counterLine) out.push(`  ${counterLine}`);
-    for (const line of this.logs.slice(-6)) out.push(`  · ${line}`);
-    return out;
+    return this.rowsById.get(id);
   }
 
   private render(): void {
     if (!this.ctx.hasUI) return;
     this.ensureWidget();
     this.widget?.invalidate();
-    this.tui?.requestRender();
+    this.requestRenderSoon();
     this.publishStatus();
   }
 
+  private requestRenderSoon(): void {
+    if (this.renderQueued) return;
+    this.renderQueued = true;
+    queueMicrotask(() => {
+      this.renderQueued = false;
+      this.tui?.requestRender();
+    });
+  }
+
   private publishStatus(): void {
-    const next = statusText(this.snapshot(), this.ctx.ui.theme);
+    const next = statusTextFromCounts(
+      {
+        title: this.title,
+        doneAt: this.doneAt,
+        currentPhase: this.currentPhase,
+        counters: [...this.counters.values()].map((counter) => ({ ...counter })),
+      },
+      this.statusCountsSnapshot(),
+      this.ctx.ui.theme,
+    );
     if (next === this.lastStatusText) return;
     this.ctx.ui.setStatus("workflow", next);
     this.lastStatusText = next;
@@ -332,12 +372,19 @@ export class ProgressTracker {
     this.ctx.ui.setWidget("workflow", undefined);
     this.ctx.ui.setStatus("workflow", status);
     this.lastStatusText = status;
+    this.renderQueued = false;
     this.widgetRegistered = false;
     this.widget = undefined;
     this.tui = undefined;
   }
 }
 
+function laneItemLimitFromEnv(): number {
+  const parsed = Number(process.env.PI_WORKFLOW_LANE_ITEM_LIMIT ?? "");
+  if (!Number.isFinite(parsed) || parsed < 1) return DEFAULT_LANE_ITEM_LIMIT;
+  return Math.trunc(parsed);
+}
+
 function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }

package/{src → .pi/extensions/pi-workflow-engine/src}/types.ts

@@ -1,5 +1,7 @@
 import type { Static, TSchema } from "typebox";
 import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import type { Pipeline } from "./concurrency.ts";
+import type { PerfSink, PerfSnapshot } from "./perf.ts";
 
 /** Metadata every workflow module must export. */
 export interface WorkflowMeta {
@@ -18,6 +20,15 @@ export interface WorkflowRunStats {
 
 export interface WorkflowRunOptions {
   inspect?: boolean;
+  perf?: boolean;
+  concurrency?: number;
+  parallelSubmissionLimit?: number;
+  /** Additional abort signal to compose with the host context signal. */
+  signal?: AbortSignal;
+  /** Internal recorder override for command/tool invocation timing. */
+  perfRecorder?: PerfSink;
+  /** Called with the final performance snapshot when perf is enabled. */
+  onPerfSnapshot?: (snapshot: PerfSnapshot) => void;
 }
 
 /** Options for a single `agent()` call. */
@@ -63,10 +74,7 @@ export interface WorkflowApi {
   /** Run every thunk concurrently and wait for all (a barrier). */
   parallel<T>(thunks: Array<() => Promise<T>>): Promise<T[]>;
   /** Run each item through all stages independently — no barrier between stages. */
-  pipeline(
-    items: unknown[],
-    ...stages: Array<(prev: unknown, item: unknown, index: number) => Promise<unknown>>
-  ): Promise<unknown[]>;
+  pipeline: Pipeline;
   /** Start a new phase; subsequent agents group under it in the progress tree. */
   phase(title: string): void;
   /** Emit a progress line. */

package/{src → .pi/extensions/pi-workflow-engine/src}/ui/workflow-format.ts

@@ -1,6 +1,6 @@
 import type { Theme } from "@earendil-works/pi-coding-agent";
 import { truncateToWidth } from "@earendil-works/pi-tui";
-import type { WorkflowProgressSnapshot } from "../progress.ts";
+import type { AgentRowSnapshot, WorkflowProgressSnapshot } from "../progress.ts";
 import type { WorkflowLaneItemStatus } from "../types.ts";
 
 export type WorkflowDisplayStatus = WorkflowLaneItemStatus | "queued" | "done" | "failed";
@@ -58,24 +58,77 @@ export function truncateDisplay(text: string, width: number): string {
   return truncateToWidth(text, width);
 }
 
+export interface AgentDetailOptions {
+  now?: number;
+  includeQueuedStatus?: boolean;
+}
+
+export function agentDetailParts(agent: AgentRowSnapshot, now?: number): string[];
+export function agentDetailParts(agent: AgentRowSnapshot, options?: AgentDetailOptions): string[];
+export function agentDetailParts(agent: AgentRowSnapshot, optionsOrNow: AgentDetailOptions | number = {}): string[] {
+  const options = typeof optionsOrNow === "number" ? { now: optionsOrNow } : optionsOrNow;
+  const now = options.now ?? Date.now();
+  const includeQueuedStatus = options.includeQueuedStatus ?? true;
+  const parts: string[] = [];
+  if (agent.toolUses > 0) parts.push(`${agent.toolUses} tool${agent.toolUses === 1 ? "" : "s"}`);
+  if (agent.lastTool) parts.push(agent.lastTool);
+  if (agent.startedAt !== undefined) parts.push(formatDuration((agent.doneAt ?? now) - agent.startedAt));
+  else if (includeQueuedStatus && agent.status === "queued") parts.push("queued");
+  if (agent.status === "failed" && agent.error) parts.push(agent.error);
+  return parts;
+}
+
+export function agentLabelColor(agent: AgentRowSnapshot): WorkflowThemeColor {
+  return agent.status === "running" ? "text" : "muted";
+}
+
+export interface WorkflowStatusCounts {
+  readonly queued: number;
+  readonly running: number;
+  readonly done: number;
+  readonly failed: number;
+  readonly total: number;
+}
+
+export interface WorkflowStatusSource {
+  readonly title: string;
+  readonly doneAt?: number;
+  readonly currentPhase: string;
+  readonly counters: readonly { readonly key: string; readonly label: string; readonly value: number }[];
+}
+
 export function statusText(snapshot: WorkflowProgressSnapshot, theme: Theme): string | undefined {
+  const counts = countSnapshotAgents(snapshot);
+  return statusTextFromCounts(snapshot, counts, theme);
+}
+
+export function statusTextFromCounts(snapshot: WorkflowStatusSource, counts: WorkflowStatusCounts, theme: Theme): string | undefined {
   if (snapshot.doneAt !== undefined) return undefined;
 
-  const agents = snapshot.phases.flatMap((phase) => phase.agents);
-  const complete = agents.filter((agent) => agent.status === "done" || agent.status === "failed").length;
-  const active = agents.filter((agent) => agent.status === "running" || agent.status === "queued").length;
-  const total = agents.length;
+  const complete = counts.done + counts.failed;
+  const active = counts.running + counts.queued;
   const kept = snapshot.counters.find((counter) => counter.key === "kept" || counter.label.toLowerCase() === "kept");
   const displayName = snapshot.title === "code-review" ? "review" : snapshot.title;
 
   const parts = [theme.fg("accent", displayName), theme.fg("muted", snapshot.currentPhase)];
-  if (total > 0) parts.push(theme.fg("muted", `${complete}/${total}`));
+  if (counts.total > 0) parts.push(theme.fg("muted", `${complete}/${counts.total}`));
   if (kept) parts.push(theme.fg("success", `${formatCount(kept.value)} kept`));
   else if (active > 0) parts.push(theme.fg("muted", `${active} active`));
 
   return parts.join(theme.fg("dim", " · "));
 }
 
+function countSnapshotAgents(snapshot: WorkflowProgressSnapshot): WorkflowStatusCounts {
+  const counts = { queued: 0, running: 0, done: 0, failed: 0, total: 0 };
+  for (const phase of snapshot.phases) {
+    for (const agent of phase.agents) {
+      counts[agent.status]++;
+      counts.total++;
+    }
+  }
+  return counts;
+}
+
 function formatCompact(value: number): string {
   if (value >= 100) return `${Math.round(value)}`;
   const rounded = Math.round(value * 10) / 10;

package/{src → .pi/extensions/pi-workflow-engine/src}/ui/workflow-inspector.ts

@@ -1,7 +1,7 @@
 import type { Theme } from "@earendil-works/pi-coding-agent";
 import { matchesKey, type TUI, visibleWidth } from "@earendil-works/pi-tui";
 import type { AgentRowSnapshot, WorkflowLaneItemSnapshot, WorkflowProgressSnapshot } from "../progress.ts";
-import { formatCount, formatDuration, statusIcon, truncateDisplay } from "./workflow-format.ts";
+import { agentDetailParts, formatCount, formatDuration, statusIcon, truncateDisplay } from "./workflow-format.ts";
 
 type Section = "Overview" | "Agents" | "Findings" | "Logs";
 
@@ -195,10 +195,9 @@ export class WorkflowInspector {
   }
 
   private agentLine(agent: AgentRowSnapshot, selected: boolean): string {
-    const elapsed = agent.startedAt === undefined ? "queued" : formatDuration((agent.doneAt ?? Date.now()) - agent.startedAt);
-    const details = [`${agent.toolUses} tools`, elapsed];
-    if (agent.lastTool) details.unshift(agent.lastTool);
-    const text = `${statusIcon(agent.status, this.theme)} ${agent.label} ${this.theme.fg("dim", `· ${details.join(" · ")}`)}`;
+    const details = agentDetailParts(agent);
+    const suffix = details.length > 0 ? ` ${this.theme.fg("dim", `· ${details.join(" · ")}`)}` : "";
+    const text = `${statusIcon(agent.status, this.theme)} ${agent.label}${suffix}`;
     return selected ? this.theme.bg("selectedBg", text) : text;
   }
 
@@ -212,8 +211,11 @@ export class WorkflowInspector {
   private findingLines(snapshot: WorkflowProgressSnapshot, _width: number): InspectorLine[] {
     const lines: InspectorLine[] = [];
     let selectableIndex = -1;
+    const overflowByLane = new Map(snapshot.laneOverflow);
     for (const [lane, items] of snapshot.lanes) {
-      lines.push({ text: this.theme.fg("dim", `${lane} (${items.length})`) });
+      const hidden = overflowByLane.get(lane) ?? 0;
+      const countText = hidden > 0 ? `${items.length} shown, ${hidden} hidden` : `${items.length}`;
+      lines.push({ text: this.theme.fg("dim", `${lane} (${countText})`) });
       items.forEach((item, index) => {
         selectableIndex++;
         const key = `finding:${lane}:${index}:${item.createdAt}`;