pi-crew 0.5.9 → 0.5.10

package/CHANGELOG.md

@@ -1,5 +1,38 @@
 # Changelog
 
+## [0.5.10] — Round 15 Audit Fixes (2026-06-02)
+
+### Phase 1: Semaphore Queue Cap (HIGH)
+- **H1**: `src/runtime/semaphore.ts:11` - `#queue` unbounded growth → added `MAX_QUEUE = 10_000` cap. `acquire()` now throws "Semaphore queue full" when at cap.
+
+### Phase 2: Observability Hardening (MEDIUM)
+- **L1**: `src/observability/event-bus.ts:47` - `console.error` → `logInternalError` for consistency
+- **OTLPExporter**: 
+  - Added `MAX_SNAPSHOTS_PER_PUSH = 5_000` cap to prevent OOM/oversized payloads
+  - Added `inFlight` promise tracking in `start()` to prevent overlapping setInterval pushes
+- **live-agent-manager**: Added `MAX_LIVE_AGENTS = 5_000` cap. `registerLiveAgent()` now evicts oldest completed agent first; if none, evicts oldest running with warning.
+
+### Phase 3: Test Coverage (LOW)
+- Added first-ever test coverage for `src/observability/`:
+  - 8 new tests in `test/unit/observability.test.ts` covering metric-registry, correlation, OTLP conversion
+- Reveals new finding: `crew.<domain>.<measure>` naming pattern enforcement is good (already validated)
+
+### Regression: Team-Runner Heartbeat (CRITICAL)
+- **CRITICAL regression** discovered via background watcher notification
+- `team-runner.ts` had NO periodic heartbeat, so any team run >5 min was being marked stale by the reconciler
+- Root cause of Round 15 review cancellation
+- Added `startTeamRunHeartbeat()` helper - writes `heartbeat.json` to stateRoot every 30s
+- Wired into `executeTeamRun()` with start/stop on both success and error paths
+- Same JSON shape as background-runner for reconciler compatibility
+
+### Tests
+- 2311 tests pass / 0 failures (was 2297 in v0.5.9)
+- +14 new tests across 3 new test files:
+  - `test/unit/team-runner-heartbeat.test.ts` (2 tests)
+  - `test/unit/round15-observability.test.ts` (4 tests)
+  - `test/unit/observability.test.ts` (8 tests)
+- TypeScript: 0 errors
+
 ## [0.5.9] — Round 14 Audit Fixes (2026-06-02)
 
 ### Phase 1: Sandbox Security (3 CRITICAL fixes)

package/README.md

@@ -9,7 +9,7 @@ npm: pi-crew
 repo: https://github.com/baphuongna/pi-crew
 ```
 
-**v0.5.9**: See [CHANGELOG.md](CHANGELOG.md).
+**v0.5.10**: See [CHANGELOG.md](CHANGELOG.md).
 
 ### Security highlights (v0.5.5)
 

package/docs/pi-crew-v0.5.10-audit-fix-plan.md

@@ -0,0 +1,60 @@
+# pi-crew v0.5.10 — Round 15 Audit Fix Plan (2026-06-02)
+
+**Source**: Round 15 dogfooding review (partial — explorer completed, reviewer/security-reviewer cancelled due to stale run reconciliation).
+
+**Findings verified from source**: 9 → 5 confirmed real, 4 false positives.
+
+## Verification Summary
+
+| Status | Count |
+|--------|-------|
+| ✅ CONFIRMED (real issue) | 5 |
+| ❌ FALSE POSITIVE | 4 |
+
+### False Positives Identified
+- **M2** (`register.ts` autoRepairTimer race): Code already guards with `cleanedUp || !currentCtx` checks
+- **M3** (`dynamic-script-runner.ts` walkNode type guard): Only runs on parsed acorn AST (parser guarantees `type: string`)
+- **H3** (event-log asyncQueues eviction): Already addressed in Round 14 — entries are deleted on success/error
+- **H2** (benchmark validateCommand footgun): Reviewer misread the validation flow
+
+### Real Issues Confirmed (5)
+
+1. **H1**: `Semaphore.#queue` unbounded growth (`src/runtime/semaphore.ts:11`)
+2. **L1**: `EventBus.emit` uses `console.error` instead of `logInternalError` (`src/observability/event-bus.ts:47`)
+3. **NEW**: `OTLPExporter.convertToOTLP` no size cap on snapshots (`src/observability/exporters/otlp-exporter.ts:33`)
+4. **NEW**: `OTLPExporter` `setInterval` can overlap if `push` is slow (no in-flight check)
+5. **NEW**: `hooks/registry.ts` Map unbounded; `Object.assign(ctx, result.data)` without validation
+
+## Plan: 3 small fixes
+
+### Phase 1: Semaphore Queue Cap (HIGH)
+- **H1**: Add `MAX_QUEUE = 10_000` cap to `Semaphore.#queue`. Reject with error when full.
+
+**Files**: `src/runtime/semaphore.ts`
+
+### Phase 2: Observability Hardening (MEDIUM)
+- **L1**: Replace `console.error` with `logInternalError` in `EventBus.emit`
+- **OTLP size**: Add snapshots.length cap + in-flight check in `OTLPExporter`
+- **Hook registry**: Add `clearHooks` after run, validate `result.data` keys
+
+**Files**: `src/observability/event-bus.ts`, `src/observability/exporters/otlp-exporter.ts`, `src/hooks/registry.ts`
+
+### Phase 3: Test Coverage (LOW)
+- Add basic tests for `observability/` (metric-registry, metric-sink, OTLP converter)
+- Add tests for `Semaphore` queue cap
+
+**Files**: new test files in `test/unit/`
+
+## Expected Outcomes
+
+- 5/5 confirmed issues fixed
+- Tests: 2300+ pass (5+ new tests)
+- TypeScript: 0 errors
+- v0.5.10 release
+
+## Backlog (deferred)
+
+- `console.log/error` in `background-runner.ts` — debug logging, intentional
+- `console.warn` in `discover-agents.ts` — informational
+- Full OTLP wire format compliance — out of scope
+- Hook `Object.assign` — needs design discussion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-crew",
-  "version": "0.5.9",
+  "version": "0.5.10",
   "description": "Pi extension for coordinated AI teams, workflows, worktrees, and async task orchestration",
   "author": "baphuongna",
   "license": "MIT",

package/src/observability/event-bus.ts

@@ -1,3 +1,4 @@
+import { logInternalError } from "../utils/internal-error.ts";
 import type { AgentProgress } from "../runtime/progress-tracker.ts";
 
 export type CrewEventType =
@@ -44,7 +45,11 @@ class EventBus {
         try {
           listener(event);
         } catch (e) {
-          console.error("[EventBus] Listener error:", e);
+          // FIX (Round 15, L1): Use logInternalError for consistency with
+          // the rest of the codebase. Previously console.error may not be
+          // visible in all environments (e.g. JSON-RPC mode, redirected
+          // stderr).
+          logInternalError("event-bus.listener", e, `type=${event.type} runId=${event.runId}`);
         }
       }
     }

package/src/observability/exporters/otlp-exporter.ts

@@ -8,6 +8,13 @@ import type { MetricExporter } from "./adapter.ts";
 
 const gzipAsync = promisify(gzip);
 
+// FIX (Round 15): Cap the number of snapshots per push to prevent OOM when
+// the metric registry has grown large. The OTLP HTTP spec allows many metrics
+// in one payload, but a single push > 10_000 metrics would balloon the
+// request body (gzipped or not) and likely exceed the collector's request
+// size limit.
+const MAX_SNAPSHOTS_PER_PUSH = 5_000;
+
 export interface OTLPExporterOptions {
 	endpoint: string;
 	headers?: Record<string, string>;
@@ -57,6 +64,9 @@ export function convertToOTLP(snapshots: MetricSnapshot[]): unknown {
 export class OTLPExporter implements MetricExporter {
 	name = "otlp";
 	private timer?: ReturnType<typeof setInterval>;
+	// FIX (Round 15): Track in-flight pushes so a slow network cannot cause
+	// the setInterval to overlap and pile up concurrent requests.
+	private inFlight: Promise<void> | null = null;
 	private readonly opts: OTLPExporterOptions;
 	private readonly registry: MetricRegistry;
 
@@ -67,12 +77,27 @@ export class OTLPExporter implements MetricExporter {
 
 	start(): void {
 		this.dispose();
-		this.timer = setInterval(() => { void this.push(this.registry.snapshot()); }, this.opts.intervalMs ?? 60_000);
+		this.timer = setInterval(() => {
+			// Skip if a previous push is still running; the next tick will retry.
+			if (this.inFlight) return;
+			const snap = this.registry.snapshot();
+			this.inFlight = this.push(snap).finally(() => { this.inFlight = null; });
+		}, this.opts.intervalMs ?? 60_000);
 		this.timer.unref();
 	}
 
 	async push(snapshots: MetricSnapshot[]): Promise<void> {
 		try {
+			// FIX (Round 15): Cap snapshots to a safe size to avoid OOM and
+			// oversized HTTP payloads. Log a warning if we are truncating.
+			let toSend = snapshots;
+			if (snapshots.length > MAX_SNAPSHOTS_PER_PUSH) {
+				logInternalError(
+					"otlp-export-cap",
+					new Error(`Snapshot count ${snapshots.length} exceeds cap ${MAX_SNAPSHOTS_PER_PUSH}; truncating`),
+				);
+				toSend = snapshots.slice(0, MAX_SNAPSHOTS_PER_PUSH);
+			}
 			const timeoutMs = this.opts.timeoutMs ?? 10_000;
 			const controller = new AbortController();
 			const timer = setTimeout(() => controller.abort(), timeoutMs);
@@ -80,7 +105,7 @@ export class OTLPExporter implements MetricExporter {
 				// 4.2: gzip body. OTLP HTTP exporters of every flavour accept
 				// `content-encoding: gzip`; collectors expect uncompressed JSON
 				// otherwise. Saves bandwidth on metric-heavy runs (often 3-5x).
-				const json = JSON.stringify(convertToOTLP(snapshots));
+				const json = JSON.stringify(convertToOTLP(toSend));
 				const body = await gzipAsync(Buffer.from(json));
 				const response = await fetch(this.opts.endpoint, {
 					method: "POST",

package/src/runtime/live-agent-manager.ts

@@ -63,6 +63,12 @@ export interface LiveAgentHandle {
 }
 
 const liveAgents = new Map<string, LiveAgentHandle>();
+// FIX (Round 15): Cap the number of tracked live agents to prevent unbounded
+// growth if a caller spawns agents but fails to unregister them. When the
+// cap is reached, the oldest completed agent is evicted first; if no
+// completed agents are present, the oldest running one is evicted (with a
+// warning) to keep memory bounded.
+const MAX_LIVE_AGENTS = 5_000;
 
 /**
  * List all live agents for a specific workspace.
@@ -100,6 +106,22 @@ export function registerLiveAgent(input: Omit<LiveAgentHandle, "createdAt" | "up
 			modelName: undefined,
 		},
 	};
+	// FIX (Round 15): Enforce the live-agent cap before adding. Prefer to
+	// evict the oldest completed agent (already finished, so caller no
+	// longer needs it). If none exist, evict the oldest running one with
+	// a warning so memory stays bounded.
+	if (liveAgents.size >= MAX_LIVE_AGENTS) {
+		const completed = [...liveAgents.entries()].find(([, h]) => h.activity.completedAtMs > 0);
+		if (completed) {
+			liveAgents.delete(completed[0]);
+		} else {
+			const oldestKey = liveAgents.keys().next().value;
+			if (oldestKey !== undefined) {
+				logInternalError("live-agent-manager.cap", new Error(`liveAgents at cap ${MAX_LIVE_AGENTS}; evicting oldest ${oldestKey}`));
+				liveAgents.delete(oldestKey);
+			}
+		}
+	}
 	liveAgents.set(input.agentId, handle);
 	try { if (eventLogFn && eventsPath) eventLogFn(eventsPath, { type: "live_agent.registered", runId: input.runId, taskId: input.taskId, message: `Live agent registered: ${input.agent} (${input.role})`, data: { agentId: input.agentId, role: input.role, agent: input.agent, workspaceId: input.workspaceId } }); } catch { /* non-critical */ }
 	if (handle.pendingSteers.length && typeof handle.session.steer === "function") {

package/src/runtime/semaphore.ts

@@ -16,6 +16,9 @@ export class Semaphore {
 	#max: number;
 	#current = 0;
 	#queue: Array<() => void> = [];
+	// FIX (Round 15): Cap the waiter queue to prevent unbounded memory growth
+	// if the semaphore is held for a long period and many tasks accumulate.
+	static readonly MAX_QUEUE = 10_000;
 
 	constructor(max: number) {
 		this.#max = Math.max(1, max);
@@ -26,6 +29,14 @@ export class Semaphore {
 			this.#current++;
 			return;
 		}
+		// FIX (Round 15): Reject when the waiter queue is full. The previous
+		// implementation let #queue grow without bound, risking memory
+		// exhaustion under sustained high concurrency with slow releases.
+		if (this.#queue.length >= Semaphore.MAX_QUEUE) {
+			throw new Error(
+				`Semaphore queue full: ${this.#queue.length} waiters (max ${Semaphore.MAX_QUEUE}); cannot acquire slot`,
+			);
+		}
 		const { promise, resolve } = (() => {
 			let res: () => void;
 			const p = new Promise<void>((r) => { res = r; });

package/src/runtime/team-runner.ts

@@ -1,4 +1,5 @@
 import * as fs from "node:fs";
+import * as path from "node:path";
 import type { AgentConfig } from "../agents/agent-config.ts";
 import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
 import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
@@ -38,6 +39,36 @@ import { CrewCancellationError, buildSyntheticTerminalEvidence, cancellationReas
 import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
 import { logInternalError } from "../utils/internal-error.ts";
 
+/**
+ * Start a periodic heartbeat for the team-level run.
+ *
+ * The stale reconciler (src/runtime/stale-reconciler.ts) marks runs as failed
+ * if their heartbeat is older than `NO_PID_HEARTBEAT_STALE_MS` (5 minutes).
+ * Without this, long-running team runs (e.g. multi-phase workflows) get
+ * cancelled by the reconciler as "stale" even when they are actively
+ * executing. The team-runner has no periodic heartbeat today, so any
+ * team run lasting >5min is at risk.
+ */
+function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
+	const heartbeatPath = path.join(stateRoot, "heartbeat.json");
+	const writeHeartbeat = (): void => {
+		try {
+			fs.writeFileSync(heartbeatPath, JSON.stringify({
+				pid: process.pid,
+				at: Date.now(),
+				runId,
+				kind: "team-runner",
+			}), "utf-8");
+		} catch {
+			// best-effort
+		}
+	};
+	writeHeartbeat();
+	const interval = setInterval(writeHeartbeat, 30_000);
+	interval.unref();
+	return () => clearInterval(interval);
+}
+
 export interface ExecuteTeamRunInput {
 	manifest: TeamRunManifest;
 	tasks: TeamTaskState[];
@@ -271,12 +302,20 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
 
 	void registerRunPromise(manifest.runId);
 
+	// FIX (Round 15, regression): Start a team-level heartbeat so the stale
+	// reconciler does not cancel long-running team runs after 5 minutes
+	// (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
+	// heartbeats; the team-level run had no heartbeat, so any multi-phase
+	// workflow lasting >5min was marked stale and cancelled.
+	const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId);
+
 	const cleanupUsage = (): void => {
 		for (const task of input.tasks) clearTrackedTaskUsage(task.id);
 	};
 
 	try {
 		const result = await executeTeamRunCore(input, manifest, workflow);
+		stopTeamHeartbeat();
 		resolveRunPromise(manifest.runId, result);
 		cleanupUsage();
 		// Terminate live agents for this run — agents are done when the run ends.
@@ -318,6 +357,7 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
 		rejectRunPromise(manifest.runId, error instanceof Error ? error : new Error(message));
 		crewHooks.emit({ type: "run_failed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: manifest.status, error: message } });
 		cleanupUsage();
+		stopTeamHeartbeat();
 		return result;
 	}
 }