pi-crew 0.9.21 → 0.9.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-crew",
-  "version": "0.9.21",
+  "version": "0.9.23",
   "description": "Pi extension for coordinated AI teams, workflows, worktrees, and async task orchestration",
   "author": "baphuongna",
   "license": "MIT",

package/src/extension/cross-extension-rpc.ts

@@ -137,7 +137,7 @@ function isAllowedRpcRunParams(params: TeamToolParamsValue): {
 	return { ok: true };
 }
 
-function on(events: EventBusLike, channel: string, handler: (raw: unknown) => void): () => void {
+function on(events: EventBusLike, channel: string, handler: (raw: any) => unknown): () => void {
 	const unsub = events.on(channel, handler);
 	return typeof unsub === "function" ? unsub : () => {};
 }

package/src/runtime/pipeline-runner.ts

@@ -1,6 +1,6 @@
 import type { AgentConfig } from "../agents/agent-config.ts";
 import { errors } from "../errors.ts";
-import { appendEventAsync } from "../state/event-log.ts";
+import { appendEventAsync, flushEventLogBuffer } from "../state/event-log.ts";
 import type { TeamTaskState } from "../state/types.ts";
 import type { TeamConfig } from "../teams/team-config.ts";
 import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
@@ -107,142 +107,151 @@ export class PipelineRunner {
 		let previousResults: unknown[] = [];
 		const startTime = Date.now();
 
-		await appendEventAsync(eventsPath, {
-			type: "pipeline:started",
-			runId,
-			message: `Pipeline '${workflow.name}' started`,
-			data: { stages: workflow.stages.map((s) => s.name) },
-		});
-
-		for (let i = 0; i < workflow.stages.length; i++) {
-			const stage = workflow.stages[i];
-			const stageStartTime = Date.now();
-
-			// Determine stop behavior for this stage
-			const effectiveStopOnError = stage.stopOnError ?? workflow.stopOnError ?? this.stopOnError;
-
+		try {
 			await appendEventAsync(eventsPath, {
-				type: "pipeline:stage_started",
+				type: "pipeline:started",
 				runId,
-				message: `Stage '${stage.name}' started`,
-				data: { stageIndex: i, stageName: stage.name },
+				message: `Pipeline '${workflow.name}' started`,
+				data: { stages: workflow.stages.map((s) => s.name) },
 			});
 
-			try {
-				// Build stage context
-				const stageContext: PipelineContext = {
-					stageIndex: i,
-					stageName: stage.name,
-					previousResults,
-					totalStages: workflow.stages.length,
-					runId,
-				};
-
-				// Resolve inputs
-				const inputs = this.resolveInputs(stage.inputs, previousResults, context);
-
-				// Execute stage (handle fan-out if enabled)
-				const results = await this.executeStageInternal(stage, inputs, stageContext, executeStage);
+			for (let i = 0; i < workflow.stages.length; i++) {
+				const stage = workflow.stages[i];
+				const stageStartTime = Date.now();
 
-				const duration = Date.now() - stageStartTime;
-				stages.push({
-					name: stage.name,
-					status: "completed",
-					results,
-					duration,
-					fanOutItems: Array.isArray(inputs) ? inputs.length : undefined,
-				});
-
-				previousResults = results;
+				// Determine stop behavior for this stage
+				const effectiveStopOnError = stage.stopOnError ?? workflow.stopOnError ?? this.stopOnError;
 
 				await appendEventAsync(eventsPath, {
-					type: "pipeline:stage_completed",
+					type: "pipeline:stage_started",
 					runId,
-					message: `Stage '${stage.name}' completed`,
-					data: {
-						stageIndex: i,
-						stageName: stage.name,
-						duration,
-						resultCount: results.length,
-					},
+					message: `Stage '${stage.name}' started`,
+					data: { stageIndex: i, stageName: stage.name },
 				});
-			} catch (error) {
-				const duration = Date.now() - stageStartTime;
-				const errorMessage = error instanceof Error ? error.message : String(error);
 
-				if (effectiveStopOnError) {
-					stages.push({
-						name: stage.name,
-						status: "failed",
-						results: [],
-						error: errorMessage,
-						duration,
-					});
-
-					await appendEventAsync(eventsPath, {
-						type: "pipeline:stage_failed",
+				try {
+					// Build stage context
+					const stageContext: PipelineContext = {
+						stageIndex: i,
+						stageName: stage.name,
+						previousResults,
+						totalStages: workflow.stages.length,
 						runId,
-						message: `Stage '${stage.name}' failed: ${errorMessage}`,
-						data: {
-							stageIndex: i,
-							stageName: stage.name,
-							duration,
-							error: errorMessage,
-						},
-					});
+					};
 
-					await appendEventAsync(eventsPath, {
-						type: "pipeline:failed",
-						runId,
-						message: `Pipeline '${workflow.name}' failed at stage '${stage.name}'`,
-						data: { failedStage: stage.name, error: errorMessage },
-					});
+					// Resolve inputs
+					const inputs = this.resolveInputs(stage.inputs, previousResults, context);
 
-					return {
-						stages,
-						totalDuration: Date.now() - startTime,
-						finalResults: previousResults,
-						status: "failed",
-					};
-				} else {
+					// Execute stage (handle fan-out if enabled)
+					const results = await this.executeStageInternal(stage, inputs, stageContext, executeStage);
+
+					const duration = Date.now() - stageStartTime;
 					stages.push({
 						name: stage.name,
-						status: "failed",
-						results: [],
-						error: errorMessage,
+						status: "completed",
+						results,
 						duration,
+						fanOutItems: Array.isArray(inputs) ? inputs.length : undefined,
 					});
 
+					previousResults = results;
+
 					await appendEventAsync(eventsPath, {
-						type: "pipeline:stage_skipped",
+						type: "pipeline:stage_completed",
 						runId,
-						message: `Stage '${stage.name}' skipped due to error`,
+						message: `Stage '${stage.name}' completed`,
 						data: {
 							stageIndex: i,
 							stageName: stage.name,
 							duration,
-							error: errorMessage,
+							resultCount: results.length,
 						},
 					});
+				} catch (error) {
+					const duration = Date.now() - stageStartTime;
+					const errorMessage = error instanceof Error ? error.message : String(error);
+
+					if (effectiveStopOnError) {
+						stages.push({
+							name: stage.name,
+							status: "failed",
+							results: [],
+							error: errorMessage,
+							duration,
+						});
+
+						await appendEventAsync(eventsPath, {
+							type: "pipeline:stage_failed",
+							runId,
+							message: `Stage '${stage.name}' failed: ${errorMessage}`,
+							data: {
+								stageIndex: i,
+								stageName: stage.name,
+								duration,
+								error: errorMessage,
+							},
+						});
+
+						await appendEventAsync(eventsPath, {
+							type: "pipeline:failed",
+							runId,
+							message: `Pipeline '${workflow.name}' failed at stage '${stage.name}'`,
+							data: { failedStage: stage.name, error: errorMessage },
+						});
+
+						return {
+							stages,
+							totalDuration: Date.now() - startTime,
+							finalResults: previousResults,
+							status: "failed",
+						};
+					} else {
+						stages.push({
+							name: stage.name,
+							status: "failed",
+							results: [],
+							error: errorMessage,
+							duration,
+						});
+
+						await appendEventAsync(eventsPath, {
+							type: "pipeline:stage_skipped",
+							runId,
+							message: `Stage '${stage.name}' skipped due to error`,
+							data: {
+								stageIndex: i,
+								stageName: stage.name,
+								duration,
+								error: errorMessage,
+							},
+						});
+					}
 				}
 			}
-		}
 
-		await appendEventAsync(eventsPath, {
-			type: "pipeline:completed",
-			runId,
-			message: `Pipeline '${workflow.name}' completed`,
-			data: {
-				stages: stages.map((s) => ({ name: s.name, status: s.status })),
-			},
-		});
+			await appendEventAsync(eventsPath, {
+				type: "pipeline:completed",
+				runId,
+				message: `Pipeline '${workflow.name}' completed`,
+				data: {
+					stages: stages.map((s) => ({ name: s.name, status: s.status })),
+				},
+			});
 
-		return {
-			stages,
-			totalDuration: Date.now() - startTime,
-			finalResults: previousResults,
-			status: stages.some((s) => s.status === "failed") ? "partial" : "completed",
-		};
+			return {
+				stages,
+				totalDuration: Date.now() - startTime,
+				finalResults: previousResults,
+				status: stages.some((s) => s.status === "failed") ? "partial" : "completed",
+			};
+		} finally {
+			// FIX (P0 follow-up): Drain the event-log buffer before returning so
+			// --test-force-exit does not detect "Promise resolution is still
+			// pending". The buffered flush uses a 20ms ref'd timer that fires
+			// asynchronously; without this explicit drain, the test runner sees
+			// pending appendEventAsync promises and cancels the test file.
+			await flushEventLogBuffer(eventsPath);
+		}
 	}
 
 	/**

package/src/runtime/tool-output-pruner.ts

@@ -90,7 +90,7 @@ export interface PruneResult {
 // Token estimation (rough char/4 heuristic, matching gajae-code)
 // ---------------------------------------------------------------------------
 
-import { countTokens } from "../utils/token-counter";
+import { countTokens } from "../utils/token-counter.ts";
 
 function estimateTokens(text: string): number {
 	return countTokens(text);

package/src/state/event-log.ts

@@ -412,15 +412,11 @@ export function resetEventLogMode(): void {
 export async function appendEventAsync(eventsPath: string, event: AppendTeamEvent): Promise<TeamEvent> {
 	// Non-terminal events: route through buffer for coalesced writes (20ms default).
 	// This eliminates per-event fsync + persistSequence overhead for high-frequency events.
+	// The buffer timer is kept alive (ref'd) so standalone contexts (tests, short-lived
+	// scripts) get their events flushed before the loop drains — no per-event keepAlive
+	// intervals needed (which previously starved the event loop at high fan-out).
 	if (!TERMINAL_EVENT_TYPES.has(event.type)) {
-		const result = appendEventBuffered(eventsPath, event);
-		// FIX: appendEventBuffered uses unref()'d timers. Keep the event loop alive
-		// so the promise resolves in standalone/test contexts where nothing else
-		// keeps the loop running. Without this, the event loop drains before the
-		// timer fires and the promise never resolves.
-		const keepAlive = setInterval(() => {}, 200);
-		void result.finally(() => clearInterval(keepAlive));
-		return result;
+		return appendEventBuffered(eventsPath, event);
 	}
 	// Terminal events: direct async path for immediate durability guarantee
 	const queueKey = eventsPath;
@@ -618,6 +614,131 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
  * `withEventLogLockSync` for `eventsPath`. Used by `appendEventBuffered` to
  * write a whole batch of pending events under a single lock acquire.
  */
+/**
+ * Batch variant used by the buffered flush path. Computes metadata for each
+ * event, writes the whole batch in a single appendFileSync + fsync, persists
+ * the sequence sidecar once with the last seq, and updates the sequence cache
+ * once. Resolves each item with its finalized event (carrying the assigned
+ * seq). This collapses N fsyncs into 1 for the buffered write path, which is
+ * the entire point of buffering — the previous per-event fsync made buffer
+ * coalescing useless and added ~30ms/event on tmpfs.
+ */
+async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedAppend[]): Promise<void> {
+	if (queue.length === 0) return;
+	fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
+
+	// Pre-flight size check (mirrors appendEventInsideLock). We do it once for
+	// the batch instead of once per event.
+	try {
+		if (fs.existsSync(eventsPath)) {
+			const stat = fs.statSync(eventsPath);
+			if (stat.size > MAX_EVENTS_BYTES) {
+				try {
+					const prepared = prepareCompaction(eventsPath);
+					if (prepared) applyCompactionUnlocked(eventsPath, prepared);
+				} catch (error) {
+					logInternalError("event-log.batch-immediate-compact", error, `eventsPath=${eventsPath}`);
+				}
+				if (fs.existsSync(eventsPath) && fs.statSync(eventsPath).size > MAX_EVENTS_BYTES) {
+					rotateEventLogUnlocked(eventsPath);
+				}
+			}
+		}
+	} catch (error) {
+		logInternalError("event-log.batch-size-check", error, `eventsPath=${eventsPath}`);
+	}
+
+	// Phase 1: compute metadata + JSON lines for every event in the batch.
+	// Initialize nextSeq ONCE from nextSequence (or the first event's baseMetadata.seq),
+	// then increment locally for each subsequent event in the batch. Calling
+	// nextSequence() per-event would re-read file stat/sidecar with no writes
+	// in between — every call would see the same file state and return the same
+	// seq, breaking the "unique monotonic seq" contract. The cache update +
+	// persistSequence at the end refreshes the sidecar to the last assigned seq.
+	const startingSeq = queue[0]?.event.metadata?.seq ?? nextSequence(eventsPath);
+	let nextSeq = startingSeq;
+	const finalized: { item: BufferedAppend; line: string; fullEvent: TeamEvent }[] = [];
+	let lastSeq = 0;
+	for (const item of queue) {
+		const baseMetadata = item.event.metadata;
+		const seq = baseMetadata?.seq ?? nextSeq++;
+		let metadata: TeamEventMetadata = {
+			seq,
+			provenance: baseMetadata?.provenance ?? "team_runner",
+			...(baseMetadata?.parentEventId ? { parentEventId: baseMetadata.parentEventId } : {}),
+			...(baseMetadata?.attemptId ? { attemptId: baseMetadata.attemptId } : {}),
+			...(baseMetadata?.branchId ? { branchId: baseMetadata.branchId } : {}),
+			...(baseMetadata?.causationId ? { causationId: baseMetadata.causationId } : {}),
+			...(baseMetadata?.correlationId ? { correlationId: baseMetadata.correlationId } : {}),
+			...(baseMetadata?.sessionIdentity ? { sessionIdentity: baseMetadata.sessionIdentity } : {}),
+			...(baseMetadata?.ownership ? { ownership: baseMetadata.ownership } : {}),
+			...(baseMetadata?.nudgeId ? { nudgeId: baseMetadata.nudgeId } : {}),
+			...(baseMetadata?.confidence ? { confidence: baseMetadata.confidence } : {}),
+		};
+		const fullEvent: TeamEvent = {
+			time: new Date().toISOString(),
+			...item.event,
+			metadata,
+		};
+		if (baseMetadata?.fingerprint || TERMINAL_EVENT_TYPES.has(fullEvent.type)) {
+			metadata = {
+				...metadata,
+				fingerprint: baseMetadata?.fingerprint ?? computeEventFingerprint(fullEvent),
+			};
+			fullEvent.metadata = metadata;
+		}
+		finalized.push({ item, line: `${JSON.stringify(redactSecrets(fullEvent))}\n`, fullEvent });
+		lastSeq = seq;
+	}
+
+	// Phase 2: single appendFileSync + single fsync + single persistSequence.
+	// Before this fix, each event in the batch triggered its own fsync, which
+	// was the dominant cost on tmpfs and CI runners.
+	try {
+		if (fs.existsSync(eventsPath) && fs.statSync(eventsPath).size > MAX_EVENTS_BYTES) {
+			logInternalError(
+				"event-log.size-limit",
+				new Error(`events file ${eventsPath} exceeds ${MAX_EVENTS_BYTES} bytes after compaction`),
+				`eventsPath=${eventsPath}`,
+			);
+			// Reject the batch — caller will surface the error per item.
+			for (const { item } of finalized) item.reject(new Error("event log size limit exceeded"));
+			return;
+		}
+	} catch (error) {
+		logInternalError("event-log.batch-size-check-post", error, `eventsPath=${eventsPath}`);
+	}
+
+	fs.appendFileSync(eventsPath, finalized.map((f) => f.line).join(""), "utf-8");
+	const fd = fs.openSync(eventsPath, "r+");
+	try {
+		fs.fsyncSync(fd);
+	} catch {
+		// EPERM on Windows CI: best-effort flush
+	} finally {
+		fs.closeSync(fd);
+	}
+	persistSequence(eventsPath, lastSeq);
+
+	// Phase 3: cache update + resolve all promises.
+	try {
+		const stat = fs.statSync(eventsPath);
+		if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
+			evictOldestSequenceCacheEntries();
+		}
+		sequenceCache.set(eventsPath, {
+			size: stat.size,
+			mtimeMs: stat.mtimeMs,
+			seq: lastSeq,
+			lastAccessMs: Date.now(),
+		});
+	} catch (error) {
+		logInternalError("event-log.batch-cache-update", error, `eventsPath=${eventsPath}`);
+	}
+
+	for (const { item, fullEvent } of finalized) item.resolve(fullEvent);
+}
+
 function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): TeamEvent {
 	fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
 	const baseMetadata = event.metadata;
@@ -784,8 +905,15 @@ export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent,
 		queue.push({ event, resolve, reject });
 		bufferedQueues.set(eventsPath, queue);
 		if (!bufferedTimers.has(eventsPath)) {
-			const timer = setTimeout(() => flushOneEventLogBuffer(eventsPath), bufferMs);
-			timer.unref();
+			// Wrap flush in async IIFE so the returned Promise is awaited (avoids
+			// "floating promise" warnings under --test-force-exit and prevents
+			// the timer from being treated as done before the flush actually
+			// completes its async work).
+			const timer = setTimeout(() => {
+				flushOneEventLogBuffer(eventsPath).catch((error) => {
+					logInternalError("event-log.buffered-flush", error, `eventsPath=${eventsPath}`);
+				});
+			}, bufferMs);
 			bufferedTimers.set(eventsPath, timer);
 		}
 	});
@@ -830,15 +958,12 @@ async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
 		// FIX (Issue 2): Use async lock instead of withEventLogLockSync to avoid
 		// blocking the event loop. The sync lock uses sleepSync which blocks for
 		// up to 5s and prevents AbortSignal handlers from firing.
+		// FIX (P0 follow-up): Batch the file write + fsync + persistSequence across
+		// the whole queue. Previously each event triggered its own fsyncSync,
+		// turning 100 buffered events into 100 fsyncs (~3s on tmpfs). Now we do
+		// 1 appendFileSync + 1 fsync + 1 persistSequence for the whole batch.
 		await withEventLogLockAsync(eventsPath, async () => {
-			for (const item of queue) {
-				try {
-					const ev = appendEventInsideLock(eventsPath, item.event);
-					item.resolve(ev);
-				} catch (error) {
-					item.reject(error);
-				}
-			}
+			await appendEventBatchInsideLock(eventsPath, queue);
 		});
 	} catch (error) {
 		// Lock acquire failed — fail every queued item so callers can fall back.
@@ -866,15 +991,52 @@ export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEv
 // Auto-flush on process exit so buffered events do not silently leak.
 // Defense-in-depth: SIGTERM/SIGINT use setImmediate so the handler returns
 // immediately and the main thread is not blocked by sync I/O.
+// FIX (P0 follow-up): Only call flushEventLogBuffer() / drainAsyncQueues() if
+// there is actually pending work. Calling them unconditionally creates a new
+// floating Promise that the test runner detects under --test-force-exit,
+// failing tests with "Promise resolution is still pending but the event loop
+// has already resolved" even when the test body completed cleanly.
 process.on("exit", () => {
-	flushEventLogBuffer();
+	if (bufferedQueues.size > 0) {
+		flushEventLogBuffer().catch(() => {
+			/* best-effort flush on exit */
+		});
+	}
 	// FIX (Issue 1): Drain asyncQueues on exit to minimize event loss.
 	// In-flight async writes are awaited (via Promise.allSettled) before
 	// the map is cleared. This reduces but does not eliminate event loss
 	// on crash — SIGKILL (kill -9) cannot be intercepted.
-	drainAsyncQueues();
+	if (asyncQueues.size > 0) {
+		drainAsyncQueues().catch(() => {
+			/* best-effort drain on exit */
+		});
+	}
 	asyncQueues.clear();
 });
+
+// FIX (P0 follow-up): Drain buffered events on `beforeExit` (async-aware).
+// The `exit` handler above is sync-only and cannot await the flush, leaving
+// pending promises that the test runner detects under --test-force-exit as
+// "Promise resolution is still pending but the event loop has already
+// resolved". `beforeExit` fires when the event loop drains naturally and
+// supports async handlers, so we can await the flush here and let the loop
+// drain cleanly before process.exit() is called.
+process.on("beforeExit", async () => {
+	if (bufferedQueues.size > 0) {
+		try {
+			await flushEventLogBuffer();
+		} catch {
+			/* best-effort */
+		}
+	}
+	if (asyncQueues.size > 0) {
+		try {
+			await drainAsyncQueues();
+		} catch {
+			/* best-effort */
+		}
+	}
+});
 process.on("SIGTERM", () => setImmediate(() => flushEventLogBuffer()));
 process.on("SIGINT", () => setImmediate(() => flushEventLogBuffer()));
 // FIX (Issue 1): Handle uncaught exceptions to flush buffered events before

package/src/worktree/worktree-manager.ts

@@ -111,7 +111,6 @@ async function gitAsync(cwd: string, args: string[]): Promise<string> {
 	const { stdout } = await execFileAsync("git", args, {
 		cwd,
 		encoding: "utf-8",
-		stdio: ["ignore", "pipe", "pipe"],
 		env: gitEnv(),
 		windowsHide: true,
 	});
@@ -445,7 +444,6 @@ async function branchExistsAsync(repoRoot: string, branch: string): Promise<{ lo
 			await execFileAsync("git", ["for-each-ref", "--format=%(refname)", `refs/remotes/*/${branch}`], {
 				cwd: repoRoot,
 				encoding: "utf-8",
-				stdio: ["ignore", "pipe", "pipe"],
 				windowsHide: true,
 			})
 		).stdout.trim();
@@ -459,7 +457,7 @@ async function pruneStaleWorktreesAsync(repoRoot: string): Promise<void> {
 	try {
 		await execFileAsync("git", ["worktree", "prune"], {
 			cwd: repoRoot,
-			stdio: ["ignore", "ignore", "ignore"],
+			windowsHide: true,
 		});
 	} catch {
 		/* best-effort */