pi-crew 0.10.1 → 0.10.3

package/src/state/event-log/event-log.ts

@@ -7,16 +7,13 @@ import { emitFromTeamEvent } from "../../ui/run-event-bus.ts";
 import { logInternalError } from "../../utils/internal-error.ts";
 import { redactSecrets } from "../../utils/redaction.ts";
 import { sleep, sleepSync } from "../../utils/sleep.ts";
-import { atomicWriteFile } from "../atomic-write.ts";
 import { applyCompactionUnlocked, needsRotation, prepareCompaction, rotateEventLogUnlocked } from "./event-log-rotation.ts";
 import {
 	advanceSequenceCounter,
-	evictOldestSequenceCacheEntries,
-	MAX_SEQUENCE_CACHE_ENTRIES,
 	persistSequenceMonotonic,
+	reservedSequenceEnd,
 	reserveSequence,
 	reserveSequenceUnderLockAsync,
-	sequenceCache,
 } from "./sequence-cache.ts";
 import { appendFileViaWorker, isWorkerAtomicWriterEnabled } from "./worker-atomic-writer.ts";
 
@@ -134,9 +131,18 @@ export function withEventLogLockSync<T>(eventsPath: string, fn: () => T, options
 			// (reduced from 120s) is appropriate.
 			fs.mkdirSync(lockDir);
 			try {
-				atomicWriteFile(pidFile, String(process.pid));
+				// PERF (2026-08-25): pid file is disposable + mtime-stale-detected, so the
+				// full atomicWriteFile (2 fsync) here was dead ceremony — the .alock async
+				// path already writes pid via "wx". Mirror it: O_EXCL open, plain write.
+				// The lock dir itself is the mutex.
+				const pidFd = fs.openSync(pidFile, "wx");
+				try {
+					fs.writeSync(pidFd, String(process.pid));
+				} finally {
+					fs.closeSync(pidFd);
+				}
 			} catch {
-				/* best-effort */
+				/* best-effort — e.g. EEXIST under a re-taken dir; stale pid is mtime-detected */
 			}
 			acquired = true;
 			break;
@@ -330,8 +336,18 @@ async function withEventLogLockAsync<T>(
 			try {
 				await fs.promises.mkdir(lockDir);
 				try {
-					// P0-4: the lock pid file is disposable stale-lock state; best-effort.
-					atomicWriteFile(pidFile, String(process.pid), { durability: "best-effort" });
+					// PERF (2026-08-24): "wx" (O_CREAT|O_EXCL) — fails rather than
+					// following a planted symlink, so no O_NOFOLLOW/temp/rename
+					// ceremony needed for this disposable, mtime-stale-detected
+					// 4-byte file. We own the lock dir (we just mkdir'd it), so
+					// EEXIST means a crashed holder's leftover under OUR fresh dir
+					// or an attack — either way, skip: the dir itself is the mutex.
+					const fh = await fs.promises.open(pidFile, "wx");
+					try {
+						await fh.write(String(process.pid));
+					} finally {
+						await fh.close();
+					}
 				} catch {
 					/* best-effort */
 				}
@@ -566,9 +582,6 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 			logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
 		}
 
-		// FIND-10: post-append stat captured from the same fd (non-worker path)
-		// for reuse in the cache update below, avoiding a redundant path stat.
-		let postAppendStat: fs.Stats | undefined;
 		if (!skippedDueToSize) {
 			const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
 			// Phase 1.5: when worker atomic writer is enabled, append via worker.
@@ -595,16 +608,6 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 					await fd.appendFile(line, "utf-8");
 					// P0-4 (F3a mirror): skip the data fsync for non-terminal events.
 					if (isTerminal) await fd.sync();
-					// FIND-10 R1 fix: the cache-optimization fd.stat() must NOT sit in the
-					// seq-durability critical path. If it threw (rare — fd invalidated),
-					// it would skip persistSequence below and reopen the seq-reuse
-					// window the fsync just closed. Guard it; fall back to undefined
-					// (the later cache-update takes a path stat instead).
-					try {
-						postAppendStat = await fd.stat();
-					} catch {
-						postAppendStat = undefined;
-					}
 				} finally {
 					await fd.close();
 				}
@@ -618,11 +621,7 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 			// can never roll the sidecar back either.
 			if (baseMetadata?.seq !== undefined) persistSequenceMonotonic(eventsPath, seq);
 		}
-		// FIND-10: track whether compaction happened after the append so the
-		// cache-update stat can safely reuse postAppendStat (file unchanged).
-		let compactedAfterAppend = false;
 		if (tickAppendCounter(eventsPath) && needsRotation(eventsPath)) {
-			compactedAfterAppend = true;
 			try {
 				const prepared = prepareCompaction(eventsPath);
 				if (prepared) applyCompactionUnlocked(eventsPath, prepared);
@@ -641,39 +640,13 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 			}
 		}
 
-		// FIX: Sequence was persisted AFTER appendFile in the append block above.
-		// Only update the cache here (the sidecar persist is already done).
-		const finalSeq = fullEvent.metadata?.seq ?? 0;
-		try {
-			// FIND-10: reuse post-append fd stat when available and no compaction
-			// happened after the append (file unchanged). Falls back to path stat
-			// for the worker path, skipped events, or post-compaction cases.
-			let statResult: fs.Stats | undefined;
-			if (postAppendStat && !compactedAfterAppend) {
-				statResult = postAppendStat;
-			} else {
-				try {
-					statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
-				} catch {
-					/* file may not exist */
-				}
-			}
-			if (statResult) {
-				if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
-					evictOldestSequenceCacheEntries();
-				}
-				sequenceCache.set(eventsPath, {
-					size: statResult.size,
-					mtimeMs: statResult.mtimeMs,
-					seq: finalSeq,
-					lastAccessMs: Date.now(),
-				});
-			}
-			// Note: persistSequence is NOT called here again - it was already called
-			// after the append to ensure the sidecar is current after the event is written.
-		} catch (error) {
-			logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
-		}
+		// PERF (2026-08-24): the per-append sequenceCache upkeep that lived here
+		// (post-append stat + Map set + occasional O(n log n) evict sort) fed no
+		// hot reader — sequenceCache is read only by nextSequence, which is
+		// consulted solely by seeding/test helpers (see the ST-12 note in
+		// sequence-cache.ts); all three append paths allocate seqs via
+		// reserveSequence, which reads the .seq sidecar + seqCounters instead.
+		// nextSequence() re-seeds via its sidecar/scan fallback when called.
 		return fullEvent;
 	};
 	// C-01: Two-tier lock — asyncQueues (in-process serialize) →
@@ -726,11 +699,13 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 /**
  * Batch variant used by the buffered flush path. Computes metadata for each
  * event, writes the whole batch in a single appendFileSync + fsync, persists
- * the sequence sidecar once with the last seq, and updates the sequence cache
- * once. Resolves each item with its finalized event (carrying the assigned
- * seq). This collapses N fsyncs into 1 for the buffered write path, which is
- * the entire point of buffering — the previous per-event fsync made buffer
- * coalescing useless and added ~30ms/event on tmpfs.
+ * the sequence sidecar once with the last seq (no per-append sequenceCache
+ * upkeep — that cache no longer exists on the append paths; see the PERF
+ * 2026-08-24 note in appendEventAsync). Resolves each item with its finalized
+ * event (carrying the assigned seq). This collapses N fsyncs into 1 for the
+ * buffered write path, which is the entire point of buffering — the previous
+ * per-event fsync made buffer coalescing useless and added ~30ms/event on
+ * tmpfs.
  */
 async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedAppend[]): Promise<void> {
 	if (queue.length === 0) return;
@@ -738,12 +713,23 @@ async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedApp
 	tickAppendCounter(eventsPath, queue.length);
 	fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
 
+	// PERF (2026-08-24): one hoisted pre-append stat replaces the pre-flight
+	// existsSync+statSync pair below. The after-compaction re-checks further
+	// down keep their own fresh stats — compaction/rotation may have changed
+	// the file since preStat was taken, so they must not see a stale
+	// (pre-compaction) size.
+	let preStat: fs.Stats | undefined;
+	try {
+		preStat = fs.statSync(eventsPath);
+	} catch {
+		/* log absent — first append */
+	}
+
 	// Pre-flight size check (mirrors appendEventInsideLock). We do it once for
 	// the batch instead of once per event.
 	try {
-		if (fs.existsSync(eventsPath)) {
-			const stat = fs.statSync(eventsPath);
-			if (stat.size > MAX_EVENTS_BYTES) {
+		if (preStat) {
+			if (preStat.size > MAX_EVENTS_BYTES) {
 				try {
 					const prepared = prepareCompaction(eventsPath);
 					if (prepared) applyCompactionUnlocked(eventsPath, prepared);
@@ -774,8 +760,8 @@ async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedApp
 	// then increment locally for each subsequent event in the batch. Calling
 	// nextSequence() per-event would re-read file stat/sidecar with no writes
 	// in between — every call would see the same file state and return the same
-	// seq, breaking the "unique monotonic seq" contract. The cache update +
-	// persistSequence at the end refreshes the sidecar to the last assigned seq.
+	// seq, breaking the "unique monotonic seq" contract. persistSequenceMonotonic
+	// at the end refreshes the sidecar to the last assigned seq.
 	// B7: use reserveSequence for atomic seq assignment across all paths.
 	// R16-B1 (Phase 3.6): reserve the WHOLE batch range (count = queue.length)
 	// under the .seqlock in one acquire — the locally incremented nextSeq below
@@ -816,6 +802,13 @@ async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedApp
 		finalized.push({ item, line: `${JSON.stringify(redactSecrets(fullEvent))}\n`, fullEvent });
 		lastSeq = seq;
 	}
+	// PERF (2026-08-24): snapshot the in-process reservation end BEFORE the B7
+	// advance below — seqCounters is about to be raised to lastSeq, so reading
+	// reservedSequenceEnd() at the persist site would ALWAYS see lastSeq ≤
+	// counter and skip the persist even for explicit seqs the reservation never
+	// covered (sidecar would lag the file → cross-process re-reservation inside
+	// the file's true range — the exact R16-B1 duplicate-seq race).
+	const reservedEnd = reservedSequenceEnd(eventsPath);
 	// B7: advance counter past the entire batch so next reserveSequence returns the correct value.
 	advanceSequenceCounter(eventsPath, lastSeq);
 
@@ -838,41 +831,56 @@ async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedApp
 	}
 
 	fs.appendFileSync(eventsPath, finalized.map((f) => f.line).join(""), "utf-8");
-	const fd = fs.openSync(eventsPath, "r+");
-	try {
-		fs.fsyncSync(fd);
-	} catch {
-		// EPERM on Windows CI: best-effort flush
-	} finally {
-		fs.closeSync(fd);
+	// PERF (2026-08-25): skip the fsync when the whole batch is non-terminal.
+	// Terminal events never route through this buffer (appendEventBuffered
+	// bypasses to appendEvent, which fsyncs itself), so a batch here has no
+	// terminal event unless a caller deliberately mixed one in — mirror F3a:
+	// the event-reconstructor tolerates an inconsistent tail; the explicit
+	// persistSequenceMonotonic below still lands the reserved end range.
+	const hasTerminal = finalized.some((f) => TERMINAL_EVENT_TYPES.has(f.fullEvent.type));
+	if (hasTerminal) {
+		const fd = fs.openSync(eventsPath, "r+");
+		try {
+			fs.fsyncSync(fd);
+		} catch {
+			// EPERM on Windows CI: best-effort flush
+		} finally {
+			fs.closeSync(fd);
+		}
 	}
 	// R16-B1 (Phase 3.6): monotonic persist under the .seqlock — the reservation
 	// already persisted the batch range end; this covers explicit (pre-assigned)
 	// seqs that may exceed it and can never REGRESS the sidecar (the old bare
 	// persistSequence could write a lower lastSeq over a higher reserved value).
-	persistSequenceMonotonic(eventsPath, lastSeq);
-
-	// Phase 3: cache update + resolve all promises.
-	try {
-		const stat = fs.statSync(eventsPath);
-		if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
-			evictOldestSequenceCacheEntries();
-		}
-		sequenceCache.set(eventsPath, {
-			size: stat.size,
-			mtimeMs: stat.mtimeMs,
-			seq: lastSeq,
-			lastAccessMs: Date.now(),
-		});
-	} catch (error) {
-		logInternalError("event-log.batch-cache-update", error, `eventsPath=${eventsPath}`);
+	// PERF (2026-08-24): R16-B1 advance-on-reserve already persisted the
+	// reserved end inside the .seqlock at reservation time. Re-acquiring
+	// the seqlock (~12 syscalls) to conclude "no write needed" is pure
+	// overhead in the single-writer common case — skip when lastSeq is
+	// covered by the reservation snapshot above; explicit seqs beyond the
+	// reserved range still persist.
+	if (lastSeq > reservedEnd) {
+		persistSequenceMonotonic(eventsPath, lastSeq);
 	}
 
+	// Phase 3: resolve all promises. (PERF 2026-08-24: the sequenceCache upkeep
+	// that used to live here fed no hot reader — see the note in
+	// appendEventAsync; nextSequence re-seeds via its sidecar/scan fallback.)
 	for (const { item, fullEvent } of finalized) item.resolve(fullEvent);
 }
 
 function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): TeamEvent {
 	fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
+	// PERF (2026-08-24): one hoisted pre-append stat replaces the overflow
+	// existsSync+statSync pair below. The after-compaction re-checks further
+	// down keep their own fresh stats — compaction/rotation may have changed
+	// the file since preStat was taken, so they must not see a stale
+	// (pre-compaction) size.
+	let preStat: fs.Stats | undefined;
+	try {
+		preStat = fs.statSync(eventsPath);
+	} catch {
+		/* log absent — first append */
+	}
 	const baseMetadata = event.metadata;
 	// B7: use reserveSequence for atomic seq assignment across all paths.
 	const explicitSeq = baseMetadata?.seq;
@@ -909,9 +917,8 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
 	// 3. After compact, if still over limit, rotate.
 	const isTerminal = TERMINAL_EVENT_TYPES.has(fullEvent.type);
 	let skippedDueToSize = false;
-	if (!isTerminal && fs.existsSync(eventsPath)) {
-		const stat = fs.statSync(eventsPath);
-		if (stat.size > MAX_EVENTS_BYTES) {
+	if (!isTerminal && preStat) {
+		if (preStat.size > MAX_EVENTS_BYTES) {
 			// Try immediate compact (not waiting for counter % 100).
 			// Round 24 (BUG 1): we are INSIDE withEventLogLockSync. Use the unlocked
 			// apply/rotate cores — the locked variants would deadlock (mkdir lock
@@ -967,7 +974,8 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
 	} catch (error) {
 		logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
 	}
-	// seq is already computed above via reserveSequence — reuse it for persist/cache.
+	// seq is already computed above via reserveSequence — reuse it for the
+	// explicit-seq persist below (no sequenceCache upkeep on append paths).
 	// const seq declaration removed (B7: seq is now computed before metadata object).
 	if (!skippedDueToSize) {
 		fs.appendFileSync(eventsPath, `${JSON.stringify(redactSecrets(fullEvent))}\n`, "utf-8");
@@ -997,24 +1005,11 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
 		// Only EXPLICIT (pre-assigned) seqs bypassed the reservation — persist
 		// those, monotonically and under the .seqlock.
 		if (explicitSeq !== undefined) persistSequenceMonotonic(eventsPath, seq);
-		// FIX: Update cache AFTER append so cache and log are consistent with each other.
-		// This matches the async path behavior where cache is updated after the append.
-		// If a crash occurs after append but before cache update, the .seq file is
-		// already correct and nextSequence() will return the correct value on restart.
-		try {
-			const stat = fs.statSync(eventsPath);
-			if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
-				evictOldestSequenceCacheEntries();
-			}
-			sequenceCache.set(eventsPath, {
-				size: stat.size,
-				mtimeMs: stat.mtimeMs,
-				seq,
-				lastAccessMs: Date.now(),
-			});
-		} catch (error) {
-			logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
-		}
+		// PERF (2026-08-24): the per-append sequenceCache upkeep that lived here
+		// (post-append stat + Map set + occasional evict sort) fed no hot reader —
+		// see the note in appendEventAsync. The .seq sidecar is already current
+		// (advance-on-reserve / persistSequenceMonotonic above), and nextSequence()
+		// re-seeds via its sidecar/scan fallback when called.
 	}
 	if (tickAppendCounter(eventsPath) && needsRotation(eventsPath)) {
 		// Round 24 (BUG 1): we are INSIDE withEventLogLockSync here (called via
@@ -1190,6 +1185,18 @@ export function flushBufferedQueuesSync(): void {
 	for (const eventsPath of [...bufferedTimers.keys()]) bufferedTimers.delete(eventsPath);
 }
 
+// Exported only for the buffered-batch unit tests. NOT part of the public API —
+// the `__test__` prefix follows the project convention used in atomic-write.ts,
+// state-store.ts, team-runner.ts, etc. `appendEventBuffered` rejects terminal
+// events BEFORE they reach the batch (see appendEventBuffered), so a batch that
+// deliberately mixes in a terminal event is unreachable through the public API;
+// this hook lets T2 assert the fsync gate directly. Both production call paths
+// (async flush via withEventLogLockAsync and the EL-2 sync flush via
+// withEventLogLockSync) route through appendEventBatchInsideLock.
+export function __test__appendBatchForUnitTest(eventsPath: string, queue: BufferedAppend[]): Promise<void> {
+	return appendEventBatchInsideLock(eventsPath, queue);
+}
+
 /**
  * Schedule an async event append without waiting for the result.
  * Uses the non-blocking async queue to avoid blocking the event loop.

package/src/state/event-log/sequence-cache.ts

@@ -221,10 +221,20 @@ function withSeqLock<T>(eventsPath: string, fn: () => T): T {
 		try {
 			fs.mkdirSync(lockDir);
 			try {
-				// P0-4: the lock pid file is disposable stale-lock state; best-effort.
-				atomicWriteFile(pidFile, String(process.pid), { durability: "best-effort" });
+				// PERF (2026-08-24): "wx" (O_CREAT|O_EXCL) — fails rather than
+				// following a planted symlink, so no O_NOFOLLOW/temp/rename
+				// ceremony needed for this disposable, mtime-stale-detected
+				// 4-byte file. We own the lock dir (we just mkdir'd it), so
+				// EEXIST means a crashed holder's leftover under OUR fresh dir
+				// or an attack — either way, skip: the dir itself is the mutex.
+				const fd = fs.openSync(pidFile, "wx");
+				try {
+					fs.writeSync(fd, String(process.pid));
+				} finally {
+					fs.closeSync(fd);
+				}
 			} catch {
-				/* best-effort */
+				/* best-effort — see withEventLogLockAsync note */
 			}
 			acquired = true;
 		} catch {
@@ -286,6 +296,14 @@ export function persistSequenceMonotonic(eventsPath: string, seq: number): void
 	});
 }
 
+/** PERF (2026-08-24): callers that just reserved a range can skip the lock+read
+ * sidecar round-trip in persistSequenceMonotonic when their value is already
+ * covered by the in-process reservation (R16-B1 advance-on-reserve persisted it
+ * inside the .seqlock at reservation time). */
+export function reservedSequenceEnd(eventsPath: string): number {
+	return seqCounters.get(eventsPath) ?? 0;
+}
+
 // B7: single in-process monotonic sequence counter per eventsPath. The three
 // append paths — sync appendEvent (withEventLogLockSync file lock), buffered
 // flush (asyncLocks promise chain), and direct appendEventAsync (asyncQueues

package/src/state/stores/plan-store.ts

@@ -50,7 +50,7 @@ export function loadPlanRecords(manifest: TeamRunManifest): PlanRecord[] {
 		const raw = fs.readFileSync(planFilePath(manifest), "utf-8");
 		const parsed = JSON.parse(raw) as Partial<PlanFile>;
 		if (!parsed || !Array.isArray(parsed.revisions)) return [];
-		return parsed.revisions.filter((r): r is PlanRecord => Boolean(r && r.id && typeof r.version === "number"));
+		return parsed.revisions.filter((r): r is PlanRecord => Boolean(r?.id && typeof r.version === "number"));
 	} catch (err) {
 		if ((err as NodeJS.ErrnoException).code === "ENOENT") return [];
 		logInternalError("plan-store.read-failed", err instanceof Error ? err : new Error(String(err)), `run=${manifest.runId}`);