pi-claude-supervisor 0.6.0 → 0.7.0

package/src/cwd-lease.ts

@@ -276,6 +276,16 @@ export class CwdLeaseStore {
     return this.#withLock(() => this.#readAll());
   }
 
+  /** Sorted quarantined lease file names, for operator inspection/cleanup. */
+  async quarantined(): Promise<string[]> {
+    try {
+      return (await readdir(this.#quarantineDirectory())).sort();
+    } catch (error) {
+      if (error instanceof Error && /ENOENT/u.test(error.message)) return [];
+      throw error;
+    }
+  }
+
   #handle(initial: CwdLeaseRecord, replacedTaskId?: string): CwdLeaseHandle {
     let current = { ...initial, worker: initial.worker ? { ...initial.worker } : undefined };
     let released = false;
@@ -326,46 +336,101 @@ export class CwdLeaseStore {
     const leases: CwdLeaseRecord[] = [];
     for (const name of names.filter((item) => item.endsWith(".json"))) {
       const leasePath = join(this.#directory, name);
-      const leaseInfo = await lstat(leasePath);
-      if (!leaseInfo.isFile()) throw new Error(`cwd lease entry is not a regular file: ${redactText(leasePath)}`);
-      const value = JSON.parse(await readFile(leasePath, "utf8")) as Partial<CwdLeaseRecord>;
-      let lease = normalizeLease(value);
-      if (lease.pendingCleanup) {
-        // A takeover transaction is not usable until the old cgroup/session
-        // and any socket marker have been independently confirmed clean.
-        const cgroupCleanedBefore = lease.pendingCleanup.cgroupCleaned === true;
-        let cleanupComplete = await cleanupPendingLease(lease.pendingCleanup);
-        // Reconciliation may have removed the cgroup successfully but failed
-        // on a later marker operation. Persist that stage before returning a
-        // still-pending record, so the next reader can continue fail-closed.
-        if (!cgroupCleanedBefore && lease.pendingCleanup.cgroupCleaned === true) {
-          await this.#write({ ...lease, updatedAt: new Date().toISOString() });
-          // The first pass deliberately stops before socket-marker removal;
-          // the cleanup stage must be durable before that second proof.
-          cleanupComplete = await cleanupPendingLease(lease.pendingCleanup);
+      let lease: CwdLeaseRecord;
+      try {
+        const leaseInfo = await lstat(leasePath);
+        if (!leaseInfo.isFile()) {
+          // A directory or symlink is never renamed or followed as part of
+          // quarantine; just skip it in place and let an operator inspect it.
+          console.error(`pi-claude-supervisor skipped a non-regular cwd lease entry: ${redactText(name)}`);
+          continue;
         }
-        if (cleanupComplete) {
-          if (lease.pendingCleanup.phase === "replacement") {
-            // The replacement is the live lease after recovery. Clear only
-            // its transaction marker; do not delete the cwd reservation.
-            delete lease.pendingCleanup;
-            await this.#write(lease);
-          } else {
-            // The old record was still in the preparation phase, so no new
-            // Worker lease exists to retain after cleanup.
-            await rm(leasePath, { force: true });
-            continue;
+        const value = JSON.parse(await readFile(leasePath, "utf8")) as Partial<CwdLeaseRecord>;
+        lease = normalizeLease(value);
+      } catch (error) {
+        if (isTransientIoError(error)) throw error;
+        // One unreadable or incompatible record (disk full, truncated write,
+        // a schema this build no longer accepts) must not block every other
+        // cwd lease in the registry. Quarantine it out of the directory this
+        // scan reads instead, and keep going.
+        await this.#quarantine(name, leasePath, error);
+        continue;
+      }
+      if (lease.pendingCleanup) {
+        try {
+          // A takeover transaction is not usable until the old cgroup/session
+          // and any socket marker have been independently confirmed clean.
+          const cgroupCleanedBefore = lease.pendingCleanup.cgroupCleaned === true;
+          let cleanupComplete = await cleanupPendingLease(lease.pendingCleanup);
+          // Reconciliation may have removed the cgroup successfully but failed
+          // on a later marker operation. Persist that stage before returning a
+          // still-pending record, so the next reader can continue fail-closed.
+          if (!cgroupCleanedBefore && lease.pendingCleanup.cgroupCleaned === true) {
+            await this.#write({ ...lease, updatedAt: new Date().toISOString() });
+            // The first pass deliberately stops before socket-marker removal;
+            // the cleanup stage must be durable before that second proof.
+            cleanupComplete = await cleanupPendingLease(lease.pendingCleanup);
           }
+          if (cleanupComplete) {
+            if (lease.pendingCleanup.phase === "replacement") {
+              // The replacement is the live lease after recovery. Clear only
+              // its transaction marker; do not delete the cwd reservation.
+              delete lease.pendingCleanup;
+              await this.#write(lease);
+            } else {
+              // The old record was still in the preparation phase, so no new
+              // Worker lease exists to retain after cleanup.
+              await rm(leasePath, { force: true });
+              continue;
+            }
+          }
+        } catch (error) {
+          if (isTransientIoError(error)) throw error;
+          // A reconciliation failure is not proof the record itself is
+          // corrupt, and quarantining it would discard a durable
+          // pendingCleanup transaction a later read could still finish. Skip
+          // it for this read only and leave the file in place.
+          console.error(`pi-claude-supervisor skipped a cwd lease record with a failed pendingCleanup reconciliation: ${redactText(name)}: ${errorMessage(error)}`);
+          continue;
         }
       }
-      // A missing or replaced cwd invalidates the registry evidence. Do not
-      // silently drop that record and allow a second worker to start.
-      lease.cwd = await realpath(lease.cwd);
+      // A missing cwd is registry evidence that this lease's directory is
+      // gone, but it must not be treated the same as an unreadable record:
+      // silently dropping the lease would let a second worker start against
+      // the same path. Callers only ever compare `existing.cwd` against a
+      // freshly canonicalized candidate cwd, so keeping the raw (already
+      // `resolve`d by normalizeLease) path here still fails closed for that
+      // same/overlapping path without blocking every unrelated cwd lookup
+      // just because one lease's directory happened to disappear.
+      try {
+        lease.cwd = await realpath(lease.cwd);
+      } catch {
+        lease.cwd = resolve(lease.cwd);
+      }
       leases.push(lease);
     }
     return leases;
   }
 
+  async #quarantine(name: string, leasePath: string, error: unknown): Promise<void> {
+    const message = errorMessage(error);
+    const quarantineDir = this.#quarantineDirectory();
+    try {
+      await mkdir(quarantineDir, { recursive: true, mode: 0o700 });
+      await rename(leasePath, join(quarantineDir, `${name}.${Date.now()}`));
+    } catch {
+      // If the rename itself fails, leave the file in place and skip it for
+      // this read; the next read will retry rather than lose it silently.
+      console.error(`pi-claude-supervisor could not quarantine an unreadable cwd lease record: ${redactText(name)}: ${message}`);
+      return;
+    }
+    console.error(`pi-claude-supervisor quarantined an unreadable cwd lease record: ${redactText(name)}: ${message}`);
+  }
+
+  #quarantineDirectory(): string {
+    return join(this.#directory, "quarantine");
+  }
+
   async #write(lease: CwdLeaseRecord): Promise<void> {
     await this.#ensureDirectory();
     const target = this.#path(lease.leaseId);
@@ -570,8 +635,8 @@ async function cgroupHasProcesses(path: string): Promise<boolean> {
   try {
     const cgroupInfo = await lstat(cgroupPath);
     if (!cgroupInfo.isDirectory() || cgroupInfo.isSymbolicLink()) return true;
-    const canonicalPath = await realpath(cgroupPath);
-    if (canonicalPath !== cgroupPath || !canonicalPath.startsWith(`${cgroupRoot}/`)) return true;
+    const canonicalPath = await canonicalCgroupPath(cgroupPath);
+    if (!canonicalPath.startsWith(`${cgroupRoot}/`)) return true;
     const eventsPath = join(canonicalPath, "cgroup.events");
     const eventsInfo = await lstat(eventsPath);
     if (!eventsInfo.isFile() || eventsInfo.isSymbolicLink()) return true;
@@ -840,6 +905,24 @@ function isCgroupPath(path: string): boolean {
   return candidate !== root && candidate.startsWith(`${root}/`);
 }
 
+/**
+ * Bun's realpath implementation decodes systemd's literal `\\x2d` cgroup
+ * escapes and then reports ENOENT. Verify every cgroup component with lstat
+ * instead; this also rejects symlinked components without relying on realpath.
+ */
+async function canonicalCgroupPath(path: string): Promise<string> {
+  const root = resolve("/sys/fs/cgroup");
+  const candidate = resolve(path);
+  if (candidate === root || !candidate.startsWith(`${root}/`)) throw new Error("invalid cgroup path");
+  let current = root;
+  for (const component of candidate.slice(root.length + 1).split("/")) {
+    current = join(current, component);
+    const info = await lstat(current);
+    if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("cgroup path is not a real directory");
+  }
+  return candidate;
+}
+
 function matchesGeneratedCgroupName(path: string, transport: CwdLeaseTransport, workerId: string): boolean {
   const prefix = transport === "tmux" ? "pi-claude-supervisor-tmux-" : "pi-claude-supervisor-";
   return basename(resolve(path)) === `${prefix}${workerId}`;
@@ -847,11 +930,8 @@ function matchesGeneratedCgroupName(path: string, transport: CwdLeaseTransport,
 
 async function readCgroupIdentity(path: string): Promise<{ device: string; inode: string }> {
   if (!isCgroupPath(path)) throw new Error("invalid cgroup path");
-  const candidate = resolve(path);
+  const candidate = await canonicalCgroupPath(path);
   const info = await lstat(candidate);
-  if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("cgroup path is not a real directory");
-  const canonicalPath = await realpath(candidate);
-  if (canonicalPath !== candidate || !isCgroupPath(canonicalPath)) throw new Error("cgroup path is not canonical");
   return { device: String(info.dev), inode: String(info.ino) };
 }
 
@@ -864,11 +944,46 @@ async function cgroupIdentityMatches(path: string, expected: { device: string; i
   }
 }
 
+/**
+ * systemd/claude can create nested cgroups under a Worker's own cgroup, so an
+ * empty parent (`populated 0`) can still contain an empty child directory
+ * that plain `rmdir` leaves behind with ENOTEMPTY. Remove bottom-up so the
+ * parent is only ever asked to `rmdir` once every descendant is gone.
+ * `hasProcesses` is injectable because `cgroupHasProcesses` refuses to
+ * inspect any path outside `/sys/fs/cgroup`, which this function's own
+ * (already `/sys/fs/cgroup`-scoped) recursion does not need to re-derive.
+ */
+export async function removeEmptyChildCgroups(path: string, hasProcesses: (candidate: string) => Promise<boolean> = cgroupHasProcesses): Promise<boolean> {
+  let entries;
+  try {
+    entries = await readdir(path, { withFileTypes: true });
+  } catch {
+    // A directory that has already been removed out from under us (e.g. by a
+    // concurrent guardian teardown) is not proof it is safe to remove; every
+    // neighboring check in this file (cgroupIdentityMatches, cgroupHasProcesses,
+    // the final rmdir) also fails closed on an unexpected fs error.
+    return false;
+  }
+  for (const entry of entries) {
+    if (!entry.isDirectory() || entry.isSymbolicLink()) continue;
+    const child = join(path, entry.name);
+    if (!await removeEmptyChildCgroups(child, hasProcesses)) return false;
+    if (await hasProcesses(child)) return false;
+    try {
+      await rmdir(child);
+    } catch {
+      return false;
+    }
+  }
+  return true;
+}
+
 async function removeEmptyCgroup(path: string, expected: { device: string; inode: string }): Promise<boolean> {
   // Re-check both the persisted directory identity and emptiness immediately
   // before removal. The initial checks only authorize attempting takeover;
   // they must not be reused after the pre-replacement hook has run.
   if (!await cgroupIdentityMatches(path, expected) || await cgroupHasProcesses(path)) return false;
+  if (!await removeEmptyChildCgroups(resolve(path))) return false;
   try {
     await rmdir(resolve(path));
     return true;
@@ -1116,6 +1231,20 @@ function redactText(value: string): string {
   return String(redactSensitive(value));
 }
 
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/**
+ * A transient condition (resource exhaustion, a concurrent lock) is not
+ * evidence a lease record is corrupt. Rethrow it instead of quarantining or
+ * skipping the record on its account.
+ */
+function isTransientIoError(error: unknown): boolean {
+  const code = (error as NodeJS.ErrnoException | undefined)?.code;
+  return code === "EAGAIN" || code === "EBUSY";
+}
+
 function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }

package/src/decision-session-store.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { chmod, lstat, mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
-import { basename, dirname, isAbsolute, join, resolve } from "node:path";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { redactSensitive } from "./redaction.ts";
 import { normalizeTaskSpec } from "./acceptance.ts";
 import type { TaskSpec } from "./types.ts";
@@ -34,6 +34,8 @@ export interface DecisionSessionRecord {
   turn: number;
   repairRound?: number;
   lastFindingSignature?: string;
+  /** Cumulative Claude Worker API cost so far; recovery seeds the next Worker's budget accounting with it. */
+  workerCostUsd?: number;
   state: "active" | "closed";
   recoveryState: DecisionRecoveryState;
   recoveryAttempt: number;
@@ -92,7 +94,44 @@ export class DecisionSessionStore {
     });
   }
 
-  async update(taskId: string, patch: Partial<Pick<DecisionSessionRecord, "turn" | "repairRound" | "lastFindingSignature" | "updatedAt">>): Promise<void> {
+  /**
+   * Remove closed records (and their session directories) whose last update is
+   * older than maxAgeMs. Active records are never touched. A record whose
+   * directory cannot be safely identified as its own task directory (moved,
+   * missing, or replaced by a symlink) is left in place and logged, not removed.
+   */
+  async prune(options: { maxAgeMs: number; now?: number }): Promise<{ removed: string[] }> {
+    if (options.maxAgeMs <= 0) return { removed: [] };
+    const now = options.now ?? Date.now();
+    return this.#withLock(async () => {
+      const removed: string[] = [];
+      for (const record of await this.list()) {
+        if (record.state !== "closed") continue;
+        const updatedAtMs = Date.parse(record.updatedAt);
+        if (!Number.isFinite(updatedAtMs) || now - updatedAtMs < options.maxAgeMs) continue;
+        try {
+          const directory = resolve(dirname(record.decisionSessionFile));
+          if (directory !== this.sessionDirectory(record.taskId)) {
+            throw new Error("session directory does not match its task id");
+          }
+          const relativeToStore = relative(this.#directory, directory);
+          if (isAbsolute(relativeToStore) || relativeToStore === ".." || relativeToStore.startsWith(`..${sep}`)) {
+            throw new Error("session directory escaped the store directory");
+          }
+          const info = await lstat(directory);
+          if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("session directory is not a real directory");
+          await rm(directory, { recursive: true, force: true });
+          await rm(this.#recordPath(record.taskId), { force: true });
+          removed.push(record.taskId);
+        } catch (error) {
+          console.error(`pi-claude-supervisor could not prune Decision Worker session ${record.taskId}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+      }
+      return { removed };
+    });
+  }
+
+  async update(taskId: string, patch: Partial<Pick<DecisionSessionRecord, "turn" | "repairRound" | "lastFindingSignature" | "workerCostUsd" | "updatedAt">>): Promise<void> {
     assertTaskId(taskId);
     await this.#withLock(async () => {
       const record = await this.#loadUnlocked(taskId);
@@ -366,6 +405,7 @@ function normalizeRecord(value: Partial<DecisionSessionRecord>, directory: strin
     || (value.recoveryOwnerStartTime !== undefined && (typeof value.recoveryOwnerStartTime !== "string" || !/^\d+$/u.test(value.recoveryOwnerStartTime)))
     || (value.recoveryState !== undefined && !isRecoveryState(value.recoveryState))
     || (value.lastFindingSignature !== undefined && (typeof value.lastFindingSignature !== "string" || value.lastFindingSignature.length > 128))
+    || (value.workerCostUsd !== undefined && (typeof value.workerCostUsd !== "number" || !Number.isFinite(value.workerCostUsd) || value.workerCostUsd < 0))
     || (value.baseCommit !== undefined && (typeof value.baseCommit !== "string" || !/^[0-9a-f]{40,64}$/iu.test(value.baseCommit)))
     || (value.baseBranch !== undefined && (typeof value.baseBranch !== "string" || !/^[A-Za-z0-9._/-]+$/u.test(value.baseBranch)))
     || (value.resolvedExecutable !== undefined && (typeof value.resolvedExecutable !== "string" || !isAbsolute(value.resolvedExecutable) || value.resolvedExecutable.length > 4_096))
@@ -403,6 +443,7 @@ function normalizeRecord(value: Partial<DecisionSessionRecord>, directory: strin
     turn: value.turn ?? 0,
     repairRound: value.repairRound ?? 0,
     ...(typeof value.lastFindingSignature === "string" ? { lastFindingSignature: value.lastFindingSignature } : {}),
+    ...(typeof value.workerCostUsd === "number" ? { workerCostUsd: value.workerCostUsd } : {}),
     state: value.state,
     recoveryState: value.recoveryState ?? "ready",
     recoveryAttempt: value.recoveryAttempt ?? 0,