pi-extended-teams 2.1.16

package/src/utils/lock.race.child.ts

@@ -0,0 +1,44 @@
+import fs from "node:fs";
+import path from "node:path";
+import { withLock } from "./lock";
+
+const [lockPath, gateDir, pauseBeforeReap, holdCallback, retriesValue] = process.argv.slice(2);
+if (!lockPath || !gateDir) throw new Error("lockPath and gateDir are required");
+
+fs.mkdirSync(gateDir, { recursive: true });
+
+const waitState = new Int32Array(new SharedArrayBuffer(4));
+function waitAtGate(name: string): void {
+  fs.writeFileSync(path.join(gateDir, `${name}.ready`), "");
+  const releaseFile = path.join(gateDir, `${name}.release`);
+  while (!fs.existsSync(releaseFile)) {
+    Atomics.wait(waitState, 0, 0, 10);
+  }
+}
+
+const lockFile = `${lockPath}.lock`;
+const originalUnlinkSync = fs.unlinkSync;
+let pausedBeforeReap = false;
+fs.unlinkSync = ((target: fs.PathLike) => {
+  if (pauseBeforeReap === "true" && String(target) === lockFile && !pausedBeforeReap) {
+    pausedBeforeReap = true;
+    waitAtGate("before-reap");
+  }
+  return originalUnlinkSync(target);
+}) as typeof fs.unlinkSync;
+
+function publishResult(result: object): void {
+  const resultFile = path.join(gateDir, "result.json");
+  const temporary = `${resultFile}.${process.pid}.tmp`;
+  fs.writeFileSync(temporary, JSON.stringify(result));
+  fs.renameSync(temporary, resultFile);
+}
+
+withLock(lockPath, async () => {
+  fs.writeFileSync(path.join(gateDir, "callback.ready"), "");
+  if (holdCallback === "true") waitAtGate("callback");
+  return process.pid;
+}, Number(retriesValue)).then(
+  pid => publishResult({ status: "success", pid }),
+  error => publishResult({ status: "error", message: error instanceof Error ? error.message : String(error) }),
+);

package/src/utils/lock.race.test.ts

@@ -0,0 +1,198 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { spawn, type ChildProcess } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { withLock } from "./lock";
+
+interface ReaperChild {
+  child: ChildProcess;
+  gateDir: string;
+  stderr: () => string;
+}
+
+function waitForAnyFile(files: string[], timeoutMs = 10_000): Promise<string> {
+  const existing = files.find(file => fs.existsSync(file));
+  if (existing) return Promise.resolve(existing);
+
+  return new Promise((resolve, reject) => {
+    const watchers: fs.FSWatcher[] = [];
+    let settled = false;
+    const timeout = setTimeout(() => finish(new Error(`Timed out waiting for ${files.join(" or ")}`)), timeoutMs);
+
+    const finish = (error?: Error, file?: string) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      for (const watcher of watchers) watcher.close();
+      if (error) reject(error);
+      else resolve(file!);
+    };
+
+    const check = () => {
+      const file = files.find(candidate => fs.existsSync(candidate));
+      if (file) finish(undefined, file);
+    };
+
+    for (const directory of new Set(files.map(file => path.dirname(file)))) {
+      watchers.push(fs.watch(directory, check));
+    }
+    check();
+  });
+}
+
+function waitForFile(file: string): Promise<string> {
+  return waitForAnyFile([file]);
+}
+
+function waitForExit(child: ChildProcess): Promise<void> {
+  if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    child.once("exit", () => resolve());
+    child.once("error", reject);
+  });
+}
+
+async function getExitedPid(): Promise<number> {
+  const child = spawn(process.execPath, ["-e", ""], { stdio: "ignore" });
+  const pid = child.pid;
+  if (pid === undefined) throw new Error("Child process did not expose a pid");
+  await waitForExit(child);
+  return pid;
+}
+
+function readResult(gateDir: string): { status: string; message?: string; pid?: number } {
+  return JSON.parse(fs.readFileSync(path.join(gateDir, "result.json"), "utf-8"));
+}
+
+describe("withLock race conditions", () => {
+  const testDir = path.join(os.tmpdir(), "pi-lock-race-test-" + Date.now());
+  const lockPath = path.join(testDir, "test");
+  const lockFile = `${lockPath}.lock`;
+  const children: ReaperChild[] = [];
+
+  function startReaper(name: string, options: { pauseBeforeReap: boolean; holdCallback: boolean; retries: number }): ReaperChild {
+    const gateDir = path.join(testDir, name);
+    fs.mkdirSync(gateDir, { recursive: true });
+    const fixture = path.join(__dirname, "lock.race.child.ts");
+    const child = spawn(process.execPath, [
+      "-r",
+      "ts-node/register",
+      fixture,
+      lockPath,
+      gateDir,
+      String(options.pauseBeforeReap),
+      String(options.holdCallback),
+      String(options.retries),
+    ], {
+      cwd: path.resolve(__dirname, "../.."),
+      stdio: ["ignore", "ignore", "pipe"],
+    });
+    let stderr = "";
+    child.stderr?.on("data", chunk => { stderr += chunk.toString(); });
+    const running = { child, gateDir, stderr: () => stderr };
+    children.push(running);
+    return running;
+  }
+
+  function release(child: ReaperChild, gate: "before-reap" | "callback"): void {
+    fs.writeFileSync(path.join(child.gateDir, `${gate}.release`), "");
+  }
+
+  beforeEach(() => {
+    if (!fs.existsSync(testDir)) fs.mkdirSync(testDir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    for (const running of children) {
+      if (running.child.exitCode === null && running.child.signalCode === null) running.child.kill("SIGKILL");
+    }
+    await Promise.all(children.map(running => waitForExit(running.child).catch(() => undefined)));
+    children.length = 0;
+    if (fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true });
+  });
+
+  it("should handle multiple concurrent attempts to acquire the lock", async () => {
+    let counter = 0;
+    const iterations = 20;
+    const concurrentCount = 5;
+
+    const runTask = async () => {
+      for (let i = 0; i < iterations; i++) {
+        await withLock(lockPath, async () => {
+          const current = counter;
+          await new Promise(resolve => setTimeout(resolve, Math.random() * 10));
+          counter = current + 1;
+        });
+      }
+    };
+
+    await Promise.all(Array.from({ length: concurrentCount }, runTask));
+
+    expect(counter).toBe(iterations * concurrentCount);
+  });
+
+  it("does not revoke a live paused reaper or let it delete a successor lock", async () => {
+    const deadOwnerPid = await getExitedPid();
+    fs.writeFileSync(lockFile, JSON.stringify({ pid: deadOwnerPid, token: "dead-lock", acquiredAt: 1 }));
+    const longAgo = new Date(Date.now() - 60_000);
+    fs.utimesSync(lockFile, longAgo, longAgo);
+
+    const first = startReaper("first", { pauseBeforeReap: true, holdCallback: true, retries: 20 });
+    await waitForFile(path.join(first.gateDir, "before-reap.ready"));
+
+    const reapGuard = `${lockFile}.reap`;
+    fs.utimesSync(reapGuard, longAgo, longAgo);
+
+    const second = startReaper("second", { pauseBeforeReap: true, holdCallback: true, retries: 10 });
+    const secondBeforeReap = path.join(second.gateDir, "before-reap.ready");
+    const secondResult = path.join(second.gateDir, "result.json");
+    const secondOutcome = await waitForAnyFile([secondBeforeReap, secondResult]);
+
+    let callbacksOverlapped = false;
+    if (secondOutcome === secondBeforeReap) {
+      release(first, "before-reap");
+      await waitForFile(path.join(first.gateDir, "callback.ready"));
+      release(second, "before-reap");
+      await waitForFile(path.join(second.gateDir, "callback.ready"));
+      callbacksOverlapped = true;
+      release(first, "callback");
+      release(second, "callback");
+      await Promise.all([
+        waitForFile(path.join(first.gateDir, "result.json")),
+        waitForFile(path.join(second.gateDir, "result.json")),
+      ]);
+    } else {
+      expect(readResult(second.gateDir)).toMatchObject({ status: "error", message: "Could not acquire lock" });
+      release(first, "before-reap");
+      await waitForFile(path.join(first.gateDir, "callback.ready"));
+      release(first, "callback");
+      await waitForFile(path.join(first.gateDir, "result.json"));
+      expect(readResult(first.gateDir)).toMatchObject({ status: "success" });
+    }
+
+    expect(callbacksOverlapped).toBe(false);
+    expect(first.stderr()).toBe("");
+    expect(second.stderr()).toBe("");
+  }, 20_000);
+
+  it("recovers after a reaper genuinely dies while holding the guard", async () => {
+    const deadOwnerPid = await getExitedPid();
+    fs.writeFileSync(lockFile, JSON.stringify({ pid: deadOwnerPid, token: "dead-lock", acquiredAt: 1 }));
+    const longAgo = new Date(Date.now() - 60_000);
+    fs.utimesSync(lockFile, longAgo, longAgo);
+
+    const crashed = startReaper("crashed", { pauseBeforeReap: true, holdCallback: false, retries: 20 });
+    await waitForFile(path.join(crashed.gateDir, "before-reap.ready"));
+    crashed.child.kill("SIGKILL");
+    await waitForExit(crashed.child);
+    fs.utimesSync(`${lockFile}.reap`, longAgo, longAgo);
+
+    const recovery = startReaper("recovery", { pauseBeforeReap: false, holdCallback: false, retries: 200 });
+    await waitForFile(path.join(recovery.gateDir, "result.json"));
+
+    expect(readResult(recovery.gateDir)).toMatchObject({ status: "success" });
+    expect(fs.existsSync(lockFile)).toBe(false);
+    expect(recovery.stderr()).toBe("");
+  }, 20_000);
+});

package/src/utils/lock.test.ts

@@ -0,0 +1,90 @@
+// Project: pi-extended-teams
+import { describe, it, expect, vi, afterEach, beforeEach } from "vitest";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { withLock } from "./lock";
+
+describe("withLock", () => {
+  const testDir = path.join(os.tmpdir(), "pi-lock-test-" + Date.now());
+  const lockPath = path.join(testDir, "test");
+  const lockFile = `${lockPath}.lock`;
+
+  beforeEach(() => {
+    if (!fs.existsSync(testDir)) fs.mkdirSync(testDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    if (fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true });
+  });
+
+  it("should successfully acquire and release the lock", async () => {
+    const fn = vi.fn().mockResolvedValue("result");
+    const result = await withLock(lockPath, fn);
+
+    expect(result).toBe("result");
+    expect(fn).toHaveBeenCalled();
+    expect(fs.existsSync(lockFile)).toBe(false);
+  });
+
+  it("should fail to acquire lock if already held", async () => {
+    // Manually create lock file
+    fs.writeFileSync(lockFile, "9999");
+
+    const fn = vi.fn().mockResolvedValue("result");
+    
+    // Test with only 2 retries to speed up the failure
+    await expect(withLock(lockPath, fn, 2)).rejects.toThrow("Could not acquire lock");
+    expect(fn).not.toHaveBeenCalled();
+  });
+
+  it("should release lock even if function fails", async () => {
+    const fn = vi.fn().mockRejectedValue(new Error("failure"));
+
+    await expect(withLock(lockPath, fn)).rejects.toThrow("failure");
+    expect(fs.existsSync(lockFile)).toBe(false);
+  });
+
+  it("removes stale lock files before acquiring", async () => {
+    fs.writeFileSync(lockFile, JSON.stringify({ pid: 9999, token: "stale", acquiredAt: 1 }));
+    const staleTime = new Date(Date.now() - 60_000);
+    fs.utimesSync(lockFile, staleTime, staleTime);
+
+    const fn = vi.fn().mockResolvedValue("result");
+    await expect(withLock(lockPath, fn)).resolves.toBe("result");
+    expect(fn).toHaveBeenCalled();
+    expect(fs.existsSync(lockFile)).toBe(false);
+  });
+
+  it("heartbeats held locks so long critical sections do not become stale", async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-06-19T12:00:00.000Z"));
+    let release!: () => void;
+
+    const running = withLock(lockPath, async () => new Promise<string>((resolve) => {
+      release = () => resolve("done");
+    }));
+    await vi.advanceTimersByTimeAsync(0);
+
+    const staleTime = new Date(Date.now() - 60_000);
+    fs.utimesSync(lockFile, staleTime, staleTime);
+    const initialMtime = fs.statSync(lockFile).mtimeMs;
+    await vi.advanceTimersByTimeAsync(12_000);
+
+    expect(fs.statSync(lockFile).mtimeMs).toBeGreaterThan(initialMtime);
+
+    release();
+    await expect(running).resolves.toBe("done");
+    expect(fs.existsSync(lockFile)).toBe(false);
+  });
+
+  it("does not remove a lock now owned by another holder", async () => {
+    await withLock(lockPath, async () => {
+      fs.writeFileSync(lockFile, JSON.stringify({ pid: 1234, token: "other-owner", acquiredAt: Date.now() }));
+    });
+
+    expect(fs.existsSync(lockFile)).toBe(true);
+    expect(JSON.parse(fs.readFileSync(lockFile, "utf-8"))).toMatchObject({ token: "other-owner" });
+  });
+});

package/src/utils/lock.ts

@@ -0,0 +1,186 @@
+// Project: pi-extended-teams
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+const LOCK_TIMEOUT = 30000; // 30 seconds of retrying under high fan-out contention
+const LOCK_RETRY_DELAY_MS = 10;
+const DEFAULT_LOCK_RETRIES = Math.ceil(LOCK_TIMEOUT / LOCK_RETRY_DELAY_MS);
+const STALE_LOCK_TIMEOUT = 30000; // 30 seconds for a lock to be considered stale
+const LOCK_HEARTBEAT_INTERVAL_MS = Math.max(1000, Math.floor(STALE_LOCK_TIMEOUT / 3));
+
+interface LockOwner {
+  pid: number;
+  token: string;
+  acquiredAt: number;
+}
+
+function readLockOwner(lockFile: string): LockOwner | null {
+  try {
+    const raw = fs.readFileSync(lockFile, "utf-8");
+    const parsed = JSON.parse(raw);
+    if (
+      parsed
+      && Number.isSafeInteger(parsed.pid)
+      && parsed.pid > 0
+      && typeof parsed.token === "string"
+      && typeof parsed.acquiredAt === "number"
+    ) return parsed;
+    if (Number.isSafeInteger(parsed) && parsed > 0) {
+      // PID reuse can delay reclaiming a legacy lock, but the liveness check
+      // below will never revoke it while any process has that PID.
+      return { pid: parsed, token: `legacy-pid:${parsed}`, acquiredAt: 0 };
+    }
+  } catch {
+    // Malformed owner records cannot be reclaimed without positive evidence.
+  }
+  return null;
+}
+
+function errorCode(error: unknown): string | undefined {
+  return error && typeof error === "object" && "code" in error
+    ? String((error as NodeJS.ErrnoException).code)
+    : undefined;
+}
+
+function processIsDefinitelyDead(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return false;
+  } catch (error) {
+    // EPERM means the process exists but cannot be signalled. Only ESRCH is
+    // positive evidence that a local process no longer exists.
+    return errorCode(error) === "ESRCH";
+  }
+}
+
+function publishReapGuard(guardFile: string, owner: LockOwner): boolean {
+  const candidateName = `.candidate-${crypto.createHash("sha256").update(owner.token).digest("hex")}`;
+  const candidate = path.join(path.dirname(guardFile), candidateName);
+
+  try {
+    try {
+      fs.writeFileSync(candidate, JSON.stringify(owner), { flag: "wx" });
+    } catch (error) {
+      if (errorCode(error) !== "EEXIST" || readLockOwner(candidate)?.token !== owner.token) return false;
+    }
+
+    // The complete candidate is published with a no-clobber hard link. A
+    // process dying before this point leaves no visible epoch; dying after it
+    // leaves a complete owner record that a later epoch can recover from.
+    fs.linkSync(candidate, guardFile);
+    return true;
+  } catch {
+    return false;
+  } finally {
+    try {
+      if (readLockOwner(candidate)?.token === owner.token) fs.unlinkSync(candidate);
+    } catch {
+      // The candidate is unique to this owner and may already be gone.
+    }
+  }
+}
+
+function acquireReapGuard(lockFile: string, staleOwner: LockOwner, reaper: LockOwner): boolean {
+  const guardDir = `${lockFile}.reap`;
+  try {
+    fs.mkdirSync(guardDir, { recursive: true });
+  } catch {
+    return false;
+  }
+
+  const generation = crypto.createHash("sha256").update(staleOwner.token).digest("hex");
+  for (let epoch = 0; ; epoch++) {
+    const guardFile = path.join(guardDir, `${generation}.${epoch}`);
+    if (publishReapGuard(guardFile, reaper)) return true;
+
+    const guardOwner = readLockOwner(guardFile);
+    if (guardOwner?.token === reaper.token) return true;
+    if (!guardOwner || !processIsDefinitelyDead(guardOwner.pid)) return false;
+    // Guard epochs are immutable and never unlinked. After a proven owner
+    // death, contenders race to publish the next epoch instead of replacing
+    // state that another live contender may already own.
+  }
+}
+
+function removeStaleLock(lockFile: string, reaper: LockOwner): void {
+  try {
+    const staleOwner = readLockOwner(lockFile);
+    if (!staleOwner) return;
+    if (Date.now() - fs.statSync(lockFile).mtimeMs <= STALE_LOCK_TIMEOUT) return;
+    if (!processIsDefinitelyDead(staleOwner.pid)) return;
+    if (!acquireReapGuard(lockFile, staleOwner, reaper)) return;
+
+    // Re-check both generation and liveness while holding the generation's
+    // immutable guard. No other live reaper for this generation can reach the
+    // unlink, so the path cannot be replaced between this check and removal.
+    const currentOwner = readLockOwner(lockFile);
+    if (currentOwner?.token !== staleOwner.token) return;
+    if (Date.now() - fs.statSync(lockFile).mtimeMs <= STALE_LOCK_TIMEOUT) return;
+    if (!processIsDefinitelyDead(currentOwner.pid)) return;
+    fs.unlinkSync(lockFile);
+  } catch {
+    // Ignore: another process may have removed it, or it may not exist yet.
+  }
+}
+
+function touchOwnedLock(lockFile: string, token: string): void {
+  const owner = readLockOwner(lockFile);
+  if (owner?.token !== token) return;
+  try {
+    const now = new Date();
+    fs.utimesSync(lockFile, now, now);
+  } catch {
+    // Ignore: another process may have removed it after the ownership check.
+  }
+}
+
+function releaseOwnedLock(lockFile: string, token: string): void {
+  const owner = readLockOwner(lockFile);
+  if (owner?.token !== token) return;
+  try {
+    fs.unlinkSync(lockFile);
+  } catch {
+    // Ignore: another process may have removed it after the ownership check.
+  }
+}
+
+export async function withLock<T>(lockPath: string, fn: () => Promise<T>, retries: number = DEFAULT_LOCK_RETRIES): Promise<T> {
+  const lockFile = `${lockPath}.lock`;
+  const lockDir = path.dirname(lockFile);
+  const owner: LockOwner = {
+    pid: process.pid,
+    token: `${process.pid}:${Date.now()}:${crypto.randomUUID()}`,
+    acquiredAt: Date.now(),
+  };
+
+  if (!fs.existsSync(lockDir)) fs.mkdirSync(lockDir, { recursive: true });
+
+  let remainingRetries = retries;
+  let acquired = false;
+  while (remainingRetries > 0) {
+    try {
+      removeStaleLock(lockFile, owner);
+      fs.writeFileSync(lockFile, JSON.stringify(owner), { flag: "wx" });
+      acquired = true;
+      break;
+    } catch {
+      remainingRetries--;
+      await new Promise(resolve => setTimeout(resolve, LOCK_RETRY_DELAY_MS));
+    }
+  }
+
+  if (!acquired) {
+    throw new Error("Could not acquire lock");
+  }
+
+  const heartbeat = setInterval(() => touchOwnedLock(lockFile, owner.token), LOCK_HEARTBEAT_INTERVAL_MS);
+  heartbeat.unref?.();
+
+  try {
+    return await fn();
+  } finally {
+    clearInterval(heartbeat);
+    releaseOwnedLock(lockFile, owner.token);
+  }
+}