pi-extended-teams 2.1.16

package/src/utils/report-events.test.ts

@@ -0,0 +1,154 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { appendTeamReportEvent, listTeamReportEvents } from "./report-events";
+import * as paths from "./paths";
+import type { TeamReportEvent } from "./models";
+
+let root: string;
+
+function reportsPath(teamName = "team"): string {
+  return path.join(root, "teams", paths.sanitizeName(teamName), "reports.json");
+}
+
+function event(overrides: Partial<TeamReportEvent>): TeamReportEvent {
+  return {
+    id: overrides.id || `event-${overrides.createdAt}`,
+    teamName: overrides.teamName || "team",
+    agentName: overrides.agentName || "reader",
+    status: overrides.status || "completed",
+    report: overrides.report || "done",
+    createdAt: overrides.createdAt || Date.now(),
+    source: overrides.source || "read-agent",
+    ...overrides,
+  };
+}
+
+describe("report events", () => {
+  beforeEach(() => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), "pi-extended-teams-reports-"));
+    vi.spyOn(paths, "teamDir").mockImplementation((teamName: unknown) => path.join(root, "teams", paths.sanitizeName(String(teamName))));
+    vi.spyOn(paths, "reportFilesDir").mockReturnValue(path.join(root, "agent", "reports"));
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    if (root && fs.existsSync(root)) fs.rmSync(root, { recursive: true, force: true });
+  });
+
+  it("filters before applying latest-limit pagination", async () => {
+    await appendTeamReportEvent("team", event({ id: "a-100", agentName: "agent-a", createdAt: 100, summary: "a100" }));
+    await appendTeamReportEvent("team", event({ id: "b-200", agentName: "agent-b", createdAt: 200, summary: "b200" }));
+    await appendTeamReportEvent("team", event({ id: "a-300", agentName: "agent-a", createdAt: 300, summary: "a300" }));
+    await appendTeamReportEvent("team", event({ id: "b-400", agentName: "agent-b", createdAt: 400, summary: "b400" }));
+    await appendTeamReportEvent("team", event({ id: "a-500", agentName: "agent-a", createdAt: 500, summary: "a500" }));
+
+    const agentReports = await listTeamReportEvents("team", { agentName: "agent-a", limit: 2 });
+    expect(agentReports.map(report => report.summary)).toEqual(["a300", "a500"]);
+
+    const sinceReports = await listTeamReportEvents("team", { since: 250, limit: 2 });
+    expect(sinceReports.map(report => report.summary)).toEqual(["b400", "a500"]);
+
+    await expect(listTeamReportEvents("team", { limit: 0 })).resolves.toEqual([]);
+  });
+
+  it("sorts externally-written files before slicing latest events", async () => {
+    const p = reportsPath();
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, JSON.stringify([
+      event({ id: "300", createdAt: 300, summary: "third" }),
+      event({ id: "100", createdAt: 100, summary: "first" }),
+      event({ id: "200", createdAt: 200, summary: "second" }),
+    ]));
+
+    const reports = await listTeamReportEvents("team", { limit: 2 });
+
+    expect(reports.map(report => report.summary)).toEqual(["second", "third"]);
+  });
+
+  it("reuses the in-process index until the report file changes", async () => {
+    await appendTeamReportEvent("team", event({ id: "one", createdAt: 100, summary: "one" }));
+
+    const readSpy = vi.spyOn(fs, "readFileSync");
+    const first = await listTeamReportEvents("team");
+    const second = await listTeamReportEvents("team");
+
+    expect(first.map(report => report.id)).toEqual(["one"]);
+    expect(second.map(report => report.id)).toEqual(["one"]);
+    expect(readSpy.mock.calls.filter(call => String(call[0]).endsWith("reports.json"))).toHaveLength(0);
+
+    const p = reportsPath();
+    fs.writeFileSync(p, JSON.stringify([
+      ...first,
+      event({ id: "two", createdAt: 200, summary: "two" }),
+    ]));
+
+    const afterExternalWrite = await listTeamReportEvents("team");
+    expect(afterExternalWrite.map(report => report.id)).toEqual(["one", "two"]);
+    expect(readSpy.mock.calls.filter(call => String(call[0]).endsWith("reports.json"))).toHaveLength(1);
+  });
+
+  it("isolates cached report objects from caller mutation", async () => {
+    const appended = await appendTeamReportEvent("team", event({
+      id: "immutable",
+      createdAt: 100,
+      summary: "original",
+      metadata: { nested: { value: "original" } },
+    }));
+
+    appended.summary = "mutated append result";
+    appended.metadata!.nested.value = "mutated append result";
+
+    const firstList = await listTeamReportEvents("team");
+    expect(firstList[0].summary).toBe("original");
+    expect(firstList[0].metadata).toEqual({ nested: { value: "original" } });
+
+    firstList[0].summary = "mutated list result";
+    firstList[0].metadata!.nested.value = "mutated list result";
+
+    const secondList = await listTeamReportEvents("team");
+    expect(secondList[0].summary).toBe("original");
+    expect(secondList[0].metadata).toEqual({ nested: { value: "original" } });
+  });
+
+  it("does not resort when inserting into an already sorted cache", async () => {
+    await appendTeamReportEvent("team", event({ id: "one", createdAt: 100, summary: "one" }));
+
+    const sortSpy = vi.spyOn(Array.prototype, "sort");
+    await appendTeamReportEvent("team", event({ id: "two", createdAt: 200, summary: "two" }));
+
+    expect(sortSpy).not.toHaveBeenCalled();
+    sortSpy.mockRestore();
+    await expect(listTeamReportEvents("team")).resolves.toMatchObject([{ id: "one" }, { id: "two" }]);
+  });
+
+  it("writes plain-text reports under the session and versions repeated agent names", async () => {
+    const reportFilesDir = path.join(root, "agent", "reports");
+    const getReportFilesDir = (paths as any).reportFilesDir;
+    if (typeof getReportFilesDir !== "function") throw new Error("reportFilesDir is not implemented");
+    vi.spyOn(paths as any, "reportFilesDir").mockReturnValue(reportFilesDir);
+
+    const first = await appendTeamReportEvent("team", event({ id: "one", createdAt: 100, report: "first report", summary: "First" }));
+    const second = await appendTeamReportEvent("team", event({ id: "two", createdAt: 200, report: "second report", summary: "Second" }));
+    const sessionDir = path.join(reportFilesDir, "team");
+
+    expect(path.dirname((first as any).reportPath)).toBe(sessionDir);
+    expect(path.basename((first as any).reportPath)).toBe("reader.md");
+    expect(path.basename((second as any).reportPath)).toBe("reader-v2.md");
+    expect(fs.readFileSync((first as any).reportPath, "utf-8")).toBe("first report");
+    expect(fs.readFileSync((second as any).reportPath, "utf-8")).toBe("second report");
+    await expect(listTeamReportEvents("team")).resolves.toMatchObject([
+      { id: "one", report: "first report", summary: "First", reportPath: (first as any).reportPath },
+      { id: "two", report: "second report", summary: "Second", reportPath: (second as any).reportPath },
+    ]);
+  });
+
+  it("returns the original event for duplicate ids", async () => {
+    const first = await appendTeamReportEvent("team", event({ id: "same", createdAt: 100, summary: "first" }));
+    const second = await appendTeamReportEvent("team", event({ id: "same", createdAt: 200, summary: "second" }));
+
+    expect(second).toEqual(first);
+    await expect(listTeamReportEvents("team")).resolves.toMatchObject([{ id: "same", summary: "first" }]);
+  });
+});

package/src/utils/report-events.ts

@@ -0,0 +1,222 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { TeamReportEvent } from "./models";
+import * as paths from "./paths";
+import { withLock } from "./lock";
+
+export type NewTeamReportEvent = Omit<TeamReportEvent, "id" | "teamName" | "createdAt"> & Partial<Pick<TeamReportEvent, "id" | "teamName" | "createdAt">>;
+
+export interface ListTeamReportEventsOptions {
+  since?: number;
+  agentName?: string;
+  limit?: number;
+}
+
+interface ReportEventsCache {
+  statKey: string;
+  events: TeamReportEvent[];
+  byId: Map<string, TeamReportEvent>;
+  byAgent: Map<string, TeamReportEvent[]>;
+}
+
+const MAX_REPORT_EVENTS_CACHE_ENTRIES = 128;
+
+const eventCache = new Map<string, ReportEventsCache>();
+
+function rememberReportEventsCache(p: string, cache: ReportEventsCache): ReportEventsCache {
+  if (eventCache.has(p)) eventCache.delete(p);
+  eventCache.set(p, cache);
+
+  while (eventCache.size > MAX_REPORT_EVENTS_CACHE_ENTRIES) {
+    const oldestKey = eventCache.keys().next().value;
+    if (oldestKey === undefined) break;
+    eventCache.delete(oldestKey);
+  }
+
+  return cache;
+}
+
+function getFreshReportEventsCache(p: string, statKey: string): ReportEventsCache | undefined {
+  const cached = eventCache.get(p);
+  if (!cached) return undefined;
+
+  if (cached.statKey !== statKey) {
+    eventCache.delete(p);
+    return undefined;
+  }
+
+  return rememberReportEventsCache(p, cached);
+}
+
+function ensureReportEventsFile(teamName: string): string {
+  const p = path.join(paths.teamDir(teamName), "reports.json");
+  const dir = path.dirname(p);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  return p;
+}
+
+function reportEventsStatKey(p: string): string {
+  if (!fs.existsSync(p)) return "missing";
+  const stat = fs.statSync(p, { bigint: true });
+  return `${stat.size}:${stat.mtimeNs}`;
+}
+
+function compareCreatedAt(a: TeamReportEvent, b: TeamReportEvent): number {
+  return a.createdAt - b.createdAt;
+}
+
+function cloneTeamReportEvent(event: TeamReportEvent): TeamReportEvent {
+  const cloned = { ...event };
+  if (event.metadata !== undefined) cloned.metadata = structuredClone(event.metadata);
+  return cloned;
+}
+
+function cloneTeamReportEvents(events: readonly TeamReportEvent[]): TeamReportEvent[] {
+  return events.map(cloneTeamReportEvent);
+}
+
+function buildCache(
+  p: string,
+  events: TeamReportEvent[],
+  statKey: string,
+  options: { sorted?: boolean } = {}
+): ReportEventsCache {
+  const sortedEvents = cloneTeamReportEvents(events);
+  if (!options.sorted) sortedEvents.sort(compareCreatedAt);
+
+  const byId = new Map<string, TeamReportEvent>();
+  const byAgent = new Map<string, TeamReportEvent[]>();
+
+  for (const event of sortedEvents) {
+    if (!byId.has(event.id)) byId.set(event.id, event);
+
+    const agentEvents = byAgent.get(event.agentName);
+    if (agentEvents) agentEvents.push(event);
+    else byAgent.set(event.agentName, [event]);
+  }
+
+  return rememberReportEventsCache(p, { statKey, events: sortedEvents, byId, byAgent });
+}
+
+function readEventsRaw(p: string): TeamReportEvent[] {
+  if (!fs.existsSync(p)) return [];
+  const parsed = JSON.parse(fs.readFileSync(p, "utf-8"));
+  return Array.isArray(parsed) ? parsed : [];
+}
+
+function readEventsCache(p: string): ReportEventsCache {
+  const statKey = reportEventsStatKey(p);
+  const cached = getFreshReportEventsCache(p, statKey);
+  if (cached) return cached;
+
+  const events = readEventsRaw(p);
+  return buildCache(p, events, reportEventsStatKey(p));
+}
+
+function writeEventsRaw(p: string, events: TeamReportEvent[], options: { sorted?: boolean } = {}): void {
+  fs.writeFileSync(p, JSON.stringify(events, null, 2));
+  buildCache(p, events, reportEventsStatKey(p), options);
+}
+
+function defaultEventId(teamName: string, event: NewTeamReportEvent): string {
+  if (event.operationId) {
+    return ["report", teamName, event.agentName, event.operationId, event.workflowRunId || ""].join(":");
+  }
+  return crypto.randomUUID();
+}
+
+function writeStandaloneReport(teamName: string, agentName: string, report: string): string {
+  const dir = path.join(paths.reportFilesDir(), paths.sanitizeName(teamName));
+  const baseName = paths.sanitizeName(agentName);
+  fs.mkdirSync(dir, { recursive: true });
+
+  for (let version = 1; ; version += 1) {
+    const suffix = version === 1 ? "" : `-v${version}`;
+    const reportPath = path.join(dir, `${baseName}${suffix}.md`);
+    try {
+      fs.writeFileSync(reportPath, report, { encoding: "utf-8", flag: "wx" });
+      return reportPath;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    }
+  }
+}
+
+function lowerBoundCreatedAt(events: TeamReportEvent[], since: number): number {
+  let low = 0;
+  let high = events.length;
+
+  while (low < high) {
+    const mid = Math.floor((low + high) / 2);
+    if (events[mid].createdAt < since) low = mid + 1;
+    else high = mid;
+  }
+
+  return low;
+}
+
+function upperBoundCreatedAt(events: TeamReportEvent[], createdAt: number): number {
+  let low = 0;
+  let high = events.length;
+
+  while (low < high) {
+    const mid = Math.floor((low + high) / 2);
+    if (events[mid].createdAt <= createdAt) low = mid + 1;
+    else high = mid;
+  }
+
+  return low;
+}
+
+function applyLimit(events: TeamReportEvent[], limit?: number): TeamReportEvent[] {
+  if (limit !== undefined && limit >= 0) return events.slice(Math.max(0, events.length - limit));
+  return events.slice();
+}
+
+function selectEvents(cache: ReportEventsCache, options: ListTeamReportEventsOptions): TeamReportEvent[] {
+  if (options.since !== undefined && Number.isNaN(options.since)) return [];
+
+  const source = options.agentName ? cache.byAgent.get(options.agentName) || [] : cache.events;
+  const start = options.since === undefined ? 0 : lowerBoundCreatedAt(source, options.since);
+  const filtered = start === 0 ? source : source.slice(start);
+  return cloneTeamReportEvents(applyLimit(filtered, options.limit));
+}
+
+function insertEvent(events: TeamReportEvent[], event: TeamReportEvent): TeamReportEvent[] {
+  const next = events.slice();
+  next.splice(upperBoundCreatedAt(next, event.createdAt), 0, event);
+  return next;
+}
+
+export async function appendTeamReportEvent(teamName: string, event: NewTeamReportEvent): Promise<TeamReportEvent> {
+  const p = ensureReportEventsFile(teamName);
+
+  return await withLock(p, async () => {
+    const cache = readEventsCache(p);
+    const normalized: TeamReportEvent = cloneTeamReportEvent({
+      ...event,
+      id: event.id || defaultEventId(teamName, event),
+      teamName,
+      createdAt: event.createdAt || Date.now(),
+    });
+
+    const existing = cache.byId.get(normalized.id);
+    if (existing) return cloneTeamReportEvent(existing);
+
+    normalized.reportPath = writeStandaloneReport(teamName, normalized.agentName, normalized.report);
+    writeEventsRaw(p, insertEvent(cache.events, normalized), { sorted: true });
+    return cloneTeamReportEvent(normalized);
+  });
+}
+
+export async function listTeamReportEvents(
+  teamName: string,
+  options: ListTeamReportEventsOptions = {}
+): Promise<TeamReportEvent[]> {
+  const p = ensureReportEventsFile(teamName);
+
+  return await withLock(p, async () => {
+    return selectEvents(readEventsCache(p), options);
+  });
+}

package/src/utils/runtime.test.ts

@@ -0,0 +1,314 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+  writeRuntimeStatus,
+  readRuntimeStatus,
+  deleteRuntimeStatus,
+  cleanupStaleRuntimeFiles,
+  createRuntimeError,
+  HEARTBEAT_STALE_MS,
+  STARTUP_STALL_MS,
+  RUNTIME_STALE_MS,
+} from "./runtime";
+import * as paths from "./paths";
+
+describe("runtime status", () => {
+  const teamName = `runtime-test-${Date.now()}`;
+  const agentName = "worker-1";
+  const runId = "worker-run";
+  let root: string;
+
+  function writeRoster(members: Array<{ name: string; lifecycleRunId: string; isActive?: boolean }>): void {
+    fs.writeFileSync(paths.configPath(teamName), JSON.stringify({ name: teamName, members }, null, 2));
+  }
+
+  beforeEach(() => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), "pi-extended-teams-runtime-"));
+    const teamPath = (teamName: string) => path.join(root, "teams", paths.sanitizeName(teamName));
+    vi.spyOn(paths, "teamDir").mockImplementation(teamPath);
+    vi.spyOn(paths, "configPath").mockImplementation(teamName => path.join(teamPath(teamName), "config.json"));
+    vi.spyOn(paths, "runtimeStatusPath").mockImplementation((teamName, agentName) => {
+      return path.join(teamPath(teamName), "runtime", `${paths.sanitizeName(agentName)}.json`);
+    });
+    vi.spyOn(paths, "lifecycleTombstonePath").mockImplementation((teamName, agentName) => {
+      return path.join(teamPath(teamName), "lifecycle", "quarantine", `${paths.sanitizeName(agentName)}.json`);
+    });
+    fs.mkdirSync(paths.teamDir(teamName), { recursive: true });
+    writeRoster([{ name: agentName, lifecycleRunId: runId, isActive: true }]);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    if (root && fs.existsSync(root)) fs.rmSync(root, { recursive: true, force: true });
+  });
+
+  it("writes and reads status", async () => {
+    await writeRuntimeStatus(teamName, agentName, runId, {
+      pid: 123,
+      startedAt: 1000,
+      ready: false,
+    });
+
+    const runtime = await readRuntimeStatus(teamName, agentName);
+    expect(runtime).not.toBeNull();
+    expect(runtime?.teamName).toBe(teamName);
+    expect(runtime?.agentName).toBe(agentName);
+    expect(runtime?.pid).toBe(123);
+    expect(runtime?.ready).toBe(false);
+  });
+
+  it("merges updates instead of overwriting status", async () => {
+    await writeRuntimeStatus(teamName, agentName, runId, {
+      pid: 123,
+      startedAt: 1000,
+      ready: false,
+      latestProgress: "Inspecting runtime state",
+      progressUpdatedAt: 1500,
+    });
+
+    await writeRuntimeStatus(teamName, agentName, runId, {
+      lastHeartbeatAt: 2000,
+      ready: true,
+      currentAction: "working",
+      activeToolName: "read",
+    });
+
+    const runtime = await readRuntimeStatus(teamName, agentName);
+    expect(runtime?.pid).toBe(123);
+    expect(runtime?.startedAt).toBe(1000);
+    expect(runtime?.lastHeartbeatAt).toBe(2000);
+    expect(runtime?.ready).toBe(true);
+    expect(runtime?.currentAction).toBe("working");
+    expect(runtime?.activeToolName).toBe("read");
+    expect(runtime?.latestProgress).toBe("Inspecting runtime state");
+    expect(runtime?.progressUpdatedAt).toBe(1500);
+  });
+
+  it("returns null when status does not exist", async () => {
+    const missing = await readRuntimeStatus(teamName, "missing-agent");
+    expect(missing).toBeNull();
+  });
+
+  it("stores status in team runtime directory", async () => {
+    await writeRuntimeStatus(teamName, agentName, runId, { ready: true });
+    const p = paths.runtimeStatusPath(teamName, agentName);
+    expect(path.basename(path.dirname(p))).toBe("runtime");
+    expect(fs.existsSync(p)).toBe(true);
+  });
+
+  describe("deleteRuntimeStatus", () => {
+    it("deletes existing runtime status", async () => {
+      await writeRuntimeStatus(teamName, agentName, runId, { ready: true });
+      const deleted = await deleteRuntimeStatus(teamName, agentName);
+      expect(deleted).toBe(true);
+
+      const runtime = await readRuntimeStatus(teamName, agentName);
+      expect(runtime).toBeNull();
+    });
+
+    it("returns false when status does not exist", async () => {
+      const deleted = await deleteRuntimeStatus(teamName, "nonexistent");
+      expect(deleted).toBe(false);
+    });
+
+    it("refuses wrong-run overwrite and deletion", async () => {
+      await writeRuntimeStatus(teamName, agentName, runId, { ready: true });
+
+      await expect(writeRuntimeStatus(teamName, agentName, "replacement-run", {
+        ready: false,
+      })).rejects.toThrow("not replacement-run");
+      await expect(deleteRuntimeStatus(teamName, agentName, "replacement-run")).resolves.toBe(false);
+      await expect(readRuntimeStatus(teamName, agentName)).resolves.toMatchObject({
+        lifecycleRunId: runId,
+        ready: true,
+      });
+      await expect(deleteRuntimeStatus(teamName, agentName, runId)).resolves.toBe(true);
+    });
+  });
+
+  describe("cleanupStaleRuntimeFiles", () => {
+    it("removes stale runtime files with old heartbeats", async () => {
+      const staleTime = Date.now() - RUNTIME_STALE_MS - 1000;
+      writeRoster([
+        { name: agentName, lifecycleRunId: runId, isActive: true },
+        { name: "stale-agent", lifecycleRunId: "stale-run", isActive: true },
+      ]);
+      await writeRuntimeStatus(teamName, "stale-agent", "stale-run", {
+        startedAt: staleTime,
+        lastHeartbeatAt: staleTime,
+        ready: true,
+      });
+
+      const cleaned = await cleanupStaleRuntimeFiles(teamName);
+      expect(cleaned).toBe(1);
+
+      const runtime = await readRuntimeStatus(teamName, "stale-agent");
+      expect(runtime).toBeNull();
+    });
+
+    it("preserves runtime files with recent heartbeats", async () => {
+      await writeRuntimeStatus(teamName, agentName, runId, {
+        startedAt: Date.now() - RUNTIME_STALE_MS - 1000,
+        lastHeartbeatAt: Date.now(),
+        ready: true,
+      });
+
+      const cleaned = await cleanupStaleRuntimeFiles(teamName);
+      expect(cleaned).toBe(0);
+
+      const runtime = await readRuntimeStatus(teamName, agentName);
+      expect(runtime).not.toBeNull();
+    });
+
+    it("removes corrupted files while holding the lifecycle and runtime locks", async () => {
+      const runtimeDir = path.join(paths.teamDir(teamName), "runtime");
+      if (!fs.existsSync(runtimeDir)) fs.mkdirSync(runtimeDir, { recursive: true });
+      fs.writeFileSync(path.join(runtimeDir, "corrupted.json"), "not valid json");
+
+      const cleaned = await cleanupStaleRuntimeFiles(teamName);
+      expect(cleaned).toBe(1);
+    });
+
+    it("re-reads after taking the lifecycle lock and preserves a concurrent fresh replacement", async () => {
+      const staleTime = Date.now() - RUNTIME_STALE_MS - 1000;
+      await writeRuntimeStatus(teamName, agentName, runId, {
+        startedAt: staleTime,
+        lastHeartbeatAt: staleTime,
+        ready: true,
+      });
+      let releaseLifecycleBarrier!: () => void;
+      const lifecycleBarrier = new Promise<void>(resolve => { releaseLifecycleBarrier = resolve; });
+      let notifyLifecycleLocked!: () => void;
+      const lifecycleLocked = new Promise<void>(resolve => { notifyLifecycleLocked = resolve; });
+
+      const cleanup = cleanupStaleRuntimeFiles(teamName, Date.now(), {
+        afterLifecycleLock: async name => {
+          if (name !== agentName) return;
+          notifyLifecycleLocked();
+          await lifecycleBarrier;
+        },
+      });
+      await lifecycleLocked;
+      fs.writeFileSync(paths.runtimeStatusPath(teamName, agentName), JSON.stringify({
+        teamName,
+        agentName,
+        lifecycleRunId: runId,
+        startedAt: staleTime,
+        lastHeartbeatAt: Date.now(),
+        ready: true,
+      }, null, 2));
+      releaseLifecycleBarrier();
+
+      await expect(cleanup).resolves.toBe(0);
+      await expect(readRuntimeStatus(teamName, agentName)).resolves.toMatchObject({
+        lifecycleRunId: runId,
+        ready: true,
+      });
+    });
+
+    it("serializes a fresh heartbeat behind stale deletion without losing the new runtime", async () => {
+      const staleTime = Date.now() - RUNTIME_STALE_MS - 1000;
+      await writeRuntimeStatus(teamName, agentName, runId, {
+        startedAt: staleTime,
+        lastHeartbeatAt: staleTime,
+        ready: true,
+      });
+      let releaseDeleteBarrier!: () => void;
+      const deleteBarrier = new Promise<void>(resolve => { releaseDeleteBarrier = resolve; });
+      let notifyBeforeDelete!: () => void;
+      const beforeDelete = new Promise<void>(resolve => { notifyBeforeDelete = resolve; });
+
+      const cleanup = cleanupStaleRuntimeFiles(teamName, Date.now(), {
+        beforeDelete: async name => {
+          if (name !== agentName) return;
+          notifyBeforeDelete();
+          await deleteBarrier;
+        },
+      });
+      await beforeDelete;
+      let heartbeatSettled = false;
+      const heartbeat = writeRuntimeStatus(teamName, agentName, runId, {
+        lastHeartbeatAt: Date.now(),
+        ready: true,
+      }).then(result => {
+        heartbeatSettled = true;
+        return result;
+      });
+      await Promise.resolve();
+      expect(heartbeatSettled).toBe(false);
+      releaseDeleteBarrier();
+
+      await expect(cleanup).resolves.toBe(1);
+      await expect(heartbeat).resolves.toMatchObject({ lifecycleRunId: runId, ready: true });
+      await expect(readRuntimeStatus(teamName, agentName)).resolves.toMatchObject({
+        lifecycleRunId: runId,
+        ready: true,
+      });
+    });
+
+    it("rejects writes while an occupied or corrupt tombstone fences the active roster member", async () => {
+      const tombstonePath = paths.lifecycleTombstonePath(teamName, agentName);
+      fs.mkdirSync(path.dirname(tombstonePath), { recursive: true });
+      fs.writeFileSync(tombstonePath, JSON.stringify({
+        version: 1,
+        team: teamName,
+        agent: agentName,
+        runId,
+        role: "write",
+        reason: "quit",
+        phase: "closing",
+        ownerPid: process.pid,
+        extensionInstanceId: "test-instance",
+        timestamps: { createdAt: Date.now(), updatedAt: Date.now() },
+      }));
+      await expect(writeRuntimeStatus(teamName, agentName, runId, { ready: false }))
+        .rejects.toThrow("lifecycle run worker-run is closing");
+
+      fs.writeFileSync(tombstonePath, "not json");
+      await expect(writeRuntimeStatus(teamName, agentName, runId, { ready: false }))
+        .rejects.toThrow("lifecycle tombstone is corrupt");
+    });
+
+    it("returns 0 when no runtime directory exists", async () => {
+      const cleaned = await cleanupStaleRuntimeFiles("nonexistent-team");
+      expect(cleaned).toBe(0);
+    });
+  });
+
+  describe("createRuntimeError", () => {
+    it("creates structured error from Error object", () => {
+      const error = new Error("Test error");
+      const runtimeError = createRuntimeError(error);
+      expect(runtimeError.message).toBe("Test error");
+      expect(runtimeError.timestamp).toBeGreaterThan(0);
+    });
+
+    it("creates structured error from string", () => {
+      const runtimeError = createRuntimeError("String error");
+      expect(runtimeError.message).toBe("String error");
+      expect(runtimeError.timestamp).toBeGreaterThan(0);
+    });
+
+    it("creates structured error from unknown type", () => {
+      const runtimeError = createRuntimeError({ weird: "object" });
+      expect(runtimeError.message).toBe("[object Object]");
+      expect(runtimeError.timestamp).toBeGreaterThan(0);
+    });
+  });
+
+  describe("constants", () => {
+    it("exports HEARTBEAT_STALE_MS with correct value", () => {
+      expect(HEARTBEAT_STALE_MS).toBe(90000);
+    });
+
+    it("exports STARTUP_STALL_MS with correct value", () => {
+      expect(STARTUP_STALL_MS).toBe(60000);
+    });
+
+    it("exports RUNTIME_STALE_MS with correct value", () => {
+      expect(RUNTIME_STALE_MS).toBe(300000);
+    });
+  });
+});