pi-extended-teams 2.2.6 → 2.2.8

package/src/orchestration/index.ts

@@ -6,6 +6,8 @@ import * as runtime from "../utils/runtime";
 import * as writeQueue from "../utils/write-queue";
 import * as reports from "../utils/report-events";
 import type { Member } from "../utils/models";
+import { readLifecycleTombstone } from "../utils/lifecycle-tombstone";
+import { isAgentActivity, projectAgentStatus } from "./status-projection";
 import { canonicalPersistedModelSlot, loadSettings, requireFavoriteModelLevel, roleForFavoriteModelSlot } from "../utils/settings";
 import type {
   BroadcastMessageOnceRequest,
@@ -19,7 +21,6 @@ import type {
   SpawnTeammatesOnceResponse,
   TeamObservation,
   TeammateResolutionDetails,
-  TeammateHealth,
   TeammateObservation,
 } from "./types";
 
@@ -41,11 +42,6 @@ export async function appendTeamReportEvent(teamName: string, event: reports.New
   return await next;
 }
 
-function readAgentIsKnownRunning(teamName: string, agentName: string, options: ObserveRuntimeOptions): boolean {
-  const key = options.readAgentKey?.(teamName, agentName) || `${teamName}:${agentName}`;
-  return !!options.runningReadAgents?.has(key);
-}
-
 function operationValue(source: { operationId?: string; workflowRunId?: string; metadata?: Record<string, any> }, key: "operationId" | "workflowRunId"): string | undefined {
   return source[key] || source.metadata?.[key] || source.metadata?.orchestration?.[key];
 }
@@ -268,40 +264,32 @@ async function observeKnownTeammate(
   const unreadCount = (await messaging.peekInbox(teamName, member.name, true).catch(() => [])).length;
   const runtimeStatus = member.name === "team-lead" ? null : await runtime.readRuntimeStatus(teamName, member.name).catch(() => null);
   const now = options.now ?? Date.now();
-  const hasRecentHeartbeat = runtime.isHeartbeatFresh(runtimeStatus, now);
-
-  let alive: boolean | null;
-  if (member.name === "team-lead") {
-    alive = true;
-  } else if (role === "read") {
-    alive = readAgentIsKnownRunning(teamName, member.name, options) || (!!runtimeStatus && hasRecentHeartbeat && member.isActive !== false);
-  } else if (member.tmuxPaneId && options.terminal?.isAlive) {
-    alive = options.terminal.isAlive(member.tmuxPaneId);
-  } else {
-    alive = null;
-  }
-
-  const startupStalled = alive === true && unreadCount > 0 && (now - member.joinedAt) > runtime.STARTUP_STALL_MS && !(runtimeStatus?.ready);
-  let health: TeammateHealth;
-  if (member.name === "team-lead") health = "lead";
-  else if (alive === null) health = "unknown";
-  else if (!alive) health = "dead";
-  else if (startupStalled) health = "stalled";
-  else if (runtimeStatus?.ready) health = hasRecentHeartbeat ? "healthy" : "idle";
-  else health = "starting";
+  const key = options.readAgentKey?.(teamName, member.name) || `${teamName}:${member.name}`;
+  const candidate = options.runningReadAgents?.get(key);
+  const fence = member.name === "team-lead" ? { status: "absent" as const }
+    : await readLifecycleTombstone(teamName, member.name).catch(error => ({
+      status: "corrupt" as const, error: error instanceof Error ? error.message : String(error),
+    }));
+  const projected = projectAgentStatus({
+    member, activity: isAgentActivity(candidate) ? candidate : undefined,
+    runtime: runtimeStatus, fence, now, unreadCount,
+    terminalAlive: member.tmuxPaneId && options.terminal?.isAlive ? options.terminal.isAlive(member.tmuxPaneId) : null,
+  });
 
   return {
     teamName,
     agentName: member.name,
     member,
     role,
-    alive,
-    health,
+    alive: member.name === "team-lead" ? true : projected.alive,
+    health: member.name === "team-lead" ? "lead" : projected.health,
+    phase: member.name === "team-lead" ? "lead" : projected.phase,
+    error: projected.error,
     unreadCount,
-    agentLoopReady: !!runtimeStatus?.ready,
-    hasRecentHeartbeat,
-    startupStalled,
-    runtime: runtimeStatus,
+    agentLoopReady: projected.agentLoopReady,
+    hasRecentHeartbeat: projected.hasRecentHeartbeat,
+    startupStalled: projected.startupStalled,
+    runtime: projected.runtime,
   };
 }
 

package/src/orchestration/status-projection.ts

@@ -0,0 +1,89 @@
+import type { Member } from "../utils/models";
+import type { LifecycleTombstoneReadResult } from "../utils/lifecycle-tombstone";
+import { isHeartbeatFresh, STARTUP_STALL_MS, type AgentRuntimeStatus } from "../utils/runtime";
+
+export const AGENT_HANGING_MS = 15 * 60_000;
+
+export type ActiveAgentPhase = "starting" | "thinking" | "working" | "finishing"
+  | "stalled" | "stopping" | "quarantined" | "persistence-failed";
+
+export interface AgentActivity {
+  runId: string;
+  status: "starting" | "thinking" | "working" | "finishing";
+  startedAt: number;
+  role?: string;
+  lastActivityAt?: number;
+  teardownState?: string;
+  handoffDetached?: boolean;
+  latestProgress?: string;
+  progressUpdatedAt?: number;
+  activeToolName?: string;
+  lastError?: { message: string };
+}
+
+export function isAgentActivity(value: unknown): value is AgentActivity {
+  return typeof value === "object" && value !== null
+    && "runId" in value && typeof value.runId === "string"
+    && "startedAt" in value && typeof value.startedAt === "number"
+    && "status" in value && typeof value.status === "string"
+    && ["starting", "thinking", "working", "finishing"].includes(value.status);
+}
+
+function lifecyclePhase(member: Member, fence: LifecycleTombstoneReadResult): { phase: ActiveAgentPhase; error?: string } | undefined {
+  if (fence.status === "absent") return;
+  if (fence.status === "corrupt") return { phase: "quarantined", error: fence.error };
+  const { tombstone } = fence;
+  const mismatch = member.lifecycleRunId && tombstone.runId !== member.lifecycleRunId
+    ? `Lifecycle fence belongs to run ${tombstone.runId}, not roster run ${member.lifecycleRunId}.`
+    : undefined;
+  return {
+    phase: tombstone.phase === "cleanup_failed" || tombstone.phase === "timed_out" ? "quarantined" : "stopping",
+    error: mismatch || tombstone.error,
+  };
+}
+
+export function projectAgentStatus(input: {
+  member: Member;
+  activity?: AgentActivity;
+  runtime: AgentRuntimeStatus | null;
+  fence: LifecycleTombstoneReadResult;
+  terminalAlive: boolean | null;
+  now: number;
+  unreadCount?: number;
+}) {
+  const { member, now } = input;
+  const state = input.activity?.runId === member.lifecycleRunId ? input.activity : undefined;
+  const runtime = input.runtime?.lifecycleRunId === member.lifecycleRunId ? input.runtime : null;
+  const persisted = lifecyclePhase(member, input.fence);
+  const hasRecentHeartbeat = isHeartbeatFresh(runtime, now);
+  const inProcess = !!state && state.teardownState !== "finalized" && !state.handoffDetached;
+  const externalAlive = member.isActive !== false && (hasRecentHeartbeat || input.terminalAlive === true);
+  const alive = inProcess || externalAlive ? true
+    : member.isActive === false || input.terminalAlive === false || member.role === "read" ? false : null;
+  const startupStalled = alive === true && (input.unreadCount ?? 0) > 0
+    && now - member.joinedAt > STARTUP_STALL_MS && !runtime?.ready;
+  const teardownPhase = state?.teardownState === "persistence_failed" ? "persistence-failed"
+    : state?.teardownState === "quarantined" ? "quarantined"
+      : state?.teardownState === "stopping" ? "stopping" : undefined;
+  const fencedPhase = teardownPhase ?? persisted?.phase;
+  let phase: ActiveAgentPhase;
+  if (fencedPhase) phase = fencedPhase;
+  else if (inProcess) phase = now - (state.lastActivityAt || state.startedAt) >= AGENT_HANGING_MS ? "stalled" : state.status;
+  else if (!externalAlive) phase = member.isActive !== false && !runtime?.ready && now - member.joinedAt <= STARTUP_STALL_MS ? "starting" : "stalled";
+  else phase = runtime?.currentAction === "done" ? "finishing" : runtime?.currentAction ?? (runtime?.ready ? "working" : "starting");
+
+  const health = fencedPhase === "quarantined" ? "quarantined"
+    : fencedPhase === "persistence-failed" ? "persistence-failed"
+      : fencedPhase ? "stopping"
+        : startupStalled || (phase === "stalled" && alive) ? "stalled"
+          : alive === null ? "unknown"
+            : !alive ? "dead"
+              : runtime?.ready ? hasRecentHeartbeat ? "healthy" : "idle"
+                : inProcess && state.status !== "starting" ? "healthy" : "starting";
+  return {
+    phase, alive, health, hasRecentHeartbeat, startupStalled,
+    agentLoopReady: !fencedPhase && alive === true && (!!runtime?.ready || (inProcess && state.status !== "starting")),
+    state, runtime,
+    error: persisted?.error || state?.lastError?.message || runtime?.lastError?.message,
+  };
+}

package/src/orchestration/types.ts

@@ -1,4 +1,5 @@
 import type { AgentRuntimeStatus } from "../utils/runtime";
+import type { ActiveAgentPhase, projectAgentStatus } from "./status-projection";
 import type { QueuedWriteSpawn } from "../utils/write-queue";
 import type { FileClaim } from "../utils/claims";
 import type { InboxMessage, Member, TaskFile, TeamConfig, TeamReportEvent, ThinkingLevel } from "../utils/models";
@@ -12,7 +13,7 @@ export interface OrchestrationOperationMetadata {
   metadata?: Record<string, any>;
 }
 
-export type TeammateHealth = "lead" | "healthy" | "idle" | "starting" | "stalled" | "dead" | "unknown";
+export type TeammateHealth = "lead" | ReturnType<typeof projectAgentStatus>["health"];
 
 export interface TeammateObservation {
   teamName: string;
@@ -21,6 +22,8 @@ export interface TeammateObservation {
   role: string;
   alive: boolean | null;
   health: TeammateHealth;
+  phase?: ActiveAgentPhase | "lead";
+  error?: string;
   unreadCount: number;
   agentLoopReady: boolean;
   hasRecentHeartbeat: boolean;

package/src/utils/models.ts

@@ -10,6 +10,7 @@ export interface Member {
   model?: string;
   joinedAt: number;
   tmuxPaneId: string;
+  herdrPaneId?: string;
   windowId?: string;
   cwd: string;
   subscriptions: any[];
@@ -33,7 +34,7 @@ export interface Member {
   parentAgentName?: string;
   /** Runtime-owned parent lifecycle identity for a restricted nested read child. */
   parentLifecycleRunId?: string;
-  /** "read" agents run in-process (no pane); "write" agents spawn in tmux. */
+  /** Tool role; execution may be in-process or in a terminal pane. */
   role?: "read" | "write";
   /** Optional category preset name from settings.json. */
   category?: string;