pi-extended-teams 2.4.2 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/reference.md +18 -0
- package/extensions/agents/read-agent.ts +2 -0
- package/extensions/events/register-events.ts +29 -0
- package/extensions/index.ts +8 -0
- package/extensions/runtime/agent-activity.ts +27 -0
- package/extensions/runtime/types.ts +2 -0
- package/extensions/team/lifecycle.ts +121 -10
- package/extensions/ui/agent-follow-view.ts +4 -17
- package/package.json +1 -1
- package/src/utils/runtime.ts +3 -1
- package/src/utils/settings.ts +9 -1
package/docs/reference.md
CHANGED
|
@@ -104,6 +104,24 @@ Public spawns respect each role's concurrency limit. Enabled overflow queues acc
|
|
|
104
104
|
|
|
105
105
|
Spawned sessions are private by default under `~/.pi/teams/<team>/agent-sessions/`, outside Pi's normal `/resume` picker.
|
|
106
106
|
|
|
107
|
+
### Idle watchdog
|
|
108
|
+
|
|
109
|
+
The runtime checks agents every 30 seconds. After five idle minutes it sends the lead a visible Pi message with the agent name, team, run ID, last activity time, and assignment excerpt. After ten idle minutes it requests normal shutdown and reports whether cleanup finished or remains pending. A late cleanup result sends a confirmation or the failure details.
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{
|
|
113
|
+
"watchdog": {
|
|
114
|
+
"idleWarningMinutes": 5,
|
|
115
|
+
"idleStopMinutes": 10,
|
|
116
|
+
"idleAutoStop": true
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Both thresholds accept positive finite numbers in minutes. Settings are read on each check; project values override global values. Set `idleAutoStop` to `false` to keep warnings without automatic idle cancellation. Heartbeat-failure cleanup is separate and unchanged except that tracked active work is protected.
|
|
122
|
+
|
|
123
|
+
Messages, streamed text or reasoning, tool activity, and progress updates reset the idle timer. Heartbeats alone do not. Active tools (including silent long-running commands), checks, compaction, and parents with accepted queued or running children are protected. Queued, starting, and finishing agents are excluded. Warnings occur once per idle period. Terminal agents need current activity telemetry; missing telemetry does not authorize an idle stop.
|
|
124
|
+
|
|
107
125
|
## Task outcomes and full reports
|
|
108
126
|
|
|
109
127
|
A completed agent, a successful task, passed verification, and lead acceptance are separate facts.
|
|
@@ -4,6 +4,7 @@ import path from "node:path";
|
|
|
4
4
|
import os from "node:os";
|
|
5
5
|
import { buildPiCommand, getPiLaunchCommand, shellQuote } from "../internal/pi-command";
|
|
6
6
|
import { herdrCommand } from "../runtime/herdr";
|
|
7
|
+
import { recordAgentActivity } from "../runtime/agent-activity";
|
|
7
8
|
import * as runtime from "../../src/utils/runtime";
|
|
8
9
|
import * as teams from "../../src/utils/teams";
|
|
9
10
|
import * as messaging from "../../src/utils/messaging";
|
|
@@ -327,6 +328,7 @@ export function handleReadAgentSessionEvent(
|
|
|
327
328
|
renderReadAgentStatus: () => void
|
|
328
329
|
): void {
|
|
329
330
|
const eventType = event.type;
|
|
331
|
+
recordAgentActivity(state, event);
|
|
330
332
|
if (eventType === "agent_start" || eventType === "turn_start") {
|
|
331
333
|
markReadAgentActivity(state, "thinking", "thinking");
|
|
332
334
|
}
|
|
@@ -5,6 +5,7 @@ import * as paths from "../../src/utils/paths";
|
|
|
5
5
|
import * as runtime from "../../src/utils/runtime";
|
|
6
6
|
import * as messaging from "../../src/utils/messaging";
|
|
7
7
|
import * as teams from "../../src/utils/teams";
|
|
8
|
+
import { recordAgentActivity, type AgentActivity } from "../runtime/agent-activity";
|
|
8
9
|
import { cleanupAgentSessionFolders, cleanupOrphanedTeams } from "../internal/session-files";
|
|
9
10
|
import { summarizeSessionUsage } from "../internal/session-usage";
|
|
10
11
|
import { formatElapsed, formatTokenCount } from "../ui/renderers";
|
|
@@ -76,6 +77,8 @@ export function registerExtensionEvents(pi: any, options: RegisterEventsOptions)
|
|
|
76
77
|
let teammateInboxDisposed = false;
|
|
77
78
|
let teammateLifecycleRunId: string | undefined = process.env.PI_LIFECYCLE_RUN_ID;
|
|
78
79
|
const teammateOneShotTimers = new Set<NodeJS.Timeout>();
|
|
80
|
+
const activity: AgentActivity = {};
|
|
81
|
+
let activityPublishedAt = 0;
|
|
79
82
|
|
|
80
83
|
const scheduleTeammateOneShot = (callback: () => void, delayMs: number) => {
|
|
81
84
|
const timer = setTimeout(() => {
|
|
@@ -127,6 +130,8 @@ export function registerExtensionEvents(pi: any, options: RegisterEventsOptions)
|
|
|
127
130
|
}
|
|
128
131
|
return {
|
|
129
132
|
...updates,
|
|
133
|
+
lastActivityAt: activity.lastActivityAt,
|
|
134
|
+
activeWorkCount: activity.activeWork?.size ?? 0,
|
|
130
135
|
...(typeof usage.tokensUsed === "number" ? { tokensUsed: usage.tokensUsed } : {}),
|
|
131
136
|
contextUsage,
|
|
132
137
|
};
|
|
@@ -149,6 +154,24 @@ export function registerExtensionEvents(pi: any, options: RegisterEventsOptions)
|
|
|
149
154
|
|
|
150
155
|
registerAgentReportRenderer(pi);
|
|
151
156
|
|
|
157
|
+
if (options.isTeammate) {
|
|
158
|
+
for (const type of ["agent_start", "agent_end", "turn_start", "turn_end", "message_start", "message_update", "message_end",
|
|
159
|
+
"tool_execution_start", "tool_execution_update", "tool_execution_end", "session_before_compact", "session_compact",
|
|
160
|
+
"session_compact_failed", "ui_prompt_start", "ui_prompt_end"]) {
|
|
161
|
+
pi.on(type, async (event: any) => {
|
|
162
|
+
recordAgentActivity(activity, { ...event, type });
|
|
163
|
+
const teamName = options.getTeamName();
|
|
164
|
+
if (!teamName || !teammateLifecycleRunId || teammateInboxDisposed) return;
|
|
165
|
+
// Stream events can arrive per token; persist at most once a second.
|
|
166
|
+
if ((type === "message_update" || type === "tool_execution_update") && Date.now() - activityPublishedAt < 1000) return;
|
|
167
|
+
activityPublishedAt = Date.now();
|
|
168
|
+
await writeTeammateRuntimeStatus(teamName, {
|
|
169
|
+
lastActivityAt: activity.lastActivityAt, activeWorkCount: activity.activeWork?.size ?? 0,
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
152
175
|
pi.on("session_start", async (_event: any, ctx: any) => {
|
|
153
176
|
paths.ensureDirs();
|
|
154
177
|
// Local-only janitor pass: no model calls, no agent wakes. This prevents
|
|
@@ -194,6 +217,8 @@ export function registerExtensionEvents(pi: any, options: RegisterEventsOptions)
|
|
|
194
217
|
if (options.isTeammate) {
|
|
195
218
|
disposeTeammateInbox();
|
|
196
219
|
teammateInboxDisposed = false;
|
|
220
|
+
activity.lastActivityAt = Date.now();
|
|
221
|
+
activity.activeWork = new Set();
|
|
197
222
|
if (teamName) {
|
|
198
223
|
if (teams.teamExists(teamName)) {
|
|
199
224
|
const persistedRunId = await teams.ensureMemberLifecycleRunId(teamName, options.agentName, teammateLifecycleRunId);
|
|
@@ -213,6 +238,8 @@ export function registerExtensionEvents(pi: any, options: RegisterEventsOptions)
|
|
|
213
238
|
pid: process.pid,
|
|
214
239
|
startedAt: Date.now(),
|
|
215
240
|
lastHeartbeatAt: Date.now(),
|
|
241
|
+
lastActivityAt: activity.lastActivityAt,
|
|
242
|
+
activeWorkCount: 0,
|
|
216
243
|
ready: false,
|
|
217
244
|
currentAction: "starting",
|
|
218
245
|
activeToolName: undefined,
|
|
@@ -263,6 +290,8 @@ export function registerExtensionEvents(pi: any, options: RegisterEventsOptions)
|
|
|
263
290
|
if (teammateInboxDisposed) return;
|
|
264
291
|
await writeTeammateRuntimeStatus(teamName, {
|
|
265
292
|
lastHeartbeatAt: Date.now(),
|
|
293
|
+
lastActivityAt: activity.lastActivityAt,
|
|
294
|
+
activeWorkCount: activity.activeWork?.size ?? 0,
|
|
266
295
|
});
|
|
267
296
|
if (unread.length > 0) {
|
|
268
297
|
const content = `You have ${unread.length} new inbox message(s). Read them with read_inbox and act.`;
|
package/extensions/index.ts
CHANGED
|
@@ -139,6 +139,14 @@ function createExecutionRuntime(pi: ExtensionAPI, host?: ReturnType<typeof creat
|
|
|
139
139
|
getSessionCwd: () => sessionCtx?.cwd,
|
|
140
140
|
getTeamName: () => teamName,
|
|
141
141
|
extensionInstanceId,
|
|
142
|
+
notifyIdleAgent: (content) => pi.sendMessage({
|
|
143
|
+
customType: "pi-extended-teams-idle", content, display: true,
|
|
144
|
+
}, { triggerTurn: true, deliverAs: "followUp" }),
|
|
145
|
+
hasPendingChildren: (targetTeamName, member) => !!member.lifecycleRunId && pendingChildController.hasPendingChildren({
|
|
146
|
+
teamName: targetTeamName,
|
|
147
|
+
parentName: member.name,
|
|
148
|
+
parentRunId: member.lifecycleRunId,
|
|
149
|
+
}),
|
|
142
150
|
onWriterInactive: (targetTeamName, member) => {
|
|
143
151
|
releaseWriteAgentSleepAssertion(targetTeamName, member);
|
|
144
152
|
removeWriterScreenTab(writerScreenState, { teamName: targetTeamName, name: member.name, paneId: member.tmuxPaneId });
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** Actual work evidence, separate from a runtime's periodic liveness heartbeat. */
|
|
2
|
+
export interface AgentActivity {
|
|
3
|
+
lastActivityAt?: number;
|
|
4
|
+
activeWork?: Set<string>;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
const workStarts: Record<string, string> = {
|
|
8
|
+
compaction_start: "compaction", auto_compaction_start: "compaction", session_before_compact: "compaction",
|
|
9
|
+
auto_retry_start: "retry", ui_prompt_start: "user-prompt",
|
|
10
|
+
};
|
|
11
|
+
const workEnds: Record<string, string> = {
|
|
12
|
+
compaction_end: "compaction", auto_compaction_end: "compaction", session_compact: "compaction",
|
|
13
|
+
session_compact_failed: "compaction", auto_retry_end: "retry", ui_prompt_end: "user-prompt",
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export function recordAgentActivity(
|
|
17
|
+
state: AgentActivity,
|
|
18
|
+
event: { type: string; toolCallId?: string; toolName?: string; assistantMessageEvent?: unknown },
|
|
19
|
+
now = Date.now(),
|
|
20
|
+
): void {
|
|
21
|
+
state.lastActivityAt = now;
|
|
22
|
+
const work = state.activeWork ??= new Set();
|
|
23
|
+
if (event.type === "tool_execution_start") work.add(`tool:${event.toolCallId ?? event.toolName}`);
|
|
24
|
+
if (event.type === "tool_execution_end") work.delete(`tool:${event.toolCallId ?? event.toolName}`);
|
|
25
|
+
if (workStarts[event.type]) work.add(workStarts[event.type]);
|
|
26
|
+
if (workEnds[event.type]) work.delete(workEnds[event.type]);
|
|
27
|
+
}
|
|
@@ -13,6 +13,8 @@ export interface RunningReadAgent extends ManagedReadAgentLifecycleState {
|
|
|
13
13
|
status: "starting" | "thinking" | "working" | "finishing";
|
|
14
14
|
recentEvents: string[];
|
|
15
15
|
lastActivityAt: number;
|
|
16
|
+
/** Exact tool calls and other work spans that must not be idle-stopped. */
|
|
17
|
+
activeWork?: Set<string>;
|
|
16
18
|
activeToolName?: string;
|
|
17
19
|
/** Monotonic identity for the current nested AgentSession turn. */
|
|
18
20
|
operationGeneration?: number;
|
|
@@ -10,7 +10,7 @@ import { createReportResult } from "../../src/results/report-result";
|
|
|
10
10
|
import { VerificationController } from "../../src/results/verification-controller";
|
|
11
11
|
import { CompletionGroup } from "../../src/results/completion-group";
|
|
12
12
|
import { enqueueCompletionGroupDeliveries } from "../../src/results/completion-group-delivery";
|
|
13
|
-
import { loadSettings } from "../../src/utils/settings";
|
|
13
|
+
import { loadSettings, type WatchdogConfig } from "../../src/utils/settings";
|
|
14
14
|
import type { Member } from "../../src/utils/models";
|
|
15
15
|
import type { RunningReadAgent } from "../runtime/types";
|
|
16
16
|
import {
|
|
@@ -46,12 +46,16 @@ export interface LifecycleRuntimeOptions {
|
|
|
46
46
|
onTeammateClosing?(teamName: string, member: Member): void;
|
|
47
47
|
onTeammateSettled?(teamName: string, member: Member): void;
|
|
48
48
|
extensionInstanceId?: string;
|
|
49
|
+
notifyIdleAgent?(content: string): void | Promise<void>;
|
|
50
|
+
hasPendingChildren?(teamName: string, member: Member): boolean;
|
|
49
51
|
}
|
|
50
52
|
|
|
51
53
|
export interface ShutdownTeammateOptions {
|
|
52
54
|
drainQueue?: boolean;
|
|
53
55
|
removeMember?: boolean;
|
|
54
56
|
reason?: unknown;
|
|
57
|
+
/** Revalidate automatic cancellation after asynchronous lifecycle reads. */
|
|
58
|
+
shouldStop?(): boolean;
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
interface ReapedTeammate {
|
|
@@ -68,6 +72,8 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
68
72
|
let leadWatchdogTimer: NodeJS.Timeout | null = null;
|
|
69
73
|
let leadWatchdogGeneration = 0;
|
|
70
74
|
const extensionInstanceId = options.extensionInstanceId ?? generateExtensionInstanceId();
|
|
75
|
+
const idleObservations = new Map<string, { activityAt: number; since: number; warned: boolean; attempted: boolean }>();
|
|
76
|
+
let watchdogInFlight = false;
|
|
71
77
|
|
|
72
78
|
async function finalizeTeammateRuntime(teamName: string, member: Member, expectedRunId: string): Promise<void> {
|
|
73
79
|
const pidFile = path.join(paths.teamDir(teamName), `${member.name}.pid`);
|
|
@@ -161,13 +167,6 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
161
167
|
}
|
|
162
168
|
member.lifecycleRunId = expectedRunId;
|
|
163
169
|
}
|
|
164
|
-
try {
|
|
165
|
-
options.onTeammateClosing?.(teamName, { ...member, lifecycleRunId: expectedRunId });
|
|
166
|
-
} catch {
|
|
167
|
-
// Lifecycle cancellation hooks are in-memory notifications only. Persisted
|
|
168
|
-
// teardown ownership remains authoritative if an observer fails.
|
|
169
|
-
}
|
|
170
|
-
|
|
171
170
|
const persistedRuntime = await runtime.readRuntimeStatus(teamName, member.name).catch(() => null);
|
|
172
171
|
if (persistedRuntime?.lifecycleRunId && persistedRuntime.lifecycleRunId !== expectedRunId) {
|
|
173
172
|
return boundedQuarantineResult(
|
|
@@ -388,6 +387,15 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
388
387
|
});
|
|
389
388
|
};
|
|
390
389
|
|
|
390
|
+
if (shutdownOptions.shouldStop && !shutdownOptions.shouldStop()) {
|
|
391
|
+
return { ...baseResult(), finalized: false, error: "Idle cancellation skipped: activity or ownership changed." };
|
|
392
|
+
}
|
|
393
|
+
try {
|
|
394
|
+
options.onTeammateClosing?.(teamName, { ...member, lifecycleRunId: expectedRunId });
|
|
395
|
+
} catch {
|
|
396
|
+
// Lifecycle cancellation hooks are in-memory notifications only. Persisted
|
|
397
|
+
// teardown ownership remains authoritative if an observer fails.
|
|
398
|
+
}
|
|
391
399
|
if (expectedState) {
|
|
392
400
|
return requestReadAgentTeardown(expectedState, {
|
|
393
401
|
reason,
|
|
@@ -461,12 +469,104 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
461
469
|
)));
|
|
462
470
|
}
|
|
463
471
|
|
|
472
|
+
async function notifyIdleAgent(teamName: string, content: string): Promise<void> {
|
|
473
|
+
if (options.notifyIdleAgent) await options.notifyIdleAgent(content);
|
|
474
|
+
else await messaging.sendPlainMessage(teamName, "watchdog", "team-lead", content, "Agent idle watchdog", "yellow");
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
async function checkIdleAgent(
|
|
478
|
+
teamName: string,
|
|
479
|
+
member: Member,
|
|
480
|
+
members: Member[],
|
|
481
|
+
status: runtime.AgentRuntimeStatus | null,
|
|
482
|
+
settings: WatchdogConfig,
|
|
483
|
+
shouldContinue: () => boolean,
|
|
484
|
+
): Promise<void> {
|
|
485
|
+
if (!shouldContinue()) return;
|
|
486
|
+
const key = options.readAgentKey(teamName, member.name);
|
|
487
|
+
const state = options.runningReadAgents.get(key);
|
|
488
|
+
const runId = member.lifecycleRunId;
|
|
489
|
+
if (!runId || member.isActive === false || (state ? state.runId !== runId : status?.lifecycleRunId !== runId)) return;
|
|
490
|
+
const activityAt = state?.lastActivityAt ?? status?.lastActivityAt;
|
|
491
|
+
if (activityAt === undefined) return; // Old terminal runtimes have no trustworthy activity evidence.
|
|
492
|
+
const identity = `${teamName}:${member.name}:${runId}`;
|
|
493
|
+
let observation = idleObservations.get(identity);
|
|
494
|
+
if (!observation || observation.activityAt !== activityAt) {
|
|
495
|
+
observation = { activityAt, since: activityAt, warned: false, attempted: false };
|
|
496
|
+
idleObservations.set(identity, observation);
|
|
497
|
+
}
|
|
498
|
+
const isProtected = () => {
|
|
499
|
+
if (options.hasPendingChildren?.(teamName, member)) return true;
|
|
500
|
+
if (state) {
|
|
501
|
+
return state.status === "starting" || state.status === "finishing" || !!state.stopRequested
|
|
502
|
+
|| state.acceptingMessages === false || !!state.activeToolName || !!state.activeWork?.size
|
|
503
|
+
|| !!state.checkOperation || !!state.checkpointOperation;
|
|
504
|
+
}
|
|
505
|
+
return status?.ready !== true || status.currentAction === "starting" || status.currentAction === "finishing"
|
|
506
|
+
|| status.currentAction === "done" || !!status.activeToolName || !!status.activeWorkCount;
|
|
507
|
+
};
|
|
508
|
+
const hasChild = members.some(child => child.isActive !== false && child.parentAgentName === member.name
|
|
509
|
+
&& child.parentLifecycleRunId === runId);
|
|
510
|
+
if (isProtected() || hasChild) {
|
|
511
|
+
observation.since = Date.now();
|
|
512
|
+
observation.warned = false;
|
|
513
|
+
observation.attempted = false;
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
const idleMs = Date.now() - observation.since;
|
|
517
|
+
const reference = `${member.name} (team ${teamName}, run ${runId})`;
|
|
518
|
+
const context = `Last activity: ${new Date(activityAt).toISOString()}.${member.prompt ? ` Assignment: ${member.prompt.slice(0, 300)}` : ""}`;
|
|
519
|
+
if (idleMs >= settings.idleWarningMinutes * 60_000 && !observation.warned) {
|
|
520
|
+
observation.warned = true;
|
|
521
|
+
try {
|
|
522
|
+
await notifyIdleAgent(teamName, `Agent ${reference} is idle: no messages, tokens, or work activity for ${Math.floor(idleMs / 60_000)} minutes. ${context}`);
|
|
523
|
+
} catch (error) {
|
|
524
|
+
observation.warned = false;
|
|
525
|
+
throw error;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
if (!settings.idleAutoStop || idleMs < settings.idleStopMinutes * 60_000 || observation.attempted || !shouldContinue()) return;
|
|
529
|
+
// Re-read persisted ownership before cancelling; a report or replacement may have won the race.
|
|
530
|
+
const current = (await teams.readConfig(teamName)).members.find(candidate => candidate.name === member.name);
|
|
531
|
+
if (current?.lifecycleRunId !== runId || current.isActive === false) return;
|
|
532
|
+
const fence = await readLifecycleTombstone(teamName, member.name);
|
|
533
|
+
if (fence.status !== "absent") return;
|
|
534
|
+
if (!state) {
|
|
535
|
+
const latest = await runtime.readRuntimeStatus(teamName, member.name);
|
|
536
|
+
if (latest?.lifecycleRunId !== runId || latest.lastActivityAt !== activityAt) return;
|
|
537
|
+
status = latest;
|
|
538
|
+
}
|
|
539
|
+
const stillIdle = () => shouldContinue() && !isProtected() && (state
|
|
540
|
+
? options.runningReadAgents.get(key) === state && state.lastActivityAt === activityAt
|
|
541
|
+
: !options.runningReadAgents.has(key));
|
|
542
|
+
if (!stillIdle()) return;
|
|
543
|
+
observation.attempted = true;
|
|
544
|
+
const describeStop = (result: ReadAgentTeardownResult) => result.status === "settled" && result.finalized
|
|
545
|
+
? `Agent ${reference} automatically stopped after ${Math.floor(idleMs / 60_000)} idle minutes.`
|
|
546
|
+
: `Agent ${reference} idle stop is not complete (${result.status}); cancellation/cleanup remains pending or failed. ${result.error ?? ""}`;
|
|
547
|
+
try {
|
|
548
|
+
const result = await shutdownTeammate(teamName, current, { shouldStop: stillIdle });
|
|
549
|
+
if (shouldContinue()) await notifyIdleAgent(teamName, describeStop(result));
|
|
550
|
+
if (!result.finalized && state?.teardownFinalizationPromise) {
|
|
551
|
+
void state.teardownFinalizationPromise.then(async final => {
|
|
552
|
+
if (shouldContinue()) await notifyIdleAgent(teamName, describeStop(final));
|
|
553
|
+
}).catch(() => {});
|
|
554
|
+
}
|
|
555
|
+
} catch (error) {
|
|
556
|
+
if (shouldContinue()) await notifyIdleAgent(teamName, `Agent ${reference} idle stop or notification failed: ${errorText(error)}`);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
464
560
|
async function runWatchdogOnce(targetTeamName: string, shouldContinue: () => boolean = () => true): Promise<void> {
|
|
465
561
|
const settings = loadSettings({ projectDir: options.getSessionCwd() || process.cwd() });
|
|
466
562
|
const staleMs = runtime.HEARTBEAT_STALE_MS + settings.watchdog.bufferSeconds * 1000;
|
|
467
563
|
const now = Date.now();
|
|
468
564
|
const config = await teams.readConfig(targetTeamName);
|
|
469
565
|
const members = config.members.filter((member) => member.name !== "team-lead");
|
|
566
|
+
const identities = new Set(members.map(member => `${targetTeamName}:${member.name}:${member.lifecycleRunId}`));
|
|
567
|
+
for (const identity of idleObservations.keys()) {
|
|
568
|
+
if (identity.startsWith(`${targetTeamName}:`) && !identities.has(identity)) idleObservations.delete(identity);
|
|
569
|
+
}
|
|
470
570
|
const runtimeStatusByMember = await readRuntimeStatusByMember(targetTeamName, members);
|
|
471
571
|
const reaped: ReapedTeammate[] = [];
|
|
472
572
|
|
|
@@ -482,6 +582,11 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
482
582
|
const key = options.readAgentKey(targetTeamName, member.name);
|
|
483
583
|
const inProcessAlive = options.runningReadAgents.has(key);
|
|
484
584
|
|
|
585
|
+
if (inProcessAlive || runtimeStatus?.activeToolName || runtimeStatus?.activeWorkCount) {
|
|
586
|
+
await checkIdleAgent(targetTeamName, member, members, runtimeStatus, settings.watchdog, shouldContinue);
|
|
587
|
+
continue;
|
|
588
|
+
}
|
|
589
|
+
|
|
485
590
|
if (runtimeStale && !inProcessAlive) {
|
|
486
591
|
const result = await shutdownTeammate(targetTeamName, member, { drainQueue: false });
|
|
487
592
|
if (result.status === "settled" && result.finalized) {
|
|
@@ -496,7 +601,9 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
496
601
|
if (result.status === "settled" && result.finalized) {
|
|
497
602
|
reaped.push({ member, reason: "legacy tmux screen is gone" });
|
|
498
603
|
}
|
|
604
|
+
continue;
|
|
499
605
|
}
|
|
606
|
+
await checkIdleAgent(targetTeamName, member, members, runtimeStatus, settings.watchdog, shouldContinue);
|
|
500
607
|
}
|
|
501
608
|
} finally {
|
|
502
609
|
await flushReapedTeammates(targetTeamName, reaped);
|
|
@@ -510,6 +617,7 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
510
617
|
if (leadWatchdogTimer) clearInterval(leadWatchdogTimer);
|
|
511
618
|
leadWatchdogTimer = null;
|
|
512
619
|
leadWatchdogStarted = false;
|
|
620
|
+
idleObservations.clear();
|
|
513
621
|
}
|
|
514
622
|
|
|
515
623
|
function startLeadWatchdog() {
|
|
@@ -517,13 +625,16 @@ export function createLifecycleRuntime(options: LifecycleRuntimeOptions) {
|
|
|
517
625
|
leadWatchdogStarted = true;
|
|
518
626
|
const generation = ++leadWatchdogGeneration;
|
|
519
627
|
leadWatchdogTimer = setInterval(async () => {
|
|
520
|
-
if (generation !== leadWatchdogGeneration) return;
|
|
628
|
+
if (generation !== leadWatchdogGeneration || watchdogInFlight) return;
|
|
521
629
|
const teamName = options.getTeamName();
|
|
522
630
|
if (!teamName) return;
|
|
631
|
+
watchdogInFlight = true;
|
|
523
632
|
try {
|
|
524
633
|
await runWatchdogOnce(teamName, () => generation === leadWatchdogGeneration);
|
|
525
634
|
} catch {
|
|
526
|
-
//
|
|
635
|
+
// Retry observations on the next tick; stop attempts are not repeated blindly.
|
|
636
|
+
} finally {
|
|
637
|
+
watchdogInFlight = false;
|
|
527
638
|
}
|
|
528
639
|
}, 30000);
|
|
529
640
|
}
|
|
@@ -8,9 +8,6 @@ import { createActivityColors } from "./activity-colors";
|
|
|
8
8
|
|
|
9
9
|
const REFRESH_INTERVAL_MS = 250;
|
|
10
10
|
const MAX_NAVIGATION_AGENTS = 6;
|
|
11
|
-
const COLLAPSED_TOOL_RESULT_LINE_LIMIT = 14;
|
|
12
|
-
const COLLAPSED_TOOL_RESULT_HEAD_LINES = 8;
|
|
13
|
-
const COLLAPSED_TOOL_RESULT_TAIL_LINES = 3;
|
|
14
11
|
|
|
15
12
|
export interface AgentFollowViewOptions {
|
|
16
13
|
getAgents(): RunningReadAgent[];
|
|
@@ -227,7 +224,7 @@ function renderToolBlock(theme: ExtendedTeamsTheme, block: Extract<TranscriptBlo
|
|
|
227
224
|
if (compactBlock) return compactBlock;
|
|
228
225
|
|
|
229
226
|
const header = renderToolHeader(theme, block);
|
|
230
|
-
if (!expandLargeToolResults
|
|
227
|
+
if (!expandLargeToolResults) {
|
|
231
228
|
const state = block.result === undefined
|
|
232
229
|
? pendingText(theme, "working")
|
|
233
230
|
: block.isError ? failureText(theme, "✗") : successText(theme, "✓");
|
|
@@ -241,22 +238,12 @@ function renderToolBlock(theme: ExtendedTeamsTheme, block: Extract<TranscriptBlo
|
|
|
241
238
|
|
|
242
239
|
const result = block.result || "(no output)";
|
|
243
240
|
const resultLines = result.split("\n");
|
|
244
|
-
const isCollapsed = !expandLargeToolResults && resultLines.length > COLLAPSED_TOOL_RESULT_LINE_LIMIT;
|
|
245
|
-
const visibleLines = isCollapsed
|
|
246
|
-
? [
|
|
247
|
-
...resultLines.slice(0, COLLAPSED_TOOL_RESULT_HEAD_LINES),
|
|
248
|
-
`… ${resultLines.length - COLLAPSED_TOOL_RESULT_HEAD_LINES - COLLAPSED_TOOL_RESULT_TAIL_LINES} lines hidden · press l to expand logs`,
|
|
249
|
-
...resultLines.slice(-COLLAPSED_TOOL_RESULT_TAIL_LINES),
|
|
250
|
-
]
|
|
251
|
-
: resultLines;
|
|
252
241
|
const resultLineWidth = width === undefined ? undefined : Math.max(1, width - visibleWidth("│ "));
|
|
253
|
-
const boundedLines =
|
|
242
|
+
const boundedLines = resultLines.map((line) => resultLineWidth === undefined
|
|
254
243
|
? line
|
|
255
244
|
: truncateToWidth(line, resultLineWidth, "…"));
|
|
256
|
-
const body = boundedLines.map((line
|
|
257
|
-
|
|
258
|
-
: `${structuralText(theme, "│")} ${line}`);
|
|
259
|
-
const summary = `${resultLines.length} line${resultLines.length === 1 ? "" : "s"} · ${formatResultSize(result)}${isCollapsed ? " · collapsed" : ""}`;
|
|
245
|
+
const body = boundedLines.map((line) => `${structuralText(theme, "│")} ${line}`);
|
|
246
|
+
const summary = `${resultLines.length} line${resultLines.length === 1 ? "" : "s"} · ${formatResultSize(result)}`;
|
|
260
247
|
const renderedSummary = block.isError ? failureText(theme, summary) : successText(theme, summary);
|
|
261
248
|
return [header, ...body, `${structuralText(theme, "╰─")} ${renderedSummary}`, ""];
|
|
262
249
|
}
|
package/package.json
CHANGED
package/src/utils/runtime.ts
CHANGED
|
@@ -38,6 +38,8 @@ export interface AgentRuntimeStatus {
|
|
|
38
38
|
pid?: number;
|
|
39
39
|
startedAt?: number;
|
|
40
40
|
lastHeartbeatAt?: number;
|
|
41
|
+
lastActivityAt?: number;
|
|
42
|
+
activeWorkCount?: number;
|
|
41
43
|
lastInboxReadAt?: number;
|
|
42
44
|
ready?: boolean;
|
|
43
45
|
currentAction?: "starting" | "thinking" | "working" | "finishing" | "done";
|
|
@@ -234,7 +236,7 @@ export async function cleanupStaleRuntimeFiles(
|
|
|
234
236
|
try {
|
|
235
237
|
const status = JSON.parse(fs.readFileSync(p, "utf-8")) as AgentRuntimeStatus;
|
|
236
238
|
const lastActivity = status.lastHeartbeatAt || status.startedAt || 0;
|
|
237
|
-
shouldDelete = (now - lastActivity) > RUNTIME_STALE_MS;
|
|
239
|
+
shouldDelete = !status.activeToolName && !status.activeWorkCount && (now - lastActivity) > RUNTIME_STALE_MS;
|
|
238
240
|
} catch {
|
|
239
241
|
shouldDelete = true;
|
|
240
242
|
}
|
package/src/utils/settings.ts
CHANGED
|
@@ -76,6 +76,9 @@ export interface CategoryConfig {
|
|
|
76
76
|
export interface WatchdogConfig {
|
|
77
77
|
/** Grace buffer (seconds) added on top of the heartbeat interval. */
|
|
78
78
|
bufferSeconds: number;
|
|
79
|
+
idleWarningMinutes: number;
|
|
80
|
+
idleStopMinutes: number;
|
|
81
|
+
idleAutoStop: boolean;
|
|
79
82
|
}
|
|
80
83
|
|
|
81
84
|
export interface WriteAgentsConfig {
|
|
@@ -151,7 +154,7 @@ export const DEFAULT_READ_HELPER_MAX_CONCURRENT = 10;
|
|
|
151
154
|
|
|
152
155
|
export const DEFAULT_SETTINGS: PiExtendedTeamsSettings = {
|
|
153
156
|
activityColors: DEFAULT_ACTIVITY_COLORS,
|
|
154
|
-
watchdog: { bufferSeconds: 30 },
|
|
157
|
+
watchdog: { bufferSeconds: 30, idleWarningMinutes: 5, idleStopMinutes: 10, idleAutoStop: true },
|
|
155
158
|
writeAgents: { maxConcurrent: DEFAULT_WRITE_AGENT_MAX_CONCURRENT, queueOverflow: true },
|
|
156
159
|
readAgents: { maxConcurrent: DEFAULT_READ_AGENT_MAX_CONCURRENT, queueOverflow: true },
|
|
157
160
|
readHelpers: { maxConcurrent: DEFAULT_READ_HELPER_MAX_CONCURRENT, queueOverflow: true },
|
|
@@ -296,6 +299,11 @@ function applyLayer(acc: PiExtendedTeamsSettings, raw: any, options: { favoriteM
|
|
|
296
299
|
if (raw.watchdog && typeof raw.watchdog === "object") {
|
|
297
300
|
const b = Number(raw.watchdog.bufferSeconds);
|
|
298
301
|
if (Number.isFinite(b) && b >= 0) acc.watchdog.bufferSeconds = b;
|
|
302
|
+
for (const key of ["idleWarningMinutes", "idleStopMinutes"] as const) {
|
|
303
|
+
const value = raw.watchdog[key];
|
|
304
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) acc.watchdog[key] = value;
|
|
305
|
+
}
|
|
306
|
+
if (typeof raw.watchdog.idleAutoStop === "boolean") acc.watchdog.idleAutoStop = raw.watchdog.idleAutoStop;
|
|
299
307
|
}
|
|
300
308
|
|
|
301
309
|
if (raw.writeAgents && typeof raw.writeAgents === "object") {
|