pi-crew 0.1.34 → 0.1.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/docs/architecture.md +8 -1
- package/docs/research-phase9-observability-reliability-plan.md +42 -42
- package/docs/research-source-pi-crew-reference.md +174 -0
- package/package.json +1 -1
- package/schema.json +42 -0
- package/src/config/config.ts +101 -0
- package/src/extension/register.ts +66 -3
- package/src/extension/registration/commands.ts +14 -3
- package/src/extension/registration/team-tool.ts +3 -1
- package/src/extension/team-tool/api.ts +27 -2
- package/src/extension/team-tool/context.ts +2 -0
- package/src/extension/team-tool/run.ts +2 -2
- package/src/extension/team-tool.ts +1 -1
- package/src/observability/correlation.ts +35 -0
- package/src/observability/event-to-metric.ts +54 -0
- package/src/observability/exporters/adapter.ts +24 -0
- package/src/observability/exporters/otlp-exporter.ts +65 -0
- package/src/observability/exporters/prometheus-exporter.ts +47 -0
- package/src/observability/metric-registry.ts +72 -0
- package/src/observability/metric-retention.ts +46 -0
- package/src/observability/metric-sink.ts +51 -0
- package/src/observability/metrics-primitives.ts +166 -0
- package/src/runtime/child-pi.ts +5 -1
- package/src/runtime/crash-recovery.ts +56 -0
- package/src/runtime/deadletter.ts +36 -0
- package/src/runtime/diagnostic-export.ts +8 -1
- package/src/runtime/heartbeat-gradient.ts +28 -0
- package/src/runtime/heartbeat-watcher.ts +80 -0
- package/src/runtime/retry-executor.ts +59 -0
- package/src/runtime/team-runner.ts +57 -5
- package/src/schema/config-schema.ts +29 -0
- package/src/state/event-log.ts +3 -2
- package/src/state/types.ts +7 -0
- package/src/ui/dashboard-panes/agents-pane.ts +4 -1
- package/src/ui/dashboard-panes/metrics-pane.ts +34 -0
- package/src/ui/heartbeat-aggregator.ts +14 -4
- package/src/ui/keybinding-map.ts +4 -2
- package/src/ui/live-run-sidebar.ts +5 -4
- package/src/ui/run-action-dispatcher.ts +3 -2
- package/src/ui/run-dashboard.ts +17 -6
- package/src/ui/spinner.ts +17 -0
|
@@ -81,6 +81,32 @@ export const PiTeamsNotificationsConfigSchema = Type.Object({
|
|
|
81
81
|
sinkRetentionDays: Type.Optional(Type.Integer({ minimum: 1, maximum: 90 })),
|
|
82
82
|
});
|
|
83
83
|
|
|
84
|
+
export const PiTeamsObservabilityConfigSchema = Type.Object({
|
|
85
|
+
enabled: Type.Optional(Type.Boolean()),
|
|
86
|
+
pollIntervalMs: Type.Optional(Type.Integer({ minimum: 1000, maximum: 60000 })),
|
|
87
|
+
metricRetentionDays: Type.Optional(Type.Integer({ minimum: 1, maximum: 365 })),
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
export const PiTeamsReliabilityConfigSchema = Type.Object({
|
|
91
|
+
autoRetry: Type.Optional(Type.Boolean()),
|
|
92
|
+
retryPolicy: Type.Optional(Type.Object({
|
|
93
|
+
maxAttempts: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })),
|
|
94
|
+
backoffMs: Type.Optional(Type.Integer({ minimum: 100, maximum: 60000 })),
|
|
95
|
+
jitterRatio: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })),
|
|
96
|
+
exponentialFactor: Type.Optional(Type.Number({ minimum: 1, maximum: 5 })),
|
|
97
|
+
retryableErrors: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
|
|
98
|
+
})),
|
|
99
|
+
autoRecover: Type.Optional(Type.Boolean()),
|
|
100
|
+
deadletterThreshold: Type.Optional(Type.Integer({ minimum: 1 })),
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
export const PiTeamsOtlpConfigSchema = Type.Object({
|
|
104
|
+
enabled: Type.Optional(Type.Boolean()),
|
|
105
|
+
endpoint: Type.Optional(Type.String({ minLength: 1 })),
|
|
106
|
+
headers: Type.Optional(Type.Record(Type.String({ minLength: 1 }), Type.String())),
|
|
107
|
+
intervalMs: Type.Optional(Type.Integer({ minimum: 5000 })),
|
|
108
|
+
});
|
|
109
|
+
|
|
84
110
|
export const PiTeamsUiConfigSchema = Type.Object({
|
|
85
111
|
widgetPlacement: Type.Optional(Type.Union([Type.Literal("aboveEditor"), Type.Literal("belowEditor")])),
|
|
86
112
|
widgetMaxLines: Type.Optional(Type.Integer({ minimum: 1 })),
|
|
@@ -112,5 +138,8 @@ export const PiTeamsConfigSchema = Type.Object({
|
|
|
112
138
|
tools: Type.Optional(PiTeamsToolsConfigSchema),
|
|
113
139
|
telemetry: Type.Optional(PiTeamsTelemetryConfigSchema),
|
|
114
140
|
notifications: Type.Optional(PiTeamsNotificationsConfigSchema),
|
|
141
|
+
observability: Type.Optional(PiTeamsObservabilityConfigSchema),
|
|
142
|
+
reliability: Type.Optional(PiTeamsReliabilityConfigSchema),
|
|
143
|
+
otlp: Type.Optional(PiTeamsOtlpConfigSchema),
|
|
115
144
|
ui: Type.Optional(PiTeamsUiConfigSchema),
|
|
116
145
|
});
|
package/src/state/event-log.ts
CHANGED
|
@@ -48,7 +48,7 @@ const MAX_EVENTS_BYTES = 50 * 1024 * 1024;
|
|
|
48
48
|
|
|
49
49
|
const sequenceCache = new Map<string, { size: number; mtimeMs: number; seq: number }>();
|
|
50
50
|
|
|
51
|
-
function sequencePath(eventsPath: string): string {
|
|
51
|
+
export function sequencePath(eventsPath: string): string {
|
|
52
52
|
return `${eventsPath}.seq`;
|
|
53
53
|
}
|
|
54
54
|
|
|
@@ -57,7 +57,8 @@ function parseSequence(raw: string): number | undefined {
|
|
|
57
57
|
return Number.isInteger(value) && value >= 0 ? value : undefined;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
function scanSequence(eventsPath: string): number {
|
|
60
|
+
export function scanSequence(eventsPath: string): number {
|
|
61
|
+
if (!fs.existsSync(eventsPath)) return 0;
|
|
61
62
|
let max = 0;
|
|
62
63
|
for (const line of fs.readFileSync(eventsPath, "utf-8").split("\n")) {
|
|
63
64
|
if (!line.trim()) continue;
|
package/src/state/types.ts
CHANGED
|
@@ -138,6 +138,12 @@ export interface TaskCheckpointState {
|
|
|
138
138
|
childPid?: number;
|
|
139
139
|
}
|
|
140
140
|
|
|
141
|
+
export interface TaskAttemptState {
|
|
142
|
+
startedAt: string;
|
|
143
|
+
endedAt?: string;
|
|
144
|
+
error?: string;
|
|
145
|
+
}
|
|
146
|
+
|
|
141
147
|
export interface TeamTaskState {
|
|
142
148
|
id: string;
|
|
143
149
|
runId: string;
|
|
@@ -166,6 +172,7 @@ export interface TeamTaskState {
|
|
|
166
172
|
claim?: TaskClaimState;
|
|
167
173
|
heartbeat?: WorkerHeartbeatState;
|
|
168
174
|
checkpoint?: TaskCheckpointState;
|
|
175
|
+
attempts?: TaskAttemptState[];
|
|
169
176
|
taskPacket?: TaskPacket;
|
|
170
177
|
verification?: VerificationEvidence;
|
|
171
178
|
graph?: TaskGraphNode;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { RunDashboardOptions } from "../run-dashboard.ts";
|
|
2
|
+
import { iconForStatus } from "../status-colors.ts";
|
|
2
3
|
import type { RunUiSnapshot } from "../snapshot-types.ts";
|
|
4
|
+
import { spinnerFrame } from "../spinner.ts";
|
|
3
5
|
|
|
4
6
|
function tokens(agent: RunUiSnapshot["agents"][number]): string {
|
|
5
7
|
const total = (agent.usage?.input ?? 0) + (agent.usage?.output ?? agent.progress?.tokens ?? 0) + (agent.usage?.cacheRead ?? 0) + (agent.usage?.cacheWrite ?? 0);
|
|
@@ -19,7 +21,8 @@ export function renderAgentsPane(snapshot: RunUiSnapshot | undefined, options: R
|
|
|
19
21
|
options.showTokens !== false ? tokens(agent) : undefined,
|
|
20
22
|
options.showModel !== false ? (agent.model ? `model=${agent.model}` : undefined) : undefined,
|
|
21
23
|
].filter((part): part is string => Boolean(part));
|
|
22
|
-
|
|
24
|
+
const icon = iconForStatus(agent.status, { runningGlyph: spinnerFrame(agent.taskId) });
|
|
25
|
+
return `${icon} ${agent.taskId} ${agent.role}->${agent.agent} · ${parts.join(" · ")}`;
|
|
23
26
|
}),
|
|
24
27
|
];
|
|
25
28
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
2
|
+
import type { HistogramPoint, MetricLabels, MetricPoint } from "../../observability/metrics-primitives.ts";
|
|
3
|
+
import type { RunUiSnapshot } from "../snapshot-types.ts";
|
|
4
|
+
|
|
5
|
+
export interface MetricsPaneOptions {
|
|
6
|
+
registry?: MetricRegistry;
|
|
7
|
+
maxCounters?: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function labelsText(labels: MetricLabels): string {
|
|
11
|
+
const entries = Object.entries(labels);
|
|
12
|
+
return entries.length ? `{${entries.map(([key, value]) => `${key}=${value}`).join(",")}}` : "";
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function isHistogramPoint(point: MetricPoint | HistogramPoint): point is HistogramPoint {
|
|
16
|
+
return "quantiles" in point;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function renderMetricsPane(_snapshot: RunUiSnapshot | undefined, opts: MetricsPaneOptions = {}): string[] {
|
|
20
|
+
if (!opts.registry) return ["Metrics pane: registry unavailable"];
|
|
21
|
+
const snapshots = opts.registry.snapshot();
|
|
22
|
+
if (!snapshots.length) return ["Metrics pane: no metrics recorded"];
|
|
23
|
+
const lines = ["Metrics pane: top metrics"];
|
|
24
|
+
for (const snapshot of snapshots.slice(0, opts.maxCounters ?? 10)) {
|
|
25
|
+
const first = snapshot.values[0];
|
|
26
|
+
if (!first) {
|
|
27
|
+
lines.push(`${snapshot.name}: empty`);
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (isHistogramPoint(first)) lines.push(`${snapshot.name}${labelsText(first.labels)} count=${first.count} p95=${Number.isFinite(first.quantiles.p95) ? Math.round(first.quantiles.p95) : "n/a"}`);
|
|
31
|
+
else lines.push(`${snapshot.name}${labelsText(first.labels)} ${first.value}`);
|
|
32
|
+
}
|
|
33
|
+
return lines;
|
|
34
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { TeamTaskState } from "../state/types.ts";
|
|
2
|
+
import { classifyHeartbeat, heartbeatAgeMs } from "../runtime/heartbeat-gradient.ts";
|
|
3
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
2
4
|
import type { RunUiSnapshot } from "./snapshot-types.ts";
|
|
3
5
|
|
|
4
6
|
export interface HeartbeatSummary {
|
|
@@ -9,12 +11,14 @@ export interface HeartbeatSummary {
|
|
|
9
11
|
dead: number;
|
|
10
12
|
missing: number;
|
|
11
13
|
worstStaleMs: number;
|
|
14
|
+
gradient: { healthy: number; warn: number; stale: number; dead: number };
|
|
12
15
|
}
|
|
13
16
|
|
|
14
17
|
export interface HeartbeatSummaryOptions {
|
|
15
18
|
staleMs?: number;
|
|
16
19
|
deadMs?: number;
|
|
17
20
|
now?: number | Date;
|
|
21
|
+
registry?: MetricRegistry;
|
|
18
22
|
}
|
|
19
23
|
|
|
20
24
|
function nowMs(now: number | Date | undefined): number {
|
|
@@ -31,22 +35,28 @@ export function summarizeHeartbeats(snapshot: RunUiSnapshot, opts: HeartbeatSumm
|
|
|
31
35
|
const staleMs = opts.staleMs ?? 60_000;
|
|
32
36
|
const deadMs = opts.deadMs ?? 5 * 60_000;
|
|
33
37
|
const current = nowMs(opts.now);
|
|
34
|
-
const summary: HeartbeatSummary = { runId: snapshot.runId, totalTasks: snapshot.tasks.length, healthy: 0, stale: 0, dead: 0, missing: 0, worstStaleMs: 0 };
|
|
38
|
+
const summary: HeartbeatSummary = { runId: snapshot.runId, totalTasks: snapshot.tasks.length, healthy: 0, stale: 0, dead: 0, missing: 0, worstStaleMs: 0, gradient: { healthy: 0, warn: 0, stale: 0, dead: 0 } };
|
|
35
39
|
for (const task of snapshot.tasks) {
|
|
36
40
|
if (!isActiveTask(task)) continue;
|
|
37
41
|
const heartbeat = task.heartbeat;
|
|
38
42
|
if (!heartbeat) {
|
|
39
43
|
summary.missing += 1;
|
|
44
|
+
summary.gradient.dead += 1;
|
|
40
45
|
continue;
|
|
41
46
|
}
|
|
42
|
-
const age =
|
|
47
|
+
const age = heartbeatAgeMs(heartbeat, current);
|
|
43
48
|
if (!Number.isFinite(age)) {
|
|
44
49
|
summary.missing += 1;
|
|
50
|
+
summary.gradient.dead += 1;
|
|
45
51
|
continue;
|
|
46
52
|
}
|
|
47
53
|
summary.worstStaleMs = Math.max(summary.worstStaleMs, age);
|
|
48
|
-
|
|
49
|
-
|
|
54
|
+
const level = classifyHeartbeat(heartbeat, { warnMs: Math.max(1, Math.floor(staleMs / 2)), staleMs, deadMs }, current);
|
|
55
|
+
summary.gradient[level] += 1;
|
|
56
|
+
opts.registry?.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: snapshot.runId, taskId: task.id }, age);
|
|
57
|
+
opts.registry?.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: snapshot.runId, level });
|
|
58
|
+
if (level === "dead") summary.dead += 1;
|
|
59
|
+
else if (level === "stale") summary.stale += 1;
|
|
50
60
|
else summary.healthy += 1;
|
|
51
61
|
}
|
|
52
62
|
return summary;
|
package/src/ui/keybinding-map.ts
CHANGED
|
@@ -13,7 +13,7 @@ export const DASHBOARD_KEYS = {
|
|
|
13
13
|
reload: ["r"],
|
|
14
14
|
progressToggle: ["p"],
|
|
15
15
|
},
|
|
16
|
-
pane: { agents: ["1"], progress: ["2"], mailbox: ["3"], output: ["4"], health: ["5"] },
|
|
16
|
+
pane: { agents: ["1"], progress: ["2"], mailbox: ["3"], output: ["4"], health: ["5"], metrics: ["6"] },
|
|
17
17
|
navigation: { up: ["k", "\u001b[A"], down: ["j", "\u001b[B"] },
|
|
18
18
|
mailbox: { ack: ["A"], nudge: ["N"], compose: ["C"], preview: ["P"], ackAll: ["X"], openDetail: ["\r", "\n"] },
|
|
19
19
|
health: { recovery: ["R"], killStale: ["K"], diagnosticExport: ["D"] },
|
|
@@ -53,6 +53,7 @@ export type DashboardKeyAction =
|
|
|
53
53
|
| "pane-mailbox"
|
|
54
54
|
| "pane-output"
|
|
55
55
|
| "pane-health"
|
|
56
|
+
| "pane-metrics"
|
|
56
57
|
| "up"
|
|
57
58
|
| "down"
|
|
58
59
|
| "mailbox-detail"
|
|
@@ -61,7 +62,7 @@ export type DashboardKeyAction =
|
|
|
61
62
|
| "health-diagnostic-export"
|
|
62
63
|
| "notifications-dismiss";
|
|
63
64
|
|
|
64
|
-
export function dashboardActionForKey(data: string, activePane?: "agents" | "progress" | "mailbox" | "output" | "health"): DashboardKeyAction | undefined {
|
|
65
|
+
export function dashboardActionForKey(data: string, activePane?: "agents" | "progress" | "mailbox" | "output" | "health" | "metrics"): DashboardKeyAction | undefined {
|
|
65
66
|
if (includes(DASHBOARD_KEYS.close, data)) return "close";
|
|
66
67
|
if (activePane === "mailbox" && includes(DASHBOARD_KEYS.mailbox.openDetail, data)) return "mailbox-detail";
|
|
67
68
|
if (activePane === "health") {
|
|
@@ -86,6 +87,7 @@ export function dashboardActionForKey(data: string, activePane?: "agents" | "pro
|
|
|
86
87
|
if (includes(DASHBOARD_KEYS.pane.mailbox, data)) return "pane-mailbox";
|
|
87
88
|
if (includes(DASHBOARD_KEYS.pane.output, data)) return "pane-output";
|
|
88
89
|
if (includes(DASHBOARD_KEYS.pane.health, data)) return "pane-health";
|
|
90
|
+
if (includes(DASHBOARD_KEYS.pane.metrics, data)) return "pane-metrics";
|
|
89
91
|
if (includes(DASHBOARD_KEYS.navigation.up, data)) return "up";
|
|
90
92
|
if (includes(DASHBOARD_KEYS.navigation.down, data)) return "down";
|
|
91
93
|
return undefined;
|
|
@@ -13,9 +13,9 @@ import type { CrewTheme } from "./theme-adapter.ts";
|
|
|
13
13
|
import { asCrewTheme, subscribeThemeChange } from "./theme-adapter.ts";
|
|
14
14
|
import { Box, Text } from "./layout-primitives.ts";
|
|
15
15
|
import type { RunSnapshotCache, RunUiSnapshot } from "./snapshot-types.ts";
|
|
16
|
+
import { spinnerBucket, spinnerFrame } from "./spinner.ts";
|
|
16
17
|
|
|
17
18
|
const TASK_READ_TTL_MS = 200;
|
|
18
|
-
const SPINNER = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
19
19
|
|
|
20
20
|
function renderLines(lines: string[], width: number): string[] {
|
|
21
21
|
const box = new Box(0, 0);
|
|
@@ -75,10 +75,11 @@ export class LiveRunSidebar {
|
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
private buildSignature(manifestStatus: string, tasks: TeamTaskState[], agents: ReturnType<typeof readCrewAgents>, waitingCount: number, snapshot?: RunUiSnapshot): string {
|
|
78
|
-
|
|
78
|
+
const animation = agents.some((agent) => agent.status === "running") ? `:spin=${spinnerBucket()}` : "";
|
|
79
|
+
if (snapshot) return `${snapshot.signature}:${waitingCount}${animation}`;
|
|
79
80
|
const taskSig = tasks.map((task) => `${task.id}:${task.status}:${task.startedAt ?? ""}:${task.finishedAt ?? ""}:${task.agentProgress?.currentTool ?? ""}:${task.agentProgress?.toolCount ?? 0}:${task.agentProgress?.tokens ?? 0}:${task.usage ? JSON.stringify(task.usage) : ""}`).join("|");
|
|
80
81
|
const agentSig = agents.map((agent) => [agent.id, agent.status, agent.startedAt, agent.completedAt ?? "", agent.progress?.currentTool ?? "", agent.progress?.toolCount ?? 0, agent.progress?.tokens ?? 0, agent.progress?.turns ?? 0, agent.progress?.lastActivityAt ?? "", agent.progress?.recentOutput.at(-1) ?? "", agent.toolUses ?? 0].join(":")).join("|");
|
|
81
|
-
return `${manifestStatus}|${agents.length}|${waitingCount}|${taskSig}|${agentSig}`;
|
|
82
|
+
return `${manifestStatus}|${agents.length}|${waitingCount}|${taskSig}|${agentSig}${animation}`;
|
|
82
83
|
}
|
|
83
84
|
|
|
84
85
|
private colorLine(line: string): string {
|
|
@@ -140,7 +141,7 @@ export class LiveRunSidebar {
|
|
|
140
141
|
line(`Active agents (${active.length})`, w),
|
|
141
142
|
];
|
|
142
143
|
for (const agent of active.slice(0, 8)) {
|
|
143
|
-
const status = iconForStatus(agent.status, { runningGlyph:
|
|
144
|
+
const status = iconForStatus(agent.status, { runningGlyph: spinnerFrame(agent.taskId) });
|
|
144
145
|
const usage = agent.usage ? formatUsage(agent.usage) : agent.progress?.tokens ? `tokens=${agent.progress.tokens}` : "usage=pending";
|
|
145
146
|
lines.push(line(`${status} ${agent.taskId} ${agent.role}->${agent.agent}`, w));
|
|
146
147
|
lines.push(line(` ${agent.routing ? `model ${agent.routing.requested ? `${agent.routing.requested} → ` : ""}${agent.routing.resolved}` : agent.model ? `model ${agent.model}` : "model pending"}`, w));
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
2
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
2
3
|
import { handleTeamTool } from "../extension/team-tool.ts";
|
|
3
4
|
import { isToolError, textFromToolResult } from "../extension/tool-result.ts";
|
|
4
5
|
import { loadRunManifestById, saveRunTasks } from "../state/state-store.ts";
|
|
@@ -91,9 +92,9 @@ export async function dispatchKillStaleWorkers(ctx: ExtensionContext, runId: str
|
|
|
91
92
|
}
|
|
92
93
|
}
|
|
93
94
|
|
|
94
|
-
export async function dispatchDiagnosticExport(ctx: ExtensionContext, runId: string): Promise<RunActionResult> {
|
|
95
|
+
export async function dispatchDiagnosticExport(ctx: ExtensionContext, runId: string, options: { registry?: MetricRegistry } = {}): Promise<RunActionResult> {
|
|
95
96
|
try {
|
|
96
|
-
const exported = await exportDiagnostic(ctx, runId);
|
|
97
|
+
const exported = await exportDiagnostic(ctx, runId, options);
|
|
97
98
|
return { ok: true, message: `Diagnostic exported to ${exported.path}`, data: exported.path };
|
|
98
99
|
} catch (error) {
|
|
99
100
|
return err(error);
|
package/src/ui/run-dashboard.ts
CHANGED
|
@@ -17,8 +17,11 @@ import { renderMailboxPane } from "./dashboard-panes/mailbox-pane.ts";
|
|
|
17
17
|
import { renderProgressPane } from "./dashboard-panes/progress-pane.ts";
|
|
18
18
|
import { renderTranscriptPane } from "./dashboard-panes/transcript-pane.ts";
|
|
19
19
|
import { renderHealthPane } from "./dashboard-panes/health-pane.ts";
|
|
20
|
+
import { renderMetricsPane } from "./dashboard-panes/metrics-pane.ts";
|
|
20
21
|
import { dashboardActionForKey } from "./keybinding-map.ts";
|
|
21
22
|
import type { RunSnapshotCache, RunUiSnapshot } from "./snapshot-types.ts";
|
|
23
|
+
import { spinnerBucket, spinnerFrame } from "./spinner.ts";
|
|
24
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
22
25
|
|
|
23
26
|
interface DashboardComponent {
|
|
24
27
|
invalidate(): void;
|
|
@@ -33,6 +36,7 @@ export interface RunDashboardOptions {
|
|
|
33
36
|
showTools?: boolean;
|
|
34
37
|
snapshotCache?: RunSnapshotCache;
|
|
35
38
|
runProvider?: () => TeamRunManifest[];
|
|
39
|
+
registry?: MetricRegistry;
|
|
36
40
|
}
|
|
37
41
|
|
|
38
42
|
export type RunDashboardAction = "status" | "summary" | "artifacts" | "api" | "events" | "agents" | "agent-events" | "agent-output" | "agent-transcript" | "mailbox" | "reload" | "mailbox-detail" | "health-recovery" | "health-kill-stale" | "health-diagnostic-export" | "notifications-dismiss";
|
|
@@ -140,7 +144,8 @@ function agentPreviewLine(agent: CrewAgentRecord, task: TeamTaskState | undefine
|
|
|
140
144
|
agent.startedAt ? `age=${formatAge(agent.completedAt ?? agent.startedAt)}` : undefined,
|
|
141
145
|
].filter((part): part is string => Boolean(part));
|
|
142
146
|
const recent = agent.progress?.recentOutput?.at(-1);
|
|
143
|
-
|
|
147
|
+
const icon = iconForStatus(agent.status, { runningGlyph: spinnerFrame(agent.taskId) });
|
|
148
|
+
return `Agent: ${icon} ${agent.taskId} ${agent.role}->${agent.agent}${stats.length ? ` · ${stats.join(" · ")}` : ""}${recent ? ` ⎿ ${recent}` : ""}`;
|
|
144
149
|
}
|
|
145
150
|
|
|
146
151
|
function readAgentPreview(run: TeamRunManifest, maxLines = 5, options: RunDashboardOptions = {}): string[] {
|
|
@@ -189,7 +194,7 @@ function runLabel(run: TeamRunManifest, selected: boolean, snapshotCache?: RunSn
|
|
|
189
194
|
const step = stale ? "orphaned queued run" : running ? `step ${running.taskId}` : queued ? `queued ${queued.taskId}` : `agents ${agents.length}`;
|
|
190
195
|
const status: RunStatus = stale ? "stale" : (run.status as RunStatus);
|
|
191
196
|
const marker = selected ? "›" : " ";
|
|
192
|
-
return `${marker} ${iconForStatus(status)} ${run.runId.slice(-8)} ${status} | ${run.team}/${run.workflow ?? "none"} | ${step} | ${run.goal}`;
|
|
197
|
+
return `${marker} ${iconForStatus(status, { runningGlyph: spinnerFrame(run.runId) })} ${run.runId.slice(-8)} ${status} | ${run.team}/${run.workflow ?? "none"} | ${step} | ${run.goal}`;
|
|
193
198
|
}
|
|
194
199
|
|
|
195
200
|
function groupedRuns(runs: TeamRunManifest[], snapshotCache?: RunSnapshotCache): Array<{ label: string; run?: TeamRunManifest }> {
|
|
@@ -219,7 +224,7 @@ function countByStatus(runs: TeamRunManifest[], snapshotCache?: RunSnapshotCache
|
|
|
219
224
|
export class RunDashboard implements DashboardComponent {
|
|
220
225
|
private selected = 0;
|
|
221
226
|
private showFullProgress = false;
|
|
222
|
-
private activePane: "agents" | "progress" | "mailbox" | "output" | "health" = "agents";
|
|
227
|
+
private activePane: "agents" | "progress" | "mailbox" | "output" | "health" | "metrics" = "agents";
|
|
223
228
|
private runs: TeamRunManifest[];
|
|
224
229
|
private readonly done: (selection: RunDashboardSelection | undefined) => void;
|
|
225
230
|
private readonly theme: CrewTheme;
|
|
@@ -253,15 +258,18 @@ export class RunDashboard implements DashboardComponent {
|
|
|
253
258
|
}
|
|
254
259
|
|
|
255
260
|
private buildSignature(): string {
|
|
261
|
+
let hasRunning = false;
|
|
256
262
|
const statuses = this.runs.map((run) => {
|
|
257
263
|
const snapshot = snapshotFor(run, this.options.snapshotCache);
|
|
258
264
|
const displayRun = snapshot?.manifest ?? run;
|
|
259
265
|
const agents = snapshot?.agents ?? agentsFor(run, this.options.snapshotCache);
|
|
260
266
|
const stale = isLikelyOrphanedActiveRun(displayRun, agents);
|
|
261
267
|
const status: RunStatus = stale ? "stale" : (displayRun.status as RunStatus);
|
|
268
|
+
if (status === "running" || agents.some((agent) => agent.status === "running")) hasRunning = true;
|
|
262
269
|
return snapshot?.signature ?? `${displayRun.runId}:${displayRun.status}:${displayRun.updatedAt}:${status}`;
|
|
263
270
|
}).join("|");
|
|
264
|
-
|
|
271
|
+
const metricsSig = this.activePane === "metrics" ? `:metrics=${this.options.registry?.snapshot().length ?? 0}:${spinnerBucket()}` : "";
|
|
272
|
+
return `${this.selected}:${this.showFullProgress ? 1 : 0}:${this.activePane}:${statuses}${hasRunning ? `:spin=${spinnerBucket()}` : ""}${metricsSig}`;
|
|
265
273
|
}
|
|
266
274
|
|
|
267
275
|
invalidate(): void {
|
|
@@ -291,7 +299,7 @@ export class RunDashboard implements DashboardComponent {
|
|
|
291
299
|
border("╭", "╮"),
|
|
292
300
|
`│ ${pad(truncate(`${fg("accent", "▐")} ${this.theme.bold(this.options.placement === "right" ? "pi-crew right sidebar (anchored top-right)" : "pi-crew dashboard")}`, innerWidth - 1), innerWidth - 1)}│`,
|
|
293
301
|
`│ ${pad(truncate(`Runs: ${this.runs.length} • ${countByStatus(this.runs, this.options.snapshotCache)}`, innerWidth - 1), innerWidth - 1)}│`,
|
|
294
|
-
`│ ${pad(truncate(`↑/↓ select • 1 agents 2 progress 3 mailbox 4 output 5 health • s/u/a/i actions • R/K/D health • H hush`, innerWidth - 1), innerWidth - 1)}│`,
|
|
302
|
+
`│ ${pad(truncate(`↑/↓ select • 1 agents 2 progress 3 mailbox 4 output 5 health 6 metrics • s/u/a/i actions • R/K/D health • H hush`, innerWidth - 1), innerWidth - 1)}│`,
|
|
295
303
|
border("├", "┤"),
|
|
296
304
|
];
|
|
297
305
|
if (this.runs.length === 0) {
|
|
@@ -336,7 +344,9 @@ export class RunDashboard implements DashboardComponent {
|
|
|
336
344
|
? renderMailboxPane(selectedSnapshot)
|
|
337
345
|
: this.activePane === "health"
|
|
338
346
|
? renderHealthPane(selectedSnapshot, { isForeground: selectedDisplayRun.async ? false : true })
|
|
339
|
-
:
|
|
347
|
+
: this.activePane === "metrics"
|
|
348
|
+
? renderMetricsPane(selectedSnapshot, { registry: this.options.registry })
|
|
349
|
+
: renderTranscriptPane(selectedSnapshot)
|
|
340
350
|
: [
|
|
341
351
|
...readAgentPreview(selectedDisplayRun, this.showFullProgress ? 20 : 8, this.options),
|
|
342
352
|
...readProgressPreview(selectedDisplayRun, this.showFullProgress ? 20 : 5),
|
|
@@ -408,6 +418,7 @@ export class RunDashboard implements DashboardComponent {
|
|
|
408
418
|
else if (action === "pane-mailbox") this.activePane = "mailbox";
|
|
409
419
|
else if (action === "pane-output") this.activePane = "output";
|
|
410
420
|
else if (action === "pane-health") this.activePane = "health";
|
|
421
|
+
else if (action === "pane-metrics") this.activePane = "metrics";
|
|
411
422
|
else if (action === "up") this.selected = Math.max(0, this.selected - 1);
|
|
412
423
|
else if (action === "down") {
|
|
413
424
|
const selectableCount = groupedRuns(this.runs, this.options.snapshotCache).filter((row) => row.run).length;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const SUBAGENT_SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] as const;
|
|
2
|
+
export const SUBAGENT_SPINNER_FRAME_MS = 160;
|
|
3
|
+
|
|
4
|
+
export function spinnerBucket(now = Date.now(), frameMs = SUBAGENT_SPINNER_FRAME_MS): number {
|
|
5
|
+
return Math.floor(now / Math.max(1, frameMs));
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function hashKey(key: string): number {
|
|
9
|
+
let hash = 0;
|
|
10
|
+
for (let index = 0; index < key.length; index += 1) hash = (hash * 31 + key.charCodeAt(index)) >>> 0;
|
|
11
|
+
return hash;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function spinnerFrame(key = "", now = Date.now()): string {
|
|
15
|
+
const offset = key ? hashKey(key) % SUBAGENT_SPINNER_FRAMES.length : 0;
|
|
16
|
+
return SUBAGENT_SPINNER_FRAMES[(spinnerBucket(now) + offset) % SUBAGENT_SPINNER_FRAMES.length] ?? SUBAGENT_SPINNER_FRAMES[0];
|
|
17
|
+
}
|