pi-crew 0.1.41 → 0.1.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +51 -0
  3. package/agents/analyst.md +11 -11
  4. package/agents/critic.md +11 -11
  5. package/agents/executor.md +11 -11
  6. package/agents/explorer.md +11 -11
  7. package/agents/planner.md +11 -11
  8. package/agents/reviewer.md +11 -11
  9. package/agents/security-reviewer.md +11 -11
  10. package/agents/test-engineer.md +11 -11
  11. package/agents/verifier.md +11 -11
  12. package/agents/writer.md +11 -11
  13. package/docs/refactor-tasks-phase3.md +394 -394
  14. package/docs/refactor-tasks-phase4.md +564 -564
  15. package/docs/refactor-tasks-phase5.md +402 -402
  16. package/docs/refactor-tasks-phase6.md +662 -662
  17. package/docs/research-extension-examples.md +297 -297
  18. package/docs/research-extension-system.md +324 -324
  19. package/docs/research-optimization-plan.md +548 -548
  20. package/docs/research-phase10-distillation.md +199 -0
  21. package/docs/research-phase11-distillation.md +201 -0
  22. package/docs/research-pi-coding-agent.md +357 -357
  23. package/docs/research-source-pi-crew-reference.md +174 -174
  24. package/docs/runtime-flow.md +148 -148
  25. package/docs/source-runtime-refactor-map.md +83 -83
  26. package/index.ts +6 -6
  27. package/package.json +1 -1
  28. package/src/agents/agent-serializer.ts +34 -34
  29. package/src/agents/discover-agents.ts +5 -4
  30. package/src/config/config.ts +28 -4
  31. package/src/extension/cross-extension-rpc.ts +82 -82
  32. package/src/extension/management.ts +37 -8
  33. package/src/extension/notification-router.ts +2 -2
  34. package/src/extension/register.ts +130 -8
  35. package/src/extension/registration/commands.ts +11 -9
  36. package/src/extension/registration/compaction-guard.ts +125 -125
  37. package/src/extension/registration/subagent-tools.ts +28 -19
  38. package/src/extension/registration/team-tool.ts +2 -1
  39. package/src/extension/result-watcher.ts +4 -4
  40. package/src/extension/run-bundle-schema.ts +8 -4
  41. package/src/extension/run-import.ts +4 -0
  42. package/src/extension/run-index.ts +23 -1
  43. package/src/extension/run-maintenance.ts +43 -24
  44. package/src/extension/team-tool/api.ts +2 -2
  45. package/src/extension/team-tool/cancel.ts +76 -4
  46. package/src/extension/team-tool/context.ts +1 -0
  47. package/src/extension/team-tool/doctor.ts +8 -1
  48. package/src/extension/team-tool/handle-settings.ts +188 -0
  49. package/src/extension/team-tool/inspect.ts +41 -41
  50. package/src/extension/team-tool/lifecycle-actions.ts +79 -79
  51. package/src/extension/team-tool/plan.ts +19 -19
  52. package/src/extension/team-tool/respond.ts +67 -0
  53. package/src/extension/team-tool/run.ts +6 -4
  54. package/src/extension/team-tool/status.ts +99 -93
  55. package/src/extension/team-tool-types.ts +4 -0
  56. package/src/extension/team-tool.ts +5 -1
  57. package/src/i18n.ts +184 -0
  58. package/src/observability/correlation.ts +2 -2
  59. package/src/observability/event-to-metric.ts +10 -3
  60. package/src/observability/exporters/adapter.ts +7 -1
  61. package/src/observability/exporters/otlp-exporter.ts +14 -2
  62. package/src/observability/exporters/prometheus-exporter.ts +9 -2
  63. package/src/observability/metric-registry.ts +18 -3
  64. package/src/observability/metric-retention.ts +11 -3
  65. package/src/observability/metric-sink.ts +9 -4
  66. package/src/observability/metrics-primitives.ts +4 -3
  67. package/src/prompt/prompt-runtime.ts +72 -68
  68. package/src/runtime/agent-control.ts +63 -63
  69. package/src/runtime/agent-memory.ts +72 -72
  70. package/src/runtime/agent-observability.ts +114 -114
  71. package/src/runtime/async-marker.ts +26 -26
  72. package/src/runtime/attention-events.ts +28 -23
  73. package/src/runtime/background-runner.ts +53 -53
  74. package/src/runtime/child-pi.ts +4 -4
  75. package/src/runtime/completion-guard.ts +95 -4
  76. package/src/runtime/concurrency.ts +1 -1
  77. package/src/runtime/crash-recovery.ts +32 -1
  78. package/src/runtime/crew-agent-runtime.ts +59 -58
  79. package/src/runtime/deadletter.ts +14 -4
  80. package/src/runtime/delivery-coordinator.ts +143 -0
  81. package/src/runtime/direct-run.ts +35 -35
  82. package/src/runtime/foreground-control.ts +82 -82
  83. package/src/runtime/green-contract.ts +46 -46
  84. package/src/runtime/group-join.ts +106 -106
  85. package/src/runtime/heartbeat-gradient.ts +28 -28
  86. package/src/runtime/heartbeat-watcher.ts +48 -4
  87. package/src/runtime/live-agent-control.ts +87 -87
  88. package/src/runtime/live-agent-manager.ts +85 -85
  89. package/src/runtime/live-control-realtime.ts +36 -36
  90. package/src/runtime/live-session-runtime.ts +305 -305
  91. package/src/runtime/manifest-cache.ts +2 -2
  92. package/src/runtime/model-fallback.ts +272 -261
  93. package/src/runtime/overflow-recovery.ts +157 -0
  94. package/src/runtime/parallel-research.ts +44 -44
  95. package/src/runtime/parallel-utils.ts +1 -1
  96. package/src/runtime/pi-json-output.ts +111 -111
  97. package/src/runtime/policy-engine.ts +79 -78
  98. package/src/runtime/post-exit-stdio-guard.ts +2 -2
  99. package/src/runtime/process-status.ts +56 -56
  100. package/src/runtime/progress-event-coalescer.ts +43 -43
  101. package/src/runtime/recovery-recipes.ts +74 -74
  102. package/src/runtime/retry-executor.ts +5 -0
  103. package/src/runtime/role-permission.ts +39 -39
  104. package/src/runtime/runtime-resolver.ts +1 -1
  105. package/src/runtime/session-resources.ts +25 -0
  106. package/src/runtime/session-snapshot.ts +59 -0
  107. package/src/runtime/session-usage.ts +79 -79
  108. package/src/runtime/sidechain-output.ts +29 -29
  109. package/src/runtime/stale-reconciler.ts +179 -0
  110. package/src/runtime/subagent-manager.ts +3 -3
  111. package/src/runtime/supervisor-contact.ts +59 -0
  112. package/src/runtime/task-display.ts +38 -38
  113. package/src/runtime/task-output-context.ts +127 -127
  114. package/src/runtime/task-runner/live-executor.ts +101 -101
  115. package/src/runtime/task-runner/progress.ts +119 -111
  116. package/src/runtime/task-runner/result-utils.ts +14 -14
  117. package/src/runtime/task-runner/state-helpers.ts +22 -22
  118. package/src/runtime/task-runner.ts +14 -0
  119. package/src/runtime/team-runner.ts +9 -10
  120. package/src/runtime/worker-heartbeat.ts +21 -21
  121. package/src/runtime/worker-startup.ts +57 -57
  122. package/src/schema/config-schema.ts +2 -1
  123. package/src/schema/team-tool-schema.ts +115 -109
  124. package/src/state/artifact-store.ts +4 -2
  125. package/src/state/atomic-write.ts +12 -4
  126. package/src/state/contracts.ts +109 -105
  127. package/src/state/event-log.ts +3 -4
  128. package/src/state/jsonl-writer.ts +4 -1
  129. package/src/state/locks.ts +9 -1
  130. package/src/state/task-claims.ts +44 -42
  131. package/src/state/usage.ts +29 -29
  132. package/src/subagents/async-entry.ts +1 -1
  133. package/src/subagents/index.ts +3 -3
  134. package/src/subagents/live/control.ts +1 -1
  135. package/src/subagents/live/manager.ts +1 -1
  136. package/src/subagents/live/realtime.ts +1 -1
  137. package/src/subagents/live/session-runtime.ts +1 -1
  138. package/src/subagents/manager.ts +1 -1
  139. package/src/subagents/spawn.ts +1 -1
  140. package/src/teams/discover-teams.ts +2 -2
  141. package/src/teams/team-serializer.ts +38 -38
  142. package/src/types/diff.d.ts +18 -18
  143. package/src/ui/crew-footer.ts +101 -101
  144. package/src/ui/crew-select-list.ts +111 -111
  145. package/src/ui/crew-widget.ts +5 -4
  146. package/src/ui/dashboard-panes/metrics-pane.ts +34 -34
  147. package/src/ui/dynamic-border.ts +25 -25
  148. package/src/ui/layout-primitives.ts +106 -106
  149. package/src/ui/live-run-sidebar.ts +1 -1
  150. package/src/ui/loaders.ts +158 -158
  151. package/src/ui/mascot.ts +3 -2
  152. package/src/ui/powerbar-publisher.ts +7 -6
  153. package/src/ui/render-diff.ts +119 -119
  154. package/src/ui/render-scheduler.ts +54 -14
  155. package/src/ui/run-dashboard.ts +39 -11
  156. package/src/ui/run-snapshot-cache.ts +336 -36
  157. package/src/ui/spinner.ts +17 -17
  158. package/src/ui/status-colors.ts +58 -54
  159. package/src/ui/syntax-highlight.ts +116 -116
  160. package/src/ui/theme-adapter.ts +1 -1
  161. package/src/ui/transcript-viewer.ts +7 -2
  162. package/src/utils/atomic-write.ts +33 -0
  163. package/src/utils/completion-dedupe.ts +63 -63
  164. package/src/utils/file-coalescer.ts +5 -3
  165. package/src/utils/frontmatter.ts +68 -36
  166. package/src/utils/git.ts +262 -262
  167. package/src/utils/ids.ts +12 -12
  168. package/src/utils/internal-error.ts +1 -1
  169. package/src/utils/names.ts +27 -26
  170. package/src/utils/paths.ts +1 -1
  171. package/src/utils/redaction.ts +44 -41
  172. package/src/utils/safe-paths.ts +47 -34
  173. package/src/utils/sleep.ts +2 -2
  174. package/src/utils/timings.ts +2 -0
  175. package/src/utils/visual.ts +9 -1
  176. package/src/workflows/discover-workflows.ts +4 -1
  177. package/src/workflows/validate-workflow.ts +40 -40
  178. package/src/worktree/branch-freshness.ts +45 -45
  179. package/src/worktree/worktree-manager.ts +6 -1
  180. package/teams/default.team.md +12 -12
  181. package/teams/fast-fix.team.md +11 -11
  182. package/teams/implementation.team.md +18 -18
  183. package/teams/parallel-research.team.md +14 -14
  184. package/teams/research.team.md +11 -11
  185. package/teams/review.team.md +12 -12
  186. package/workflows/default.workflow.md +29 -29
  187. package/workflows/fast-fix.workflow.md +22 -22
  188. package/workflows/implementation.workflow.md +38 -38
  189. package/workflows/parallel-research.workflow.md +46 -46
  190. package/workflows/research.workflow.md +22 -22
  191. package/workflows/review.workflow.md +30 -30
@@ -1,46 +1,46 @@
1
- import type { GreenLevel, VerificationContract, VerificationEvidence } from "../state/types.ts";
2
-
3
- const GREEN_ORDER: Record<GreenLevel, number> = {
4
- none: 0,
5
- targeted: 1,
6
- package: 2,
7
- workspace: 3,
8
- merge_ready: 4,
9
- };
10
-
11
- export interface GreenContractOutcome {
12
- requiredGreenLevel: GreenLevel;
13
- observedGreenLevel: GreenLevel;
14
- satisfied: boolean;
15
- }
16
-
17
- export function greenLevelSatisfies(observed: GreenLevel, required: GreenLevel): boolean {
18
- return GREEN_ORDER[observed] >= GREEN_ORDER[required];
19
- }
20
-
21
- export function evaluateGreenContract(contract: VerificationContract, evidence?: VerificationEvidence): GreenContractOutcome {
22
- const observedGreenLevel = evidence?.observedGreenLevel ?? "none";
23
- return {
24
- requiredGreenLevel: contract.requiredGreenLevel,
25
- observedGreenLevel,
26
- satisfied: greenLevelSatisfies(observedGreenLevel, contract.requiredGreenLevel),
27
- };
28
- }
29
-
30
- export function inferGreenLevelFromTask(success: boolean, contract: VerificationContract): GreenLevel {
31
- if (!success) return "none";
32
- if (contract.requiredGreenLevel === "none") return "none";
33
- return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
34
- }
35
-
36
- export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
37
- const observedGreenLevel = inferGreenLevelFromTask(success, contract);
38
- const outcome = evaluateGreenContract(contract, { requiredGreenLevel: contract.requiredGreenLevel, observedGreenLevel, satisfied: false, commands: [], notes });
39
- return {
40
- requiredGreenLevel: contract.requiredGreenLevel,
41
- observedGreenLevel,
42
- satisfied: outcome.satisfied,
43
- commands: contract.commands.map((cmd) => ({ cmd, status: "not_run" as const })),
44
- notes,
45
- };
46
- }
1
+ import type { GreenLevel, VerificationContract, VerificationEvidence } from "../state/types.ts";
2
+
3
+ const GREEN_ORDER: Record<GreenLevel, number> = {
4
+ none: 0,
5
+ targeted: 1,
6
+ package: 2,
7
+ workspace: 3,
8
+ merge_ready: 4,
9
+ };
10
+
11
+ export interface GreenContractOutcome {
12
+ requiredGreenLevel: GreenLevel;
13
+ observedGreenLevel: GreenLevel;
14
+ satisfied: boolean;
15
+ }
16
+
17
+ export function greenLevelSatisfies(observed: GreenLevel, required: GreenLevel): boolean {
18
+ return GREEN_ORDER[observed] >= GREEN_ORDER[required];
19
+ }
20
+
21
+ export function evaluateGreenContract(contract: VerificationContract, evidence?: VerificationEvidence): GreenContractOutcome {
22
+ const observedGreenLevel = evidence?.observedGreenLevel ?? "none";
23
+ return {
24
+ requiredGreenLevel: contract.requiredGreenLevel,
25
+ observedGreenLevel,
26
+ satisfied: greenLevelSatisfies(observedGreenLevel, contract.requiredGreenLevel),
27
+ };
28
+ }
29
+
30
+ export function inferGreenLevelFromTask(success: boolean, contract: VerificationContract): GreenLevel {
31
+ if (!success) return "none";
32
+ if (contract.requiredGreenLevel === "none") return "none";
33
+ return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
34
+ }
35
+
36
+ export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
37
+ const observedGreenLevel = inferGreenLevelFromTask(success, contract);
38
+ const outcome = evaluateGreenContract(contract, { requiredGreenLevel: contract.requiredGreenLevel, observedGreenLevel, satisfied: false, commands: [], notes });
39
+ return {
40
+ requiredGreenLevel: contract.requiredGreenLevel,
41
+ observedGreenLevel,
42
+ satisfied: outcome.satisfied,
43
+ commands: contract.commands.map((cmd) => ({ cmd, status: "not_run" as const })),
44
+ notes,
45
+ };
46
+ }
@@ -1,106 +1,106 @@
1
- import type { CrewRuntimeConfig } from "../config/config.ts";
2
- import { writeArtifact } from "../state/artifact-store.ts";
3
- import { appendEvent } from "../state/event-log.ts";
4
- import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
5
- import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
6
- import { aggregateTaskOutputs } from "./task-output-context.ts";
7
-
8
- export type CrewGroupJoinMode = "off" | "group" | "smart";
9
-
10
- export interface CrewGroupJoinDelivery {
11
- batchId: string;
12
- mode: CrewGroupJoinMode;
13
- partial: boolean;
14
- taskIds: string[];
15
- completed: string[];
16
- failed: string[];
17
- skipped: string[];
18
- remaining: string[];
19
- artifact?: ArtifactDescriptor;
20
- messageId?: string;
21
- requestId?: string;
22
- ackRequired?: boolean;
23
- ackStatus?: "pending" | "acknowledged";
24
- }
25
-
26
- export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
27
- return runtime?.groupJoin ?? "smart";
28
- }
29
-
30
- export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
31
- if (mode === "off") return false;
32
- if (mode === "group") return batch.length > 0;
33
- return batch.length > 1;
34
- }
35
-
36
- function batchIdFor(runId: string, taskIds: string[]): string {
37
- return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
38
- }
39
-
40
- function requestIdFor(runId: string, batchId: string, partial: boolean): string {
41
- return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
42
- }
43
-
44
- function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
45
- return tasks.filter((task) => task.status === status).map((task) => task.id);
46
- }
47
-
48
- export function deliverGroupJoin(input: {
49
- manifest: TeamRunManifest;
50
- mode: CrewGroupJoinMode;
51
- batch: TeamTaskState[];
52
- allTasks: TeamTaskState[];
53
- partial?: boolean;
54
- }): CrewGroupJoinDelivery | undefined {
55
- if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
56
- const taskIds = input.batch.map((task) => task.id);
57
- const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
58
- const completed = statusList(latest, "completed");
59
- const failed = statusList(latest, "failed");
60
- const skipped = statusList(latest, "skipped");
61
- const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
62
- const partial = input.partial ?? remaining.length > 0;
63
- const batchId = batchIdFor(input.manifest.runId, taskIds);
64
- const summary = aggregateTaskOutputs(latest, input.manifest);
65
- const requestId = requestIdFor(input.manifest.runId, batchId, partial);
66
- const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
67
- const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
68
- const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
69
- const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
70
- const artifact = writeArtifact(input.manifest.artifactsRoot, {
71
- kind: "metadata",
72
- relativePath: `metadata/group-joins/${batchId}.json`,
73
- producer: "group-join",
74
- content,
75
- });
76
- const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
77
- direction: "outbox",
78
- from: "group-join",
79
- to: "leader",
80
- body: [
81
- `Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
82
- `Request: ${requestId}`,
83
- `Completed: ${completed.join(", ") || "none"}`,
84
- `Failed: ${failed.join(", ") || "none"}`,
85
- `Skipped: ${skipped.join(", ") || "none"}`,
86
- `Remaining: ${remaining.join(", ") || "none"}`,
87
- "",
88
- summary,
89
- ].join("\n"),
90
- status: "delivered",
91
- data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
92
- });
93
- appendEvent(input.manifest.eventsPath, {
94
- type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
95
- runId: input.manifest.runId,
96
- message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
97
- data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
98
- });
99
- if (existingMailbox) appendEvent(input.manifest.eventsPath, {
100
- type: "agent.group_join.delivery_reused",
101
- runId: input.manifest.runId,
102
- message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
103
- data: { requestId, messageId: mailbox.id, batchId, partial },
104
- });
105
- return { ...delivery, artifact, messageId: mailbox.id };
106
- }
1
+ import type { CrewRuntimeConfig } from "../config/config.ts";
2
+ import { writeArtifact } from "../state/artifact-store.ts";
3
+ import { appendEvent } from "../state/event-log.ts";
4
+ import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
5
+ import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
6
+ import { aggregateTaskOutputs } from "./task-output-context.ts";
7
+
8
+ export type CrewGroupJoinMode = "off" | "group" | "smart";
9
+
10
+ export interface CrewGroupJoinDelivery {
11
+ batchId: string;
12
+ mode: CrewGroupJoinMode;
13
+ partial: boolean;
14
+ taskIds: string[];
15
+ completed: string[];
16
+ failed: string[];
17
+ skipped: string[];
18
+ remaining: string[];
19
+ artifact?: ArtifactDescriptor;
20
+ messageId?: string;
21
+ requestId?: string;
22
+ ackRequired?: boolean;
23
+ ackStatus?: "pending" | "acknowledged";
24
+ }
25
+
26
+ export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
27
+ return runtime?.groupJoin ?? "smart";
28
+ }
29
+
30
+ export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
31
+ if (mode === "off") return false;
32
+ if (mode === "group") return batch.length > 0;
33
+ return batch.length > 1;
34
+ }
35
+
36
+ function batchIdFor(runId: string, taskIds: string[]): string {
37
+ return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
38
+ }
39
+
40
+ function requestIdFor(runId: string, batchId: string, partial: boolean): string {
41
+ return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
42
+ }
43
+
44
+ function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
45
+ return tasks.filter((task) => task.status === status).map((task) => task.id);
46
+ }
47
+
48
+ export function deliverGroupJoin(input: {
49
+ manifest: TeamRunManifest;
50
+ mode: CrewGroupJoinMode;
51
+ batch: TeamTaskState[];
52
+ allTasks: TeamTaskState[];
53
+ partial?: boolean;
54
+ }): CrewGroupJoinDelivery | undefined {
55
+ if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
56
+ const taskIds = input.batch.map((task) => task.id);
57
+ const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
58
+ const completed = statusList(latest, "completed");
59
+ const failed = statusList(latest, "failed");
60
+ const skipped = statusList(latest, "skipped");
61
+ const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
62
+ const partial = input.partial ?? remaining.length > 0;
63
+ const batchId = batchIdFor(input.manifest.runId, taskIds);
64
+ const summary = aggregateTaskOutputs(latest, input.manifest);
65
+ const requestId = requestIdFor(input.manifest.runId, batchId, partial);
66
+ const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
67
+ const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
68
+ const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
69
+ const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
70
+ const artifact = writeArtifact(input.manifest.artifactsRoot, {
71
+ kind: "metadata",
72
+ relativePath: `metadata/group-joins/${batchId}.json`,
73
+ producer: "group-join",
74
+ content,
75
+ });
76
+ const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
77
+ direction: "outbox",
78
+ from: "group-join",
79
+ to: "leader",
80
+ body: [
81
+ `Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
82
+ `Request: ${requestId}`,
83
+ `Completed: ${completed.join(", ") || "none"}`,
84
+ `Failed: ${failed.join(", ") || "none"}`,
85
+ `Skipped: ${skipped.join(", ") || "none"}`,
86
+ `Remaining: ${remaining.join(", ") || "none"}`,
87
+ "",
88
+ summary,
89
+ ].join("\n"),
90
+ status: "delivered",
91
+ data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
92
+ });
93
+ appendEvent(input.manifest.eventsPath, {
94
+ type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
95
+ runId: input.manifest.runId,
96
+ message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
97
+ data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
98
+ });
99
+ if (existingMailbox) appendEvent(input.manifest.eventsPath, {
100
+ type: "agent.group_join.delivery_reused",
101
+ runId: input.manifest.runId,
102
+ message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
103
+ data: { requestId, messageId: mailbox.id, batchId, partial },
104
+ });
105
+ return { ...delivery, artifact, messageId: mailbox.id };
106
+ }
@@ -1,28 +1,28 @@
1
- import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
2
-
3
- export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
4
-
5
- export interface GradientThresholds {
6
- warnMs: number;
7
- staleMs: number;
8
- deadMs: number;
9
- }
10
-
11
- export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
12
-
13
- export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
14
- if (!heartbeat) return Number.POSITIVE_INFINITY;
15
- const lastSeen = Date.parse(heartbeat.lastSeenAt);
16
- return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
17
- }
18
-
19
- export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
20
- if (!heartbeat) return "dead";
21
- if (heartbeat.alive === false) return "dead";
22
- const elapsed = heartbeatAgeMs(heartbeat, now);
23
- if (!Number.isFinite(elapsed)) return "dead";
24
- if (elapsed > thresholds.deadMs) return "dead";
25
- if (elapsed > thresholds.staleMs) return "stale";
26
- if (elapsed > thresholds.warnMs) return "warn";
27
- return "healthy";
28
- }
1
+ import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
2
+
3
+ export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
4
+
5
+ export interface GradientThresholds {
6
+ warnMs: number;
7
+ staleMs: number;
8
+ deadMs: number;
9
+ }
10
+
11
+ export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
12
+
13
+ export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
14
+ if (!heartbeat) return Number.POSITIVE_INFINITY;
15
+ const lastSeen = Date.parse(heartbeat.lastSeenAt);
16
+ return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
17
+ }
18
+
19
+ export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
20
+ if (!heartbeat) return "dead";
21
+ if (heartbeat.alive === false) return "dead";
22
+ const elapsed = heartbeatAgeMs(heartbeat, now);
23
+ if (!Number.isFinite(elapsed)) return "dead";
24
+ if (elapsed > thresholds.deadMs) return "dead";
25
+ if (elapsed > thresholds.staleMs) return "stale";
26
+ if (elapsed > thresholds.warnMs) return "warn";
27
+ return "healthy";
28
+ }
@@ -3,6 +3,7 @@ import type { MetricRegistry } from "../observability/metric-registry.ts";
3
3
  import { appendEvent } from "../state/event-log.ts";
4
4
  import { loadRunManifestById } from "../state/state-store.ts";
5
5
  import type { TeamRunManifest } from "../state/types.ts";
6
+ import { logInternalError } from "../utils/internal-error.ts";
6
7
  import type { ManifestCache } from "./manifest-cache.ts";
7
8
  import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
8
9
 
@@ -22,10 +23,21 @@ export interface HeartbeatWatcherOptions {
22
23
  onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
23
24
  }
24
25
 
26
+ /**
27
+ * Polls running runs for heartbeat staleness.
28
+ *
29
+ * Uses recursive setTimeout to avoid timer storms.
30
+ * Cleanup is done in the same pass — no second scan over manifests.
31
+ * Keys for runs that disappear from the cache are cleaned via staleness-age policy
32
+ * rather than being leaked forever.
33
+ */
25
34
  export class HeartbeatWatcher {
26
- private timer?: ReturnType<typeof setInterval>;
35
+ private timer?: ReturnType<typeof setTimeout>;
27
36
  private lastLevel = new Map<string, HeartbeatLevel>();
28
37
  private consecutiveDead = new Map<string, number>();
38
+ private lastSeen = new Map<string, number>(); // key → last time it was active
39
+ /** Max age (ms) to retain a stale key before garbage-collecting it. */
40
+ private readonly maxKeyAgeMs = 600_000; // 10 minutes
29
41
  private readonly opts: HeartbeatWatcherOptions;
30
42
 
31
43
  constructor(opts: HeartbeatWatcherOptions) {
@@ -34,13 +46,29 @@ export class HeartbeatWatcher {
34
46
 
35
47
  start(): void {
36
48
  this.dispose();
37
- this.timer = setInterval(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
38
- this.timer.unref?.();
49
+ this.scheduleTick();
50
+ }
51
+
52
+ private scheduleTick(): void {
53
+ this.timer = setTimeout(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
54
+ this.timer.unref();
39
55
  }
40
56
 
41
57
  tick(now = Date.now()): void {
58
+ try {
59
+ this.tickUnsafe(now);
60
+ } catch (error) {
61
+ logInternalError("heartbeat-watcher.tick", error);
62
+ } finally {
63
+ this.scheduleTick();
64
+ }
65
+ }
66
+
67
+ private tickUnsafe(now: number): void {
42
68
  const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
43
69
  const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
70
+ const activeKeys = new Set<string>();
71
+
44
72
  for (const run of this.opts.manifestCache.list(50)) {
45
73
  if (run.status !== "running") continue;
46
74
  const loaded = loadRunManifestById(this.opts.cwd, run.runId);
@@ -48,6 +76,9 @@ export class HeartbeatWatcher {
48
76
  for (const task of loaded.tasks) {
49
77
  if (task.status !== "running") continue;
50
78
  const key = `${run.runId}:${task.id}`;
79
+ activeKeys.add(key);
80
+ this.lastSeen.set(key, now);
81
+
51
82
  const elapsed = heartbeatAgeMs(task.heartbeat, now);
52
83
  const level = classifyHeartbeat(task.heartbeat, thresholds, now);
53
84
  this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
@@ -69,12 +100,25 @@ export class HeartbeatWatcher {
69
100
  }
70
101
  }
71
102
  }
103
+
104
+ // Cleanup: drop keys that were NOT in this tick's active set AND
105
+ // haven't been seen for > maxKeyAgeMs. This covers runs that
106
+ // completed or fell out of the manifest cache's top-50 window.
107
+ const cutoff = now - this.maxKeyAgeMs;
108
+ for (const [key, ts] of this.lastSeen) {
109
+ if (!activeKeys.has(key) && ts < cutoff) {
110
+ this.lastLevel.delete(key);
111
+ this.consecutiveDead.delete(key);
112
+ this.lastSeen.delete(key);
113
+ }
114
+ }
72
115
  }
73
116
 
74
117
  dispose(): void {
75
- if (this.timer) clearInterval(this.timer);
118
+ if (this.timer) clearTimeout(this.timer);
76
119
  this.timer = undefined;
77
120
  this.lastLevel.clear();
78
121
  this.consecutiveDead.clear();
122
+ this.lastSeen.clear();
79
123
  }
80
124
  }
@@ -1,87 +1,87 @@
1
- import * as fs from "node:fs";
2
- import * as path from "node:path";
3
- import type { TeamRunManifest } from "../state/types.ts";
4
- import { agentStateFile, ensureAgentStateDir } from "./crew-agent-records.ts";
5
-
6
- export type LiveAgentControlOperation = "steer" | "stop" | "resume";
7
-
8
- export interface LiveAgentControlRequest {
9
- id: string;
10
- runId: string;
11
- taskId: string;
12
- agentId?: string;
13
- operation: LiveAgentControlOperation;
14
- message?: string;
15
- createdAt: string;
16
- processedAt?: string;
17
- error?: string;
18
- }
19
-
20
- export interface LiveAgentControlCursor {
21
- offset: number;
22
- }
23
-
24
- export function liveAgentControlPath(manifest: TeamRunManifest, taskId: string): string {
25
- return path.join(ensureAgentStateDir(manifest, taskId), "live-control.jsonl");
26
- }
27
-
28
- function liveAgentControlFile(manifest: TeamRunManifest, taskId: string): string {
29
- return agentStateFile(manifest, taskId, "live-control.jsonl");
30
- }
31
-
32
- function requestId(): string {
33
- return `ctrl_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`;
34
- }
35
-
36
- export function appendLiveAgentControlRequest(manifest: TeamRunManifest, input: { taskId: string; agentId?: string; operation: LiveAgentControlOperation; message?: string }): LiveAgentControlRequest {
37
- const request: LiveAgentControlRequest = {
38
- id: requestId(),
39
- runId: manifest.runId,
40
- taskId: input.taskId,
41
- agentId: input.agentId,
42
- operation: input.operation,
43
- message: input.message,
44
- createdAt: new Date().toISOString(),
45
- };
46
- const filePath = liveAgentControlFile(manifest, input.taskId);
47
- fs.appendFileSync(filePath, `${JSON.stringify(request)}\n`, "utf-8");
48
- return request;
49
- }
50
-
51
- export function readLiveAgentControlRequests(manifest: TeamRunManifest, taskId: string, cursor: LiveAgentControlCursor = { offset: 0 }): { requests: LiveAgentControlRequest[]; cursor: LiveAgentControlCursor } {
52
- let filePath: string;
53
- try {
54
- filePath = liveAgentControlFile(manifest, taskId);
55
- } catch {
56
- return { requests: [], cursor };
57
- }
58
- if (!fs.existsSync(filePath)) return { requests: [], cursor };
59
- const text = fs.readFileSync(filePath, "utf-8");
60
- const lines = text.split(/\r?\n/).filter(Boolean);
61
- const requests = lines.slice(cursor.offset).flatMap((line) => {
62
- try {
63
- const parsed = JSON.parse(line) as LiveAgentControlRequest;
64
- return parsed && parsed.runId === manifest.runId && parsed.taskId === taskId ? [parsed] : [];
65
- } catch {
66
- return [];
67
- }
68
- });
69
- return { requests, cursor: { offset: lines.length } };
70
- }
71
-
72
- export async function applyLiveAgentControlRequest(input: { request: LiveAgentControlRequest; taskId: string; agentId: string; session: { steer?: (text: string) => Promise<void>; prompt?: (text: string, options?: Record<string, unknown>) => Promise<void>; abort?: () => Promise<void> | void }; seenRequestIds?: Set<string> }): Promise<boolean> {
73
- const { request, taskId, agentId, session, seenRequestIds } = input;
74
- if (seenRequestIds?.has(request.id)) return false;
75
- if (request.agentId && request.agentId !== agentId && request.agentId !== taskId) return false;
76
- seenRequestIds?.add(request.id);
77
- if (request.operation === "steer") await session.steer?.(request.message ?? "Please report current status and wrap up if possible.");
78
- else if (request.operation === "resume") await session.prompt?.(request.message ?? "Please resume and report final status.", { source: "api", expandPromptTemplates: false });
79
- else if (request.operation === "stop") await session.abort?.();
80
- return true;
81
- }
82
-
83
- export async function applyLiveAgentControlRequests(input: { manifest: TeamRunManifest; taskId: string; agentId: string; session: { steer?: (text: string) => Promise<void>; prompt?: (text: string, options?: Record<string, unknown>) => Promise<void>; abort?: () => Promise<void> | void }; cursor: LiveAgentControlCursor; seenRequestIds?: Set<string> }): Promise<LiveAgentControlCursor> {
84
- const batch = readLiveAgentControlRequests(input.manifest, input.taskId, input.cursor);
85
- for (const request of batch.requests) await applyLiveAgentControlRequest({ request, taskId: input.taskId, agentId: input.agentId, session: input.session, seenRequestIds: input.seenRequestIds });
86
- return batch.cursor;
87
- }
1
+ import * as fs from "node:fs";
2
+ import * as path from "node:path";
3
+ import type { TeamRunManifest } from "../state/types.ts";
4
+ import { agentStateFile, ensureAgentStateDir } from "./crew-agent-records.ts";
5
+
6
+ export type LiveAgentControlOperation = "steer" | "stop" | "resume";
7
+
8
+ export interface LiveAgentControlRequest {
9
+ id: string;
10
+ runId: string;
11
+ taskId: string;
12
+ agentId?: string;
13
+ operation: LiveAgentControlOperation;
14
+ message?: string;
15
+ createdAt: string;
16
+ processedAt?: string;
17
+ error?: string;
18
+ }
19
+
20
+ export interface LiveAgentControlCursor {
21
+ offset: number;
22
+ }
23
+
24
+ export function liveAgentControlPath(manifest: TeamRunManifest, taskId: string): string {
25
+ return path.join(ensureAgentStateDir(manifest, taskId), "live-control.jsonl");
26
+ }
27
+
28
+ function liveAgentControlFile(manifest: TeamRunManifest, taskId: string): string {
29
+ return agentStateFile(manifest, taskId, "live-control.jsonl");
30
+ }
31
+
32
+ function requestId(): string {
33
+ return `ctrl_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`;
34
+ }
35
+
36
+ export function appendLiveAgentControlRequest(manifest: TeamRunManifest, input: { taskId: string; agentId?: string; operation: LiveAgentControlOperation; message?: string }): LiveAgentControlRequest {
37
+ const request: LiveAgentControlRequest = {
38
+ id: requestId(),
39
+ runId: manifest.runId,
40
+ taskId: input.taskId,
41
+ agentId: input.agentId,
42
+ operation: input.operation,
43
+ message: input.message,
44
+ createdAt: new Date().toISOString(),
45
+ };
46
+ const filePath = liveAgentControlFile(manifest, input.taskId);
47
+ fs.appendFileSync(filePath, `${JSON.stringify(request)}\n`, "utf-8");
48
+ return request;
49
+ }
50
+
51
+ export function readLiveAgentControlRequests(manifest: TeamRunManifest, taskId: string, cursor: LiveAgentControlCursor = { offset: 0 }): { requests: LiveAgentControlRequest[]; cursor: LiveAgentControlCursor } {
52
+ let filePath: string;
53
+ try {
54
+ filePath = liveAgentControlFile(manifest, taskId);
55
+ } catch {
56
+ return { requests: [], cursor };
57
+ }
58
+ if (!fs.existsSync(filePath)) return { requests: [], cursor };
59
+ const text = fs.readFileSync(filePath, "utf-8");
60
+ const lines = text.split(/\r?\n/).filter(Boolean);
61
+ const requests = lines.slice(cursor.offset).flatMap((line) => {
62
+ try {
63
+ const parsed = JSON.parse(line) as LiveAgentControlRequest;
64
+ return parsed && parsed.runId === manifest.runId && parsed.taskId === taskId ? [parsed] : [];
65
+ } catch {
66
+ return [];
67
+ }
68
+ });
69
+ return { requests, cursor: { offset: lines.length } };
70
+ }
71
+
72
+ export async function applyLiveAgentControlRequest(input: { request: LiveAgentControlRequest; taskId: string; agentId: string; session: { steer?: (text: string) => Promise<void>; prompt?: (text: string, options?: Record<string, unknown>) => Promise<void>; abort?: () => Promise<void> | void }; seenRequestIds?: Set<string> }): Promise<boolean> {
73
+ const { request, taskId, agentId, session, seenRequestIds } = input;
74
+ if (seenRequestIds?.has(request.id)) return false;
75
+ if (request.agentId && request.agentId !== agentId && request.agentId !== taskId) return false;
76
+ seenRequestIds?.add(request.id);
77
+ if (request.operation === "steer") await session.steer?.(request.message ?? "Please report current status and wrap up if possible.");
78
+ else if (request.operation === "resume") await session.prompt?.(request.message ?? "Please resume and report final status.", { source: "api", expandPromptTemplates: false });
79
+ else if (request.operation === "stop") await session.abort?.();
80
+ return true;
81
+ }
82
+
83
+ export async function applyLiveAgentControlRequests(input: { manifest: TeamRunManifest; taskId: string; agentId: string; session: { steer?: (text: string) => Promise<void>; prompt?: (text: string, options?: Record<string, unknown>) => Promise<void>; abort?: () => Promise<void> | void }; cursor: LiveAgentControlCursor; seenRequestIds?: Set<string> }): Promise<LiveAgentControlCursor> {
84
+ const batch = readLiveAgentControlRequests(input.manifest, input.taskId, input.cursor);
85
+ for (const request of batch.requests) await applyLiveAgentControlRequest({ request, taskId: input.taskId, agentId: input.agentId, session: input.session, seenRequestIds: input.seenRequestIds });
86
+ return batch.cursor;
87
+ }