pi-amq 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/CHANGELOG.md +66 -0
  2. package/LICENSE +21 -0
  3. package/README.md +92 -0
  4. package/THIRD_PARTY_NOTICES.md +33 -0
  5. package/dist/extension/guidance.d.ts +25 -0
  6. package/dist/extension/guidance.d.ts.map +1 -0
  7. package/dist/extension/guidance.js +121 -0
  8. package/dist/extension/guidance.js.map +1 -0
  9. package/dist/extension/guide.d.ts +30 -0
  10. package/dist/extension/guide.d.ts.map +1 -0
  11. package/dist/extension/guide.js +128 -0
  12. package/dist/extension/guide.js.map +1 -0
  13. package/dist/extension/index.d.ts +4 -0
  14. package/dist/extension/index.d.ts.map +1 -0
  15. package/dist/extension/index.js +6 -0
  16. package/dist/extension/index.js.map +1 -0
  17. package/dist/extension/runtime.d.ts +48 -0
  18. package/dist/extension/runtime.d.ts.map +1 -0
  19. package/dist/extension/runtime.js +1095 -0
  20. package/dist/extension/runtime.js.map +1 -0
  21. package/dist/extension/tools.d.ts +12 -0
  22. package/dist/extension/tools.d.ts.map +1 -0
  23. package/dist/extension/tools.js +198 -0
  24. package/dist/extension/tools.js.map +1 -0
  25. package/dist/extension/worker-runtime.d.ts +34 -0
  26. package/dist/extension/worker-runtime.d.ts.map +1 -0
  27. package/dist/extension/worker-runtime.js +974 -0
  28. package/dist/extension/worker-runtime.js.map +1 -0
  29. package/dist/extension/worker-tools.d.ts +140 -0
  30. package/dist/extension/worker-tools.d.ts.map +1 -0
  31. package/dist/extension/worker-tools.js +217 -0
  32. package/dist/extension/worker-tools.js.map +1 -0
  33. package/dist/integration/confirmed-bind-onboarding.d.ts +44 -0
  34. package/dist/integration/confirmed-bind-onboarding.d.ts.map +1 -0
  35. package/dist/integration/confirmed-bind-onboarding.js +199 -0
  36. package/dist/integration/confirmed-bind-onboarding.js.map +1 -0
  37. package/dist/integration/fleet-view.d.ts +66 -0
  38. package/dist/integration/fleet-view.d.ts.map +1 -0
  39. package/dist/integration/fleet-view.js +238 -0
  40. package/dist/integration/fleet-view.js.map +1 -0
  41. package/dist/integration/fresh-session-dispatcher.d.ts +66 -0
  42. package/dist/integration/fresh-session-dispatcher.d.ts.map +1 -0
  43. package/dist/integration/fresh-session-dispatcher.js +103 -0
  44. package/dist/integration/fresh-session-dispatcher.js.map +1 -0
  45. package/dist/integration/launch-preview.d.ts +31 -0
  46. package/dist/integration/launch-preview.d.ts.map +1 -0
  47. package/dist/integration/launch-preview.js +325 -0
  48. package/dist/integration/launch-preview.js.map +1 -0
  49. package/dist/integration/transport-port.d.ts +39 -0
  50. package/dist/integration/transport-port.d.ts.map +1 -0
  51. package/dist/integration/transport-port.js +2 -0
  52. package/dist/integration/transport-port.js.map +1 -0
  53. package/dist/integration/worker-completion-notifier.d.ts +22 -0
  54. package/dist/integration/worker-completion-notifier.d.ts.map +1 -0
  55. package/dist/integration/worker-completion-notifier.js +81 -0
  56. package/dist/integration/worker-completion-notifier.js.map +1 -0
  57. package/dist/transport/amq-cli-adapter.d.ts +41 -0
  58. package/dist/transport/amq-cli-adapter.d.ts.map +1 -0
  59. package/dist/transport/amq-cli-adapter.js +291 -0
  60. package/dist/transport/amq-cli-adapter.js.map +1 -0
  61. package/dist/transport/amq-message.d.ts +47 -0
  62. package/dist/transport/amq-message.d.ts.map +1 -0
  63. package/dist/transport/amq-message.js +173 -0
  64. package/dist/transport/amq-message.js.map +1 -0
  65. package/dist/transport/amq-operations.d.ts +105 -0
  66. package/dist/transport/amq-operations.d.ts.map +1 -0
  67. package/dist/transport/amq-operations.js +320 -0
  68. package/dist/transport/amq-operations.js.map +1 -0
  69. package/dist/transport/atomic-json.d.ts +18 -0
  70. package/dist/transport/atomic-json.d.ts.map +1 -0
  71. package/dist/transport/atomic-json.js +90 -0
  72. package/dist/transport/atomic-json.js.map +1 -0
  73. package/dist/transport/binding-resolver.d.ts +46 -0
  74. package/dist/transport/binding-resolver.d.ts.map +1 -0
  75. package/dist/transport/binding-resolver.js +389 -0
  76. package/dist/transport/binding-resolver.js.map +1 -0
  77. package/dist/transport/bridge-journal.d.ts +100 -0
  78. package/dist/transport/bridge-journal.d.ts.map +1 -0
  79. package/dist/transport/bridge-journal.js +459 -0
  80. package/dist/transport/bridge-journal.js.map +1 -0
  81. package/dist/transport/dlq-retry-audit.d.ts +28 -0
  82. package/dist/transport/dlq-retry-audit.d.ts.map +1 -0
  83. package/dist/transport/dlq-retry-audit.js +66 -0
  84. package/dist/transport/dlq-retry-audit.js.map +1 -0
  85. package/dist/transport/inbox-pump.d.ts +81 -0
  86. package/dist/transport/inbox-pump.d.ts.map +1 -0
  87. package/dist/transport/inbox-pump.js +558 -0
  88. package/dist/transport/inbox-pump.js.map +1 -0
  89. package/dist/transport/index.d.ts +12 -0
  90. package/dist/transport/index.d.ts.map +1 -0
  91. package/dist/transport/index.js +12 -0
  92. package/dist/transport/index.js.map +1 -0
  93. package/dist/transport/presence-projector.d.ts +29 -0
  94. package/dist/transport/presence-projector.d.ts.map +1 -0
  95. package/dist/transport/presence-projector.js +73 -0
  96. package/dist/transport/presence-projector.js.map +1 -0
  97. package/dist/transport/priority-delivery-scheduler.d.ts +85 -0
  98. package/dist/transport/priority-delivery-scheduler.d.ts.map +1 -0
  99. package/dist/transport/priority-delivery-scheduler.js +210 -0
  100. package/dist/transport/priority-delivery-scheduler.js.map +1 -0
  101. package/dist/transport/receiver-lease.d.ts +102 -0
  102. package/dist/transport/receiver-lease.d.ts.map +1 -0
  103. package/dist/transport/receiver-lease.js +778 -0
  104. package/dist/transport/receiver-lease.js.map +1 -0
  105. package/dist/transport/runtime-status.d.ts +48 -0
  106. package/dist/transport/runtime-status.d.ts.map +1 -0
  107. package/dist/transport/runtime-status.js +71 -0
  108. package/dist/transport/runtime-status.js.map +1 -0
  109. package/dist/workflow/atomic-storage.d.ts +12 -0
  110. package/dist/workflow/atomic-storage.d.ts.map +1 -0
  111. package/dist/workflow/atomic-storage.js +104 -0
  112. package/dist/workflow/atomic-storage.js.map +1 -0
  113. package/dist/workflow/budget.d.ts +18 -0
  114. package/dist/workflow/budget.d.ts.map +1 -0
  115. package/dist/workflow/budget.js +74 -0
  116. package/dist/workflow/budget.js.map +1 -0
  117. package/dist/workflow/index.d.ts +17 -0
  118. package/dist/workflow/index.d.ts.map +1 -0
  119. package/dist/workflow/index.js +17 -0
  120. package/dist/workflow/index.js.map +1 -0
  121. package/dist/workflow/lifecycle-projection.d.ts +18 -0
  122. package/dist/workflow/lifecycle-projection.d.ts.map +1 -0
  123. package/dist/workflow/lifecycle-projection.js +22 -0
  124. package/dist/workflow/lifecycle-projection.js.map +1 -0
  125. package/dist/workflow/model-fallback.d.ts +2 -0
  126. package/dist/workflow/model-fallback.d.ts.map +1 -0
  127. package/dist/workflow/model-fallback.js +17 -0
  128. package/dist/workflow/model-fallback.js.map +1 -0
  129. package/dist/workflow/pi-session-fork.d.ts +18 -0
  130. package/dist/workflow/pi-session-fork.d.ts.map +1 -0
  131. package/dist/workflow/pi-session-fork.js +80 -0
  132. package/dist/workflow/pi-session-fork.js.map +1 -0
  133. package/dist/workflow/run-artifacts.d.ts +5 -0
  134. package/dist/workflow/run-artifacts.d.ts.map +1 -0
  135. package/dist/workflow/run-artifacts.js +25 -0
  136. package/dist/workflow/run-artifacts.js.map +1 -0
  137. package/dist/workflow/run-cleanup.d.ts +70 -0
  138. package/dist/workflow/run-cleanup.d.ts.map +1 -0
  139. package/dist/workflow/run-cleanup.js +222 -0
  140. package/dist/workflow/run-cleanup.js.map +1 -0
  141. package/dist/workflow/run-gate.d.ts +43 -0
  142. package/dist/workflow/run-gate.d.ts.map +1 -0
  143. package/dist/workflow/run-gate.js +70 -0
  144. package/dist/workflow/run-gate.js.map +1 -0
  145. package/dist/workflow/run-store.d.ts +208 -0
  146. package/dist/workflow/run-store.d.ts.map +1 -0
  147. package/dist/workflow/run-store.js +2965 -0
  148. package/dist/workflow/run-store.js.map +1 -0
  149. package/dist/workflow/spawn-confirmation.d.ts +44 -0
  150. package/dist/workflow/spawn-confirmation.d.ts.map +1 -0
  151. package/dist/workflow/spawn-confirmation.js +228 -0
  152. package/dist/workflow/spawn-confirmation.js.map +1 -0
  153. package/dist/workflow/system-process-inspector.d.ts +9 -0
  154. package/dist/workflow/system-process-inspector.d.ts.map +1 -0
  155. package/dist/workflow/system-process-inspector.js +129 -0
  156. package/dist/workflow/system-process-inspector.js.map +1 -0
  157. package/dist/workflow/worker-child-extension.d.ts +3 -0
  158. package/dist/workflow/worker-child-extension.d.ts.map +1 -0
  159. package/dist/workflow/worker-child-extension.js +182 -0
  160. package/dist/workflow/worker-child-extension.js.map +1 -0
  161. package/dist/workflow/worker-gate.d.ts +74 -0
  162. package/dist/workflow/worker-gate.d.ts.map +1 -0
  163. package/dist/workflow/worker-gate.js +165 -0
  164. package/dist/workflow/worker-gate.js.map +1 -0
  165. package/dist/workflow/worker-launch.d.ts +65 -0
  166. package/dist/workflow/worker-launch.d.ts.map +1 -0
  167. package/dist/workflow/worker-launch.js +338 -0
  168. package/dist/workflow/worker-launch.js.map +1 -0
  169. package/dist/workflow/worker-model-catalog.d.ts +36 -0
  170. package/dist/workflow/worker-model-catalog.d.ts.map +1 -0
  171. package/dist/workflow/worker-model-catalog.js +105 -0
  172. package/dist/workflow/worker-model-catalog.js.map +1 -0
  173. package/dist/workflow/worker-process-supervisor.d.ts +110 -0
  174. package/dist/workflow/worker-process-supervisor.d.ts.map +1 -0
  175. package/dist/workflow/worker-process-supervisor.js +1188 -0
  176. package/dist/workflow/worker-process-supervisor.js.map +1 -0
  177. package/dist/workflow/worker-prompt-capsule.d.ts +32 -0
  178. package/dist/workflow/worker-prompt-capsule.d.ts.map +1 -0
  179. package/dist/workflow/worker-prompt-capsule.js +52 -0
  180. package/dist/workflow/worker-prompt-capsule.js.map +1 -0
  181. package/dist/workflow/worker-role-presets.d.ts +38 -0
  182. package/dist/workflow/worker-role-presets.d.ts.map +1 -0
  183. package/dist/workflow/worker-role-presets.js +201 -0
  184. package/dist/workflow/worker-role-presets.js.map +1 -0
  185. package/dist/workflow/workflow-doctor.d.ts +57 -0
  186. package/dist/workflow/workflow-doctor.d.ts.map +1 -0
  187. package/dist/workflow/workflow-doctor.js +289 -0
  188. package/dist/workflow/workflow-doctor.js.map +1 -0
  189. package/dist/workflow/workflow-lease.d.ts +125 -0
  190. package/dist/workflow/workflow-lease.d.ts.map +1 -0
  191. package/dist/workflow/workflow-lease.js +689 -0
  192. package/dist/workflow/workflow-lease.js.map +1 -0
  193. package/dist/workflow/workflow-types.d.ts +335 -0
  194. package/dist/workflow/workflow-types.d.ts.map +1 -0
  195. package/dist/workflow/workflow-types.js +11 -0
  196. package/dist/workflow/workflow-types.js.map +1 -0
  197. package/dist/workflow/workspace-paths.d.ts +4 -0
  198. package/dist/workflow/workspace-paths.d.ts.map +1 -0
  199. package/dist/workflow/workspace-paths.js +22 -0
  200. package/dist/workflow/workspace-paths.js.map +1 -0
  201. package/docs/conformance.md +116 -0
  202. package/docs/demo-transcript.md +34 -0
  203. package/docs/design.md +1481 -0
  204. package/docs/operator-guide.md +100 -0
  205. package/extension/guidance.ts +162 -0
  206. package/extension/guide.ts +155 -0
  207. package/extension/index.ts +8 -0
  208. package/extension/runtime.ts +1224 -0
  209. package/extension/tools.ts +221 -0
  210. package/extension/worker-runtime.ts +1150 -0
  211. package/extension/worker-tools.ts +276 -0
  212. package/integration/confirmed-bind-onboarding.ts +296 -0
  213. package/integration/fleet-view.ts +286 -0
  214. package/integration/fresh-session-dispatcher.ts +166 -0
  215. package/integration/launch-preview.ts +396 -0
  216. package/integration/transport-port.ts +29 -0
  217. package/integration/worker-completion-notifier.ts +98 -0
  218. package/package.json +55 -0
  219. package/roles/oracle.md +9 -0
  220. package/roles/researcher.md +8 -0
  221. package/roles/reviewer.md +9 -0
  222. package/roles/scout.md +8 -0
  223. package/roles/worker.md +10 -0
  224. package/scripts/demo-two-terminal.sh +59 -0
  225. package/skills/coordination-patterns/SKILL.md +41 -0
  226. package/skills/coordination-patterns/references/supervised-worker-fan-out.md +12 -0
  227. package/skills/coordination-patterns/references/today-mail-team.md +36 -0
  228. package/transport/amq-cli-adapter.ts +381 -0
  229. package/transport/amq-message.ts +218 -0
  230. package/transport/amq-operations.ts +482 -0
  231. package/transport/atomic-json.ts +115 -0
  232. package/transport/binding-resolver.ts +565 -0
  233. package/transport/bridge-journal.ts +588 -0
  234. package/transport/dlq-retry-audit.ts +97 -0
  235. package/transport/inbox-pump.ts +729 -0
  236. package/transport/index.ts +11 -0
  237. package/transport/presence-projector.ts +104 -0
  238. package/transport/priority-delivery-scheduler.ts +328 -0
  239. package/transport/receiver-lease.ts +945 -0
  240. package/transport/runtime-status.ts +112 -0
  241. package/workflow/.gitkeep +1 -0
  242. package/workflow/atomic-storage.ts +120 -0
  243. package/workflow/budget.ts +90 -0
  244. package/workflow/index.ts +16 -0
  245. package/workflow/lifecycle-projection.ts +37 -0
  246. package/workflow/model-fallback.ts +17 -0
  247. package/workflow/pi-session-fork.ts +96 -0
  248. package/workflow/run-artifacts.ts +36 -0
  249. package/workflow/run-cleanup.ts +315 -0
  250. package/workflow/run-gate.ts +122 -0
  251. package/workflow/run-store.ts +3890 -0
  252. package/workflow/spawn-confirmation.ts +305 -0
  253. package/workflow/system-process-inspector.ts +142 -0
  254. package/workflow/worker-child-extension.ts +193 -0
  255. package/workflow/worker-gate.ts +238 -0
  256. package/workflow/worker-launch.ts +485 -0
  257. package/workflow/worker-model-catalog.ts +165 -0
  258. package/workflow/worker-process-supervisor.ts +1504 -0
  259. package/workflow/worker-prompt-capsule.ts +81 -0
  260. package/workflow/worker-role-presets.ts +241 -0
  261. package/workflow/workflow-doctor.ts +400 -0
  262. package/workflow/workflow-lease.ts +905 -0
  263. package/workflow/workflow-types.ts +367 -0
  264. package/workflow/workspace-paths.ts +32 -0
@@ -0,0 +1,1504 @@
1
+ import { spawn as spawnChild, type ChildProcess } from "node:child_process";
2
+ import { mkdir } from "node:fs/promises";
3
+ import { join } from "node:path";
4
+
5
+ import { sha256, writeWorkflowJson } from "./atomic-storage.js";
6
+ import { budgetExceeded, budgetValue } from "./budget.js";
7
+ import { NodePiSessionForker, type PiSessionForker } from "./pi-session-fork.js";
8
+ import { classifyProviderFallbackFailure } from "./model-fallback.js";
9
+ import {
10
+ effectiveWorkerContract,
11
+ BudgetExceededError,
12
+ WorkflowRunStore,
13
+ } from "./run-store.js";
14
+ import { buildWorkerPrompt } from "./worker-prompt-capsule.js";
15
+ import {
16
+ resolveWorkerLaunch,
17
+ type PiProcessLaunchCommand,
18
+ type WorkerLaunchAuthority,
19
+ } from "./worker-launch.js";
20
+ import {
21
+ WorkflowLeaseManager,
22
+ type ProcessIdentity,
23
+ type ProcessIdentityInspector,
24
+ } from "./workflow-lease.js";
25
+ import type {
26
+ AttemptId,
27
+ AcceptanceCheck,
28
+ ContractRevisionId,
29
+ PeerMessageId,
30
+ ReportSchemaV1,
31
+ RunProjectionV2,
32
+ WorkerAttemptProjection,
33
+ WorkerId,
34
+ WorkerProjection,
35
+ WorkerRequest,
36
+ } from "./workflow-types.js";
37
+
38
+ export const WORKER_PROTOCOL_LINE_LIMIT_BYTES = 4 * 1024 * 1024;
39
+ export const WORKER_STDERR_TAIL_LIMIT_BYTES = 128 * 1024;
40
+
41
+ export class WorkerProtocolError extends Error {
42
+ constructor(
43
+ readonly code: "protocol_output_limit" | "protocol_invalid_json",
44
+ message: string,
45
+ readonly limitBytes?: number,
46
+ ) {
47
+ super(message);
48
+ this.name = "WorkerProtocolError";
49
+ }
50
+ }
51
+
52
+ export { BudgetExceededError } from "./run-store.js";
53
+ export type { PiProcessLaunchCommand } from "./worker-launch.js";
54
+
55
+ export interface ProcessExit {
56
+ readonly code: number | null;
57
+ readonly signal: string | null;
58
+ }
59
+
60
+ export interface ManagedWorkerProcess {
61
+ readonly pid: number;
62
+ readonly stdout: AsyncIterable<Uint8Array>;
63
+ readonly stderr: AsyncIterable<Uint8Array>;
64
+ readonly exited: Promise<ProcessExit>;
65
+ writeLine(line: string): Promise<void>;
66
+ endInput(): Promise<void>;
67
+ signalProcessGroup(signal: NodeJS.Signals): Promise<void>;
68
+ closeParentLiveness(): Promise<void>;
69
+ }
70
+
71
+ export interface PiProcessFactory {
72
+ spawn(command: PiProcessLaunchCommand): Promise<ManagedWorkerProcess>;
73
+ }
74
+
75
+ export interface WorkerTimerScheduler {
76
+ schedule(delayMs: number, callback: () => void): { cancel(): void };
77
+ }
78
+
79
+ export interface WorkerProcessSupervisorOptions {
80
+ readonly namespaceDirectory: string;
81
+ readonly store: WorkflowRunStore;
82
+ readonly leaseManager: WorkflowLeaseManager;
83
+ readonly supervisorGeneration: string;
84
+ readonly inspector: ProcessIdentityInspector;
85
+ readonly processFactory: PiProcessFactory;
86
+ readonly launchAuthority: WorkerLaunchAuthority;
87
+ readonly timerScheduler?: WorkerTimerScheduler;
88
+ readonly now?: () => Date;
89
+ readonly cancellationGraceMs?: number;
90
+ readonly sessionForker?: PiSessionForker;
91
+ }
92
+
93
+ export type SpawnWorkerResult =
94
+ | {
95
+ readonly kind: "started";
96
+ readonly worker: WorkerProjection;
97
+ readonly attempt: WorkerAttemptProjection;
98
+ readonly settlement: Promise<RunProjectionV2>;
99
+ }
100
+ | {
101
+ readonly kind: "existing" | "needs-reconcile";
102
+ readonly worker: WorkerProjection;
103
+ readonly attempt: WorkerAttemptProjection;
104
+ };
105
+
106
+ export type CancelWorkerResult =
107
+ | { readonly kind: "requested"; readonly projection: RunProjectionV2 }
108
+ | { readonly kind: "indeterminate"; readonly projection: RunProjectionV2; readonly reason: string };
109
+
110
+ export class ResumeRefusedError extends Error {
111
+ constructor(message: string) {
112
+ super(message);
113
+ this.name = "ResumeRefusedError";
114
+ }
115
+ }
116
+
117
+ interface ManagedAttempt {
118
+ readonly requestKey: string;
119
+ readonly workerId: WorkerId;
120
+ readonly attemptId: AttemptId;
121
+ readonly workerAttemptGeneration: string;
122
+ readonly leaseGeneration: string;
123
+ readonly child: ManagedWorkerProcess;
124
+ readonly deadline: { cancel(): void };
125
+ inferenceActive: boolean;
126
+ readonly pendingRevisions: Array<{
127
+ readonly revisionId: ContractRevisionId;
128
+ readonly prompt: string;
129
+ }>;
130
+ readonly pendingPeerMessages: Array<{
131
+ readonly messageId: PeerMessageId;
132
+ readonly senderWorkerId: WorkerId;
133
+ readonly message: string;
134
+ readonly correlation: string;
135
+ }>;
136
+ cancellationEscalation?: { cancel(): void };
137
+ }
138
+
139
+ interface MonitorState {
140
+ outputBytes: number;
141
+ tokens: number;
142
+ costUsd: number;
143
+ candidateValue: string | null;
144
+ candidateHash: string | null;
145
+ ready: boolean;
146
+ stderrTail: Uint8Array;
147
+ fallbackReason: string | null;
148
+ }
149
+
150
+ interface MonitorResult {
151
+ readonly projection: RunProjectionV2;
152
+ readonly fallbackReason: string | null;
153
+ }
154
+
155
+ class ModelFallbackEligibleError extends Error {
156
+ constructor(readonly fallbackReason: string, message: string) {
157
+ super(message);
158
+ this.name = "ModelFallbackEligibleError";
159
+ }
160
+ }
161
+
162
+ export class WorkerProcessSupervisor {
163
+ readonly #namespaceDirectory: string;
164
+ readonly #store: WorkflowRunStore;
165
+ readonly #leaseManager: WorkflowLeaseManager;
166
+ readonly #supervisorGeneration: string;
167
+ readonly #inspector: ProcessIdentityInspector;
168
+ readonly #processFactory: PiProcessFactory;
169
+ #launchAuthority: WorkerLaunchAuthority;
170
+ readonly #timers: WorkerTimerScheduler;
171
+ readonly #now: () => Date;
172
+ readonly #cancellationGraceMs: number;
173
+ readonly #sessionForker: PiSessionForker;
174
+ readonly #managed = new Map<WorkerId, ManagedAttempt>();
175
+
176
+ constructor(options: WorkerProcessSupervisorOptions) {
177
+ this.#namespaceDirectory = options.namespaceDirectory;
178
+ this.#store = options.store;
179
+ this.#leaseManager = options.leaseManager;
180
+ this.#supervisorGeneration = options.supervisorGeneration;
181
+ this.#inspector = options.inspector;
182
+ this.#processFactory = options.processFactory;
183
+ this.#launchAuthority = options.launchAuthority;
184
+ this.#timers = options.timerScheduler ?? new NodeWorkerTimerScheduler();
185
+ this.#now = options.now ?? (() => new Date());
186
+ this.#cancellationGraceMs = options.cancellationGraceMs ?? 2_000;
187
+ this.#sessionForker = options.sessionForker ?? new NodePiSessionForker();
188
+ }
189
+
190
+ updateLaunchAuthority(authority: WorkerLaunchAuthority): void {
191
+ this.#launchAuthority = authority;
192
+ }
193
+
194
+ async spawnWorker(requestKey: string, input: {
195
+ readonly request: WorkerRequest;
196
+ readonly prompt: string;
197
+ readonly depth: number;
198
+ readonly parentWorkerId?: WorkerId | null;
199
+ readonly continueWorkerId?: WorkerId;
200
+ }): Promise<SpawnWorkerResult> {
201
+ const run = await this.#requiredRun(requestKey);
202
+ if (input.continueWorkerId !== undefined && input.request.context === "fork") {
203
+ throw new TypeError("A worker cannot continue another worker session and fork driver context together");
204
+ }
205
+ const continuation = input.continueWorkerId === undefined
206
+ ? undefined
207
+ : await this.#continuationSource(run, input.continueWorkerId);
208
+ const sessionPath = continuation?.sessionPath
209
+ ?? this.#sessionDirectory(run.runId, input.request.requestKey);
210
+ const forkedSessionFile = input.request.context === "fork"
211
+ ? join(sessionPath, "session.jsonl")
212
+ : undefined;
213
+ const prompt = continuation === undefined
214
+ ? input.prompt
215
+ : buildWorkerPrompt({
216
+ ...input.request,
217
+ continuation: {
218
+ priorWorkerId: continuation.priorWorkerId,
219
+ priorAttemptId: continuation.priorAttemptId,
220
+ sessionFile: continuation.sessionFile,
221
+ },
222
+ });
223
+ // Pure preflight uses disposable runtime IDs. It proves launch authority before any ledger,
224
+ // lease, filesystem, or process effect.
225
+ resolveWorkerLaunch({
226
+ contract: {
227
+ ...input.request,
228
+ context: input.request.context ?? "fresh",
229
+ runId: run.runId,
230
+ workerId: "worker_preflight",
231
+ createdByDriverSessionId: run.source.driverSessionId,
232
+ },
233
+ attempt: {
234
+ attemptId: "attempt_preflight",
235
+ workerAttemptGeneration: "preflight-worker-generation",
236
+ supervisorGeneration: this.#supervisorGeneration,
237
+ },
238
+ depth: input.depth,
239
+ prompt,
240
+ sessionDirectory: sessionPath,
241
+ launchContractPath: join(this.#namespaceDirectory, "launch-contracts", "preflight.json"),
242
+ ...(continuation ? { continuedSessionFile: continuation.sessionFile } : {}),
243
+ ...(forkedSessionFile ? { forkedSessionFile } : {}),
244
+ modelOverride: input.request.model,
245
+ authority: this.#launchAuthority,
246
+ });
247
+ const prepared = await this.#store.prepareWorkerIntent(requestKey, {
248
+ createdByDriverSessionId: run.source.driverSessionId,
249
+ sessionPath,
250
+ request: input.request,
251
+ parentWorkerId: input.parentWorkerId ?? null,
252
+ depth: input.depth,
253
+ ...(continuation ? { continuation: {
254
+ priorWorkerId: continuation.priorWorkerId,
255
+ priorAttemptId: continuation.priorAttemptId,
256
+ priorLeaseGeneration: continuation.priorLeaseGeneration,
257
+ sessionFile: continuation.sessionFile,
258
+ } } : {}),
259
+ });
260
+ const attempt = currentAttempt(prepared.worker);
261
+ if (prepared.kind === "existing") {
262
+ const managed = this.#managed.get(prepared.worker.workerId);
263
+ if (managed?.attemptId === attempt.attemptId) {
264
+ return { kind: "existing", worker: prepared.worker, attempt };
265
+ }
266
+ return { kind: "needs-reconcile", worker: prepared.worker, attempt };
267
+ }
268
+ if (prepared.worker.contract.context === "fork") {
269
+ const forkSource = prepared.worker.contract.forkSource;
270
+ if (!forkSource || !forkedSessionFile) throw new Error("Fork worker context is incomplete");
271
+ try {
272
+ await this.#sessionForker.fork({ ...forkSource, targetSessionFile: forkedSessionFile });
273
+ } catch (error) {
274
+ await this.#store.recordAttemptExit(requestKey, {
275
+ workerId: prepared.worker.workerId,
276
+ attemptId: attempt.attemptId,
277
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
278
+ exitStatus: { code: 1, signal: null },
279
+ candidateOutcome: { valueOrRef: null, hash: null },
280
+ uncertainSideEffects: false,
281
+ reason: `Pi session fork failed before process creation: ${toError(error).message}`,
282
+ });
283
+ throw error;
284
+ }
285
+ }
286
+ return this.#startPreparedAttempt(requestKey, prepared.worker, attempt, prompt);
287
+ }
288
+
289
+ async resumeWorker(requestKey: string, input: {
290
+ readonly workerId: WorkerId;
291
+ readonly prompt: string;
292
+ }): Promise<SpawnWorkerResult> {
293
+ let run = await this.#requiredRun(requestKey);
294
+ let worker = requiredWorker(run, input.workerId);
295
+ let attempt = currentAttempt(worker);
296
+ if (attempt.state === "ambiguous" || attempt.sideEffects.status === "uncertain") {
297
+ throw new ResumeRefusedError(
298
+ `Worker ${input.workerId} has unresolved side-effect ambiguity`,
299
+ );
300
+ }
301
+ if (attempt.state === "running") {
302
+ const inspection = await this.#inspectAttempt(attempt);
303
+ if (inspection !== "dead") {
304
+ throw new ResumeRefusedError(
305
+ `Worker ${input.workerId} cannot resume because prior liveness is ${inspection}`,
306
+ );
307
+ }
308
+ run = await this.#store.recordAttemptExit(requestKey, {
309
+ workerId: worker.workerId,
310
+ attemptId: attempt.attemptId,
311
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
312
+ exitStatus: { code: 1, signal: "proven-dead-before-resume" },
313
+ candidateOutcome: attempt.candidateOutcome,
314
+ uncertainSideEffects: false,
315
+ reason: "Prior worker process is proven dead before explicit resume",
316
+ });
317
+ worker = requiredWorker(run, input.workerId);
318
+ attempt = currentAttempt(worker);
319
+ }
320
+ if (attempt.state === "pending") {
321
+ throw new ResumeRefusedError(
322
+ `Worker ${input.workerId} has an unreconciled spawn intent; replay is forbidden`,
323
+ );
324
+ }
325
+ const prepared = await this.#store.prepareReplacementAttempt(requestKey, {
326
+ workerId: input.workerId,
327
+ });
328
+ return this.#startPreparedAttempt(
329
+ requestKey,
330
+ prepared.worker,
331
+ prepared.attempt,
332
+ input.prompt,
333
+ );
334
+ }
335
+
336
+ async cancelWorker(requestKey: string, input: {
337
+ readonly workerId: WorkerId;
338
+ readonly reason: string;
339
+ }): Promise<CancelWorkerResult> {
340
+ let projection = await this.#store.requestWorkerCancellation(requestKey, input);
341
+ const attempt = currentAttempt(requiredWorker(projection, input.workerId));
342
+ const managed = this.#managed.get(input.workerId);
343
+ if (!managed || managed.attemptId !== attempt.attemptId
344
+ || managed.workerAttemptGeneration !== attempt.workerAttemptGeneration) {
345
+ return {
346
+ kind: "indeterminate",
347
+ projection,
348
+ reason: "No current supervised process is attached to the durable cancellation intent",
349
+ };
350
+ }
351
+ if (attempt.launchContract?.channel === "rpc") {
352
+ await managed.child.writeLine(JSON.stringify({
353
+ id: `cancel-${attempt.attemptId}`,
354
+ type: "abort",
355
+ })).catch(() => {});
356
+ }
357
+ try {
358
+ await managed.child.signalProcessGroup("SIGTERM");
359
+ } catch (error) {
360
+ projection = await this.#store.recordWorkerNeedsAttention(requestKey, {
361
+ workerId: input.workerId,
362
+ attemptId: attempt.attemptId,
363
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
364
+ reason: `Cancellation signal outcome is unknown: ${toError(error).message}`,
365
+ ambiguous: true,
366
+ });
367
+ return { kind: "indeterminate", projection, reason: toError(error).message };
368
+ }
369
+ projection = await this.#store.recordCancellationSignalSent(requestKey, {
370
+ workerId: input.workerId,
371
+ attemptId: attempt.attemptId,
372
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
373
+ signal: "SIGTERM",
374
+ });
375
+ managed.cancellationEscalation?.cancel();
376
+ managed.cancellationEscalation = this.#timers.schedule(this.#cancellationGraceMs, () => {
377
+ const current = this.#managed.get(input.workerId);
378
+ if (current?.attemptId === attempt.attemptId) {
379
+ void current.child.signalProcessGroup("SIGKILL").catch(() => {});
380
+ }
381
+ });
382
+ return { kind: "requested", projection };
383
+ }
384
+
385
+ async steerWorker(requestKey: string, input: {
386
+ readonly workerId: WorkerId;
387
+ readonly message: string;
388
+ readonly correlation?: string;
389
+ }): Promise<RunProjectionV2> {
390
+ const managed = this.#managed.get(input.workerId);
391
+ if (!managed) throw new Error(`Worker ${input.workerId} has no attached RPC process`);
392
+ const projection = await this.#requiredRun(requestKey);
393
+ const worker = requiredWorker(projection, input.workerId);
394
+ if (effectiveWorkerContract(worker).channel !== "rpc") {
395
+ throw new Error("Only an RPC worker can be steered");
396
+ }
397
+ const message = nonEmpty(input.message, "worker steer message");
398
+ const correlation = input.correlation === undefined
399
+ ? null
400
+ : boundedProtocolText(input.correlation, "worker steer correlation", 256);
401
+ assertBlockedWakeCorrelation(worker, correlation);
402
+ await managed.child.writeLine(JSON.stringify({
403
+ id: `steer-${managed.attemptId}`,
404
+ type: "steer",
405
+ message,
406
+ }));
407
+ managed.inferenceActive = true;
408
+ try {
409
+ return await this.#store.recordSteerDelivered(requestKey, {
410
+ workerId: input.workerId,
411
+ messageHash: sha256(message),
412
+ ...(correlation === null ? {} : { correlation }),
413
+ });
414
+ } catch (error) {
415
+ await this.#store.recordWorkerNeedsAttention(requestKey, {
416
+ workerId: worker.workerId,
417
+ attemptId: managed.attemptId,
418
+ workerAttemptGeneration: managed.workerAttemptGeneration,
419
+ reason: `Steer reached the worker but correlated delivery ledgering failed: ${toError(error).message}`,
420
+ ambiguous: true,
421
+ }).catch(() => {});
422
+ throw error;
423
+ }
424
+ }
425
+
426
+ async reviseWorkerContract(requestKey: string, input: {
427
+ readonly workerId: WorkerId;
428
+ readonly correlation?: string;
429
+ readonly goal?: string;
430
+ readonly acceptance?: readonly string[];
431
+ readonly constraints?: readonly string[];
432
+ readonly report_schema?: ReportSchemaV1;
433
+ readonly requires_human_approval?: boolean;
434
+ readonly acceptance_check?: AcceptanceCheck;
435
+ }): Promise<RunProjectionV2> {
436
+ const before = await this.#requiredRun(requestKey);
437
+ const workerBefore = requiredWorker(before, input.workerId);
438
+ if (effectiveWorkerContract(workerBefore).channel !== "rpc") {
439
+ throw new Error("Contract revision delivery requires an RPC worker");
440
+ }
441
+ let projection = await this.#store.requestContractRevision(requestKey, input);
442
+ const worker = requiredWorker(projection, input.workerId);
443
+ const revision = worker.contractRevisions.at(-1);
444
+ if (!revision || revision.status !== "pending") {
445
+ throw new Error("Contract revision was not recorded as pending");
446
+ }
447
+ const managed = this.#managed.get(input.workerId);
448
+ if (!managed) return projection;
449
+ const prompt = [
450
+ `Apply immutable contract revision ${revision.revision} at this inference boundary.`,
451
+ buildWorkerPrompt(revision.contract),
452
+ ].join("\n\n");
453
+ if (managed.inferenceActive) {
454
+ managed.pendingRevisions.push({ revisionId: revision.revisionId, prompt });
455
+ return projection;
456
+ }
457
+ await managed.child.writeLine(JSON.stringify({
458
+ id: `revision-${revision.revisionId}`,
459
+ type: "prompt",
460
+ message: prompt,
461
+ }));
462
+ managed.inferenceActive = true;
463
+ projection = await this.#recordRevisionDelivered(managed, revision.revisionId);
464
+ return projection;
465
+ }
466
+
467
+ async reconcileRun(requestKey: string): Promise<RunProjectionV2> {
468
+ let projection = await this.#requiredRun(requestKey);
469
+ const reconciledPeerMessages = new Set<PeerMessageId>();
470
+ for (const worker of Object.values(projection.workers)) {
471
+ for (const peer of worker.peerMessages) {
472
+ if (peer.status !== "pending" || reconciledPeerMessages.has(peer.messageId)) continue;
473
+ reconciledPeerMessages.add(peer.messageId);
474
+ const recipient = requiredWorker(projection, peer.recipientWorkerId);
475
+ const recipientAttempt = currentAttempt(recipient);
476
+ projection = await this.#store.recordPeerMessageDelivery(requestKey, {
477
+ messageId: peer.messageId,
478
+ recipientWorkerId: recipient.workerId,
479
+ recipientAttemptId: recipientAttempt.attemptId,
480
+ recipientWorkerAttemptGeneration: recipientAttempt.workerAttemptGeneration,
481
+ status: "indeterminate",
482
+ diagnostic: "Pending peer-message body was not persisted and cannot be replayed after supervisor restart",
483
+ });
484
+ }
485
+ }
486
+ for (const worker of Object.values(projection.workers)) {
487
+ const attempt = currentAttempt(worker);
488
+ if (attempt.state === "pending") {
489
+ projection = await this.#store.recordWorkerNeedsAttention(requestKey, {
490
+ workerId: worker.workerId,
491
+ attemptId: attempt.attemptId,
492
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
493
+ reason: "Recovered spawn intent has no durable process-start evidence; no replay performed",
494
+ ambiguous: true,
495
+ });
496
+ continue;
497
+ }
498
+ if (attempt.state !== "running") continue;
499
+ const liveness = await this.#inspectAttempt(attempt);
500
+ if (liveness === "dead") {
501
+ projection = await this.#store.recordAttemptExit(requestKey, {
502
+ workerId: worker.workerId,
503
+ attemptId: attempt.attemptId,
504
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
505
+ exitStatus: { code: 1, signal: "proven-dead-on-recovery" },
506
+ candidateOutcome: attempt.candidateOutcome,
507
+ uncertainSideEffects: attempt.sideEffects.policy === "external"
508
+ || attempt.sideEffects.policy === "reconcilable",
509
+ reason: "Worker was proven dead during restart reconciliation",
510
+ });
511
+ } else {
512
+ projection = await this.#store.recordWorkerNeedsAttention(requestKey, {
513
+ workerId: worker.workerId,
514
+ attemptId: attempt.attemptId,
515
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
516
+ reason: liveness === "live"
517
+ ? "Worker remains live but its control pipe was not reattached"
518
+ : "Worker liveness could not be verified after restart",
519
+ ambiguous: liveness === "unknown",
520
+ });
521
+ }
522
+ }
523
+ return projection;
524
+ }
525
+
526
+ async #startPreparedAttempt(
527
+ requestKey: string,
528
+ worker: WorkerProjection,
529
+ attempt: WorkerAttemptProjection,
530
+ prompt: string,
531
+ ): Promise<SpawnWorkerResult> {
532
+ const launchContractPath = join(
533
+ this.#namespaceDirectory,
534
+ "launch-contracts",
535
+ worker.workerId,
536
+ `${attempt.attemptId}.json`,
537
+ );
538
+ const resolved = resolveWorkerLaunch({
539
+ contract: worker.contract,
540
+ attempt: {
541
+ attemptId: attempt.attemptId,
542
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
543
+ supervisorGeneration: attempt.supervisorGeneration,
544
+ },
545
+ depth: worker.depth,
546
+ prompt,
547
+ sessionDirectory: attempt.session.path,
548
+ launchContractPath,
549
+ ...(worker.continuation
550
+ ? { continuedSessionFile: worker.continuation.sessionFile }
551
+ : {}),
552
+ ...(worker.contract.context === "fork"
553
+ ? { forkedSessionFile: join(attempt.session.path, "session.jsonl") }
554
+ : {}),
555
+ modelOverride: attempt.modelSelection?.model ?? worker.contract.model,
556
+ authority: this.#launchAuthority,
557
+ });
558
+ await mkdir(attempt.session.path, { recursive: true, mode: 0o700 });
559
+ await mkdir(join(this.#namespaceDirectory, "launch-contracts", worker.workerId), {
560
+ recursive: true,
561
+ mode: 0o700,
562
+ });
563
+ await writeWorkflowJson(launchContractPath, resolved.contract, { createOnly: true });
564
+ const reservation = await this.#leaseManager.reserveWorkerSession({
565
+ sessionPath: attempt.session.path,
566
+ runId: worker.contract.runId,
567
+ workerId: worker.workerId,
568
+ attemptId: attempt.attemptId,
569
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
570
+ supervisorGeneration: this.#supervisorGeneration,
571
+ });
572
+ if (reservation.kind !== "acquired") {
573
+ await this.#store.recordWorkerNeedsAttention(requestKey, {
574
+ workerId: worker.workerId,
575
+ attemptId: attempt.attemptId,
576
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
577
+ reason: reservation.kind === "held"
578
+ ? `Worker session is held by generation ${reservation.owner.generation}`
579
+ : reservation.diagnostic,
580
+ ambiguous: reservation.kind !== "held",
581
+ });
582
+ throw new ResumeRefusedError("Worker session could not be reserved before process creation");
583
+ }
584
+ let child: ManagedWorkerProcess;
585
+ try {
586
+ child = await this.#processFactory.spawn(resolved.command);
587
+ } catch (error) {
588
+ await this.#leaseManager.releaseWorkerSession({
589
+ sessionPath: attempt.session.path,
590
+ generation: reservation.owner.generation,
591
+ supervisorGeneration: this.#supervisorGeneration,
592
+ });
593
+ await this.#store.recordAttemptExit(requestKey, {
594
+ workerId: worker.workerId,
595
+ attemptId: attempt.attemptId,
596
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
597
+ exitStatus: { code: 127, signal: null },
598
+ candidateOutcome: { valueOrRef: null, hash: null },
599
+ uncertainSideEffects: false,
600
+ reason: `Pi process creation failed: ${toError(error).message}`,
601
+ });
602
+ throw error;
603
+ }
604
+ const inspection = await this.#inspector.inspect(child.pid);
605
+ if (inspection.kind !== "live") {
606
+ await child.signalProcessGroup("SIGKILL").catch(() => {});
607
+ const postKillInspection = await this.#inspector.inspect(child.pid);
608
+ if (postKillInspection.kind === "absent") {
609
+ await this.#leaseManager.releaseWorkerSession({
610
+ sessionPath: attempt.session.path,
611
+ generation: reservation.owner.generation,
612
+ supervisorGeneration: this.#supervisorGeneration,
613
+ });
614
+ }
615
+ await this.#store.recordWorkerNeedsAttention(requestKey, {
616
+ workerId: worker.workerId,
617
+ attemptId: attempt.attemptId,
618
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
619
+ reason: postKillInspection.kind === "unknown"
620
+ ? postKillInspection.diagnostic
621
+ : "Spawned child disappeared before process identity could be recorded",
622
+ ambiguous: true,
623
+ });
624
+ throw new Error("Spawned worker identity could not be proven");
625
+ }
626
+ const activeLease = await this.#leaseManager.activateWorkerSession({
627
+ sessionPath: attempt.session.path,
628
+ generation: reservation.owner.generation,
629
+ supervisorGeneration: this.#supervisorGeneration,
630
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
631
+ processIdentity: inspection.identity,
632
+ });
633
+ let projection = await this.#store.recordAttemptStarted(requestKey, {
634
+ workerId: worker.workerId,
635
+ attemptId: attempt.attemptId,
636
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
637
+ leaseGeneration: activeLease.generation,
638
+ processIdentity: inspection.identity,
639
+ launchContract: resolved.contract,
640
+ });
641
+ worker = requiredWorker(projection, worker.workerId);
642
+ attempt = currentAttempt(worker);
643
+
644
+ let resolveReady!: () => void;
645
+ let rejectReady!: (error: Error) => void;
646
+ const ready = new Promise<void>((resolve, reject) => {
647
+ resolveReady = resolve;
648
+ rejectReady = reject;
649
+ });
650
+ const state: MonitorState = {
651
+ outputBytes: 0,
652
+ tokens: 0,
653
+ costUsd: 0,
654
+ candidateValue: null,
655
+ candidateHash: null,
656
+ ready: false,
657
+ stderrTail: new Uint8Array(),
658
+ fallbackReason: null,
659
+ };
660
+ const wallClockMs = budgetValue(worker.contract.budget.maxWallClockMs);
661
+ const deadline = wallClockMs === undefined ? { cancel(): void {} } : this.#timers.schedule(
662
+ wallClockMs,
663
+ () => {
664
+ void this.cancelWorker(requestKey, {
665
+ workerId: worker.workerId,
666
+ reason: "worker wall-clock budget exhausted",
667
+ }).catch(() => {});
668
+ },
669
+ );
670
+ const managedPlaceholder = {} as ManagedAttempt;
671
+ const monitored = this.#monitorAttempt({
672
+ requestKey,
673
+ worker,
674
+ attempt,
675
+ child,
676
+ leaseGeneration: activeLease.generation,
677
+ state,
678
+ resolveReady,
679
+ rejectReady,
680
+ });
681
+ const managed = Object.assign(managedPlaceholder, {
682
+ requestKey,
683
+ workerId: worker.workerId,
684
+ attemptId: attempt.attemptId,
685
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
686
+ leaseGeneration: activeLease.generation,
687
+ child,
688
+ deadline,
689
+ inferenceActive: false,
690
+ pendingRevisions: [],
691
+ pendingPeerMessages: [],
692
+ }) as ManagedAttempt;
693
+ this.#managed.set(worker.workerId, managed);
694
+ try {
695
+ await Promise.race([
696
+ ready,
697
+ child.exited.then((exit) => {
698
+ throw new Error(`Worker exited before launch parity was ready: ${formatExit(exit)}`);
699
+ }),
700
+ ]);
701
+ } catch (error) {
702
+ const result = await monitored.catch(() => undefined);
703
+ if (result?.fallbackReason && this.#nextFallbackModel(result.projection, worker.workerId, attempt)) {
704
+ return this.#startModelFallback(
705
+ requestKey,
706
+ result.projection,
707
+ worker.workerId,
708
+ attempt,
709
+ result.fallbackReason,
710
+ prompt,
711
+ );
712
+ }
713
+ throw error;
714
+ }
715
+ if (worker.contract.channel === "rpc") {
716
+ await child.writeLine(JSON.stringify({
717
+ id: `prompt-${attempt.attemptId}`,
718
+ type: "prompt",
719
+ message: prompt,
720
+ }));
721
+ managed.inferenceActive = true;
722
+ } else {
723
+ await child.endInput();
724
+ }
725
+ projection = await this.#requiredRun(requestKey);
726
+ worker = requiredWorker(projection, worker.workerId);
727
+ attempt = currentAttempt(worker);
728
+ const settlement = monitored.then((result) => this.#settleModelFallback(
729
+ requestKey,
730
+ result,
731
+ worker.workerId,
732
+ attempt,
733
+ prompt,
734
+ ));
735
+ return { kind: "started", worker, attempt, settlement };
736
+ }
737
+
738
+ async #settleModelFallback(
739
+ requestKey: string,
740
+ result: MonitorResult,
741
+ workerId: WorkerId,
742
+ attempt: WorkerAttemptProjection,
743
+ prompt: string,
744
+ ): Promise<RunProjectionV2> {
745
+ if (!result.fallbackReason) return result.projection;
746
+ const next = this.#nextFallbackModel(result.projection, workerId, attempt);
747
+ if (!next) return result.projection;
748
+ const fallback = await this.#startModelFallback(
749
+ requestKey,
750
+ result.projection,
751
+ workerId,
752
+ attempt,
753
+ result.fallbackReason,
754
+ prompt,
755
+ );
756
+ return fallback.kind === "started"
757
+ ? fallback.settlement
758
+ : this.#requiredRun(requestKey);
759
+ }
760
+
761
+ async #startModelFallback(
762
+ requestKey: string,
763
+ projection: RunProjectionV2,
764
+ workerId: WorkerId,
765
+ attempt: WorkerAttemptProjection,
766
+ reason: string,
767
+ prompt: string,
768
+ ): Promise<SpawnWorkerResult> {
769
+ const next = this.#nextFallbackModel(projection, workerId, attempt);
770
+ if (!next) throw new Error(`Model fallback chain exhausted after ${reason}`);
771
+ const prepared = await this.#store.prepareReplacementAttempt(requestKey, {
772
+ workerId,
773
+ model: next,
774
+ fallbackReason: reason,
775
+ });
776
+ return this.#startPreparedAttempt(
777
+ requestKey,
778
+ prepared.worker,
779
+ prepared.attempt,
780
+ prompt,
781
+ );
782
+ }
783
+
784
+ #nextFallbackModel(
785
+ projection: RunProjectionV2,
786
+ workerId: WorkerId,
787
+ attempt: WorkerAttemptProjection,
788
+ ) {
789
+ const worker = requiredWorker(projection, workerId);
790
+ const chain = [worker.contract.model, ...(worker.contract.modelFallbacks ?? [])];
791
+ const attempted = attempt.modelSelection?.model ?? attempt.launchContract?.model ?? worker.contract.model;
792
+ const index = chain.findIndex((model) =>
793
+ model.provider === attempted.provider && model.id === attempted.id);
794
+ return index < 0 ? undefined : chain[index + 1];
795
+ }
796
+
797
+ async #monitorAttempt(input: {
798
+ readonly requestKey: string;
799
+ readonly worker: WorkerProjection;
800
+ readonly attempt: WorkerAttemptProjection;
801
+ readonly child: ManagedWorkerProcess;
802
+ readonly leaseGeneration: string;
803
+ readonly state: MonitorState;
804
+ readonly resolveReady: () => void;
805
+ readonly rejectReady: (error: Error) => void;
806
+ }): Promise<MonitorResult> {
807
+ let stdoutError: Error | undefined;
808
+ const stdout = this.#consumeStdout(input).catch((error: unknown) => {
809
+ stdoutError = toError(error);
810
+ if (stdoutError instanceof ModelFallbackEligibleError) {
811
+ input.state.fallbackReason = stdoutError.fallbackReason;
812
+ input.rejectReady(stdoutError);
813
+ return;
814
+ }
815
+ input.rejectReady(stdoutError);
816
+ return this.cancelWorker(input.requestKey, {
817
+ workerId: input.worker.workerId,
818
+ reason: `worker protocol failed: ${stdoutError.message}`,
819
+ }).then(() => {}).catch(() => {});
820
+ });
821
+ const stderr = this.#consumeStderr(input);
822
+ const exit = await input.child.exited;
823
+ await Promise.all([stdout, stderr]);
824
+ const managed = this.#managed.get(input.worker.workerId);
825
+ managed?.deadline.cancel();
826
+ managed?.cancellationEscalation?.cancel();
827
+ await input.child.closeParentLiveness().catch(() => {});
828
+ let projection = await this.#store.recordAttemptUsage(input.requestKey, {
829
+ workerId: input.worker.workerId,
830
+ attemptId: input.attempt.attemptId,
831
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
832
+ tokens: input.state.tokens,
833
+ costUsd: input.state.costUsd,
834
+ outputBytes: input.state.outputBytes,
835
+ });
836
+ const uncertain = input.worker.contract.sideEffectPolicy === "external"
837
+ || input.worker.contract.sideEffectPolicy === "reconcilable";
838
+ projection = await this.#store.recordAttemptExit(input.requestKey, {
839
+ workerId: input.worker.workerId,
840
+ attemptId: input.attempt.attemptId,
841
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
842
+ exitStatus: stdoutError ? { code: 1, signal: exit.signal } : exit,
843
+ candidateOutcome: {
844
+ valueOrRef: input.state.candidateValue,
845
+ hash: input.state.candidateHash,
846
+ },
847
+ uncertainSideEffects: uncertain && (exit.code !== 0 || stdoutError !== undefined),
848
+ reason: stdoutError
849
+ ? `Worker protocol failed: ${protocolErrorDetails(stdoutError)}`
850
+ : `Worker process exited: ${formatExit(exit)}${
851
+ exit.code === 0 || input.state.stderrTail.byteLength === 0
852
+ ? ""
853
+ : `; stderr_tail=${JSON.stringify(new TextDecoder().decode(input.state.stderrTail))}`
854
+ }`,
855
+ });
856
+ await this.#leaseManager.releaseWorkerSession({
857
+ sessionPath: input.attempt.session.path,
858
+ generation: input.leaseGeneration,
859
+ supervisorGeneration: this.#supervisorGeneration,
860
+ });
861
+ if (this.#managed.get(input.worker.workerId)?.attemptId === input.attempt.attemptId) {
862
+ this.#managed.delete(input.worker.workerId);
863
+ }
864
+ return { projection, fallbackReason: input.state.fallbackReason };
865
+ }
866
+
867
+ async #consumeStdout(input: {
868
+ readonly requestKey: string;
869
+ readonly worker: WorkerProjection;
870
+ readonly attempt: WorkerAttemptProjection;
871
+ readonly child: ManagedWorkerProcess;
872
+ readonly state: MonitorState;
873
+ readonly resolveReady: () => void;
874
+ }): Promise<void> {
875
+ const decoder = new TextDecoder();
876
+ let buffer = "";
877
+ for await (const chunk of input.child.stdout) {
878
+ input.state.outputBytes += chunk.byteLength;
879
+ buffer += decoder.decode(chunk, { stream: true });
880
+ if (!buffer.includes("\n") && Buffer.byteLength(buffer, "utf8") > WORKER_PROTOCOL_LINE_LIMIT_BYTES) {
881
+ throw protocolOutputLimit();
882
+ }
883
+ await this.#consumeLines(buffer, false, async (line, rest) => {
884
+ buffer = rest;
885
+ await this.#handleProtocolLine(input, line);
886
+ });
887
+ await this.#enforceObservedBudgets(input);
888
+ }
889
+ buffer += decoder.decode();
890
+ if (buffer.length > 0) await this.#handleProtocolLine(input, stripCarriageReturn(buffer));
891
+ }
892
+
893
+ async #consumeStderr(input: {
894
+ readonly requestKey: string;
895
+ readonly worker: WorkerProjection;
896
+ readonly attempt: WorkerAttemptProjection;
897
+ readonly child: ManagedWorkerProcess;
898
+ readonly state: MonitorState;
899
+ }): Promise<void> {
900
+ for await (const chunk of input.child.stderr) {
901
+ input.state.outputBytes += chunk.byteLength;
902
+ input.state.stderrTail = appendTail(
903
+ input.state.stderrTail,
904
+ chunk,
905
+ WORKER_STDERR_TAIL_LIMIT_BYTES,
906
+ );
907
+ await this.#enforceObservedBudgets(input);
908
+ }
909
+ }
910
+
911
+ async #consumeLines(
912
+ initial: string,
913
+ flush: boolean,
914
+ consume: (line: string, rest: string) => Promise<void>,
915
+ ): Promise<void> {
916
+ let buffer = initial;
917
+ while (true) {
918
+ const index = buffer.indexOf("\n");
919
+ if (index < 0) break;
920
+ const line = stripCarriageReturn(buffer.slice(0, index));
921
+ const rest = buffer.slice(index + 1);
922
+ await consume(line, rest);
923
+ buffer = rest;
924
+ }
925
+ if (flush && buffer.length > 0) await consume(stripCarriageReturn(buffer), "");
926
+ }
927
+
928
+ async #handleProtocolLine(input: {
929
+ readonly requestKey: string;
930
+ readonly worker: WorkerProjection;
931
+ readonly attempt: WorkerAttemptProjection;
932
+ readonly state: MonitorState;
933
+ readonly resolveReady: () => void;
934
+ }, line: string): Promise<void> {
935
+ if (line.length === 0) return;
936
+ if (Buffer.byteLength(line, "utf8") > WORKER_PROTOCOL_LINE_LIMIT_BYTES) {
937
+ throw protocolOutputLimit();
938
+ }
939
+ let value: unknown;
940
+ try {
941
+ value = JSON.parse(line) as unknown;
942
+ } catch (error) {
943
+ throw new WorkerProtocolError(
944
+ "protocol_invalid_json",
945
+ `Worker protocol line is invalid JSON: ${toError(error).message}`,
946
+ );
947
+ }
948
+ if (!isRecord(value)) throw new Error("Worker protocol line must be a JSON object");
949
+ if (value["type"] === "pi_amq_worker_refused") {
950
+ const diagnostic = String(value["error"] ?? "unknown reason");
951
+ throw new ModelFallbackEligibleError(
952
+ `launch-validation: ${diagnostic}`,
953
+ `Child launch refused: ${diagnostic}`,
954
+ );
955
+ }
956
+ if (value["type"] === "pi_amq_worker_provider_failure") {
957
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
958
+ const reason = typeof value["reason"] === "string" ? value["reason"] : "";
959
+ if (!/^provider-(auth|quota|capacity): /.test(reason)) {
960
+ throw new Error("Worker provider-failure envelope has an invalid reason");
961
+ }
962
+ throw new ModelFallbackEligibleError(reason, `Provider failure: ${reason}`);
963
+ }
964
+ if (value["type"] === "pi_amq_worker_wait") {
965
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
966
+ const reason = typeof value["reason"] === "string"
967
+ ? boundedProtocolText(value["reason"], "worker wait reason", 2_048)
968
+ : "";
969
+ const correlation = typeof value["correlation"] === "string"
970
+ ? boundedProtocolText(value["correlation"], "worker wait correlation", 256)
971
+ : "";
972
+ await this.#store.recordWorkerWaitDeclared(input.requestKey, {
973
+ workerId: input.worker.workerId,
974
+ attemptId: input.attempt.attemptId,
975
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
976
+ reason,
977
+ correlation,
978
+ });
979
+ return;
980
+ }
981
+ if (value["type"] === "pi_amq_worker_peer_message") {
982
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
983
+ const target = typeof value["target"] === "string" ? nonEmpty(value["target"], "peer target") : "";
984
+ const message = typeof value["message"] === "string" ? nonEmpty(value["message"], "peer message") : "";
985
+ const correlation = typeof value["correlation"] === "string"
986
+ ? nonEmpty(value["correlation"], "peer correlation")
987
+ : "";
988
+ if (message.length > 32_768 || target.length > 256 || correlation.length > 256) {
989
+ throw new Error("Peer message exceeds its bounded protocol fields");
990
+ }
991
+ await this.#routePeerMessage(input.requestKey, input.worker.workerId, target, message, correlation);
992
+ return;
993
+ }
994
+ if (value["type"] === "pi_amq_worker_ready") {
995
+ assertReadyEnvelope(value, input.worker, input.attempt, this.#supervisorGeneration);
996
+ const sessionFile = value["sessionFile"];
997
+ if (typeof sessionFile !== "string") {
998
+ throw new Error("Worker ready envelope requires a session file");
999
+ }
1000
+ await this.#store.recordWorkerReady(input.requestKey, {
1001
+ workerId: input.worker.workerId,
1002
+ attemptId: input.attempt.attemptId,
1003
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
1004
+ sessionFile,
1005
+ });
1006
+ input.state.ready = true;
1007
+ input.resolveReady();
1008
+ return;
1009
+ }
1010
+ if (value["type"] === "message_end") {
1011
+ const message = isRecord(value["message"]) ? value["message"] : undefined;
1012
+ if (!message || message["role"] !== "assistant") return;
1013
+ const usage = isRecord(message["usage"]) ? message["usage"] : undefined;
1014
+ const tokens = usage?.["totalTokens"];
1015
+ if (typeof tokens === "number" && Number.isFinite(tokens) && tokens >= 0) {
1016
+ input.state.tokens += Math.floor(tokens);
1017
+ }
1018
+ const cost = isRecord(usage?.["cost"]) ? usage["cost"]["total"] : undefined;
1019
+ if (typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
1020
+ input.state.costUsd += cost;
1021
+ }
1022
+ const text = assistantText(message["content"]);
1023
+ if (text !== null) {
1024
+ input.state.candidateValue = text;
1025
+ input.state.candidateHash = sha256(text);
1026
+ }
1027
+ await this.#store.recordAttemptUsage(input.requestKey, {
1028
+ workerId: input.worker.workerId,
1029
+ attemptId: input.attempt.attemptId,
1030
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
1031
+ tokens: input.state.tokens,
1032
+ costUsd: input.state.costUsd,
1033
+ outputBytes: input.state.outputBytes,
1034
+ });
1035
+ const providerFailure = providerFallbackReason(message);
1036
+ if (providerFailure) {
1037
+ throw new ModelFallbackEligibleError(providerFailure, `Provider failure: ${providerFailure}`);
1038
+ }
1039
+ return;
1040
+ }
1041
+ if (value["type"] === "pi_amq_worker_settled") {
1042
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
1043
+ const managed = this.#managed.get(input.worker.workerId);
1044
+ if (managed?.attemptId === input.attempt.attemptId) {
1045
+ managed.inferenceActive = false;
1046
+ const revisionDelivered = await this.#deliverNextRevision(managed);
1047
+ if (!revisionDelivered) await this.#deliverNextPeerMessage(managed);
1048
+ }
1049
+ }
1050
+ }
1051
+
1052
+ async #routePeerMessage(
1053
+ requestKey: string,
1054
+ senderWorkerId: WorkerId,
1055
+ target: string,
1056
+ message: string,
1057
+ correlation: string,
1058
+ ): Promise<void> {
1059
+ const projection = await this.#requiredRun(requestKey);
1060
+ const sender = requiredWorker(projection, senderWorkerId);
1061
+ const grant = `message_worker:${target}`;
1062
+ if (!effectiveWorkerContract(sender).capabilities.includes(grant)) {
1063
+ throw new Error(`Peer message target is not directionally granted: ${target}`);
1064
+ }
1065
+ const candidates = target.startsWith("worker_")
1066
+ ? [projection.workers[target as WorkerId]].filter((worker): worker is WorkerProjection => worker !== undefined)
1067
+ : Object.values(projection.workers).filter((worker) =>
1068
+ effectiveWorkerContract(worker).role?.name === target);
1069
+ if (candidates.length !== 1) {
1070
+ throw new Error(`Peer message target ${target} resolved to ${candidates.length} workers`);
1071
+ }
1072
+ const recipient = candidates[0]!;
1073
+ if (recipient.workerId === senderWorkerId) throw new Error("Peer message cannot target its sender");
1074
+ const recipientAttempt = currentAttempt(recipient);
1075
+ const managed = this.#managed.get(recipient.workerId);
1076
+ if (!managed
1077
+ || managed.attemptId !== recipientAttempt.attemptId
1078
+ || managed.workerAttemptGeneration !== recipientAttempt.workerAttemptGeneration) {
1079
+ throw new Error(`Peer message recipient ${recipient.workerId} has no current attached process`);
1080
+ }
1081
+ const requested = await this.#store.recordPeerMessageRequested(requestKey, {
1082
+ senderWorkerId,
1083
+ recipientWorkerId: recipient.workerId,
1084
+ grant,
1085
+ correlation,
1086
+ messageHash: sha256(message),
1087
+ });
1088
+ const pending = {
1089
+ messageId: requested.message.messageId,
1090
+ senderWorkerId,
1091
+ message,
1092
+ correlation,
1093
+ };
1094
+ if (managed.inferenceActive) {
1095
+ managed.pendingPeerMessages.push(pending);
1096
+ return;
1097
+ }
1098
+ await this.#deliverPeerMessage(managed, pending);
1099
+ }
1100
+
1101
+ async #deliverNextPeerMessage(managed: ManagedAttempt): Promise<boolean> {
1102
+ const projection = await this.#requiredRun(managed.requestKey);
1103
+ const worker = requiredWorker(projection, managed.workerId);
1104
+ const index = managed.pendingPeerMessages.findIndex((peer) =>
1105
+ correlationCanWake(worker, peer.correlation));
1106
+ if (index < 0) return false;
1107
+ const [next] = managed.pendingPeerMessages.splice(index, 1);
1108
+ if (!next) return false;
1109
+ await this.#deliverPeerMessage(managed, next);
1110
+ return true;
1111
+ }
1112
+
1113
+ async #deliverPeerMessage(
1114
+ managed: ManagedAttempt,
1115
+ peer: ManagedAttempt["pendingPeerMessages"][number],
1116
+ ): Promise<void> {
1117
+ const prompt = [
1118
+ "<peer-message>",
1119
+ JSON.stringify({
1120
+ senderWorkerId: peer.senderWorkerId,
1121
+ correlation: peer.correlation,
1122
+ message: peer.message,
1123
+ }),
1124
+ "</peer-message>",
1125
+ ].join("\n");
1126
+ try {
1127
+ await managed.child.writeLine(JSON.stringify({
1128
+ id: `peer-${peer.messageId}`,
1129
+ type: "prompt",
1130
+ message: prompt,
1131
+ }));
1132
+ managed.inferenceActive = true;
1133
+ await this.#store.recordPeerMessageDelivery(managed.requestKey, {
1134
+ messageId: peer.messageId,
1135
+ recipientWorkerId: managed.workerId,
1136
+ recipientAttemptId: managed.attemptId,
1137
+ recipientWorkerAttemptGeneration: managed.workerAttemptGeneration,
1138
+ status: "delivered",
1139
+ });
1140
+ } catch (error) {
1141
+ await this.#store.recordPeerMessageDelivery(managed.requestKey, {
1142
+ messageId: peer.messageId,
1143
+ recipientWorkerId: managed.workerId,
1144
+ recipientAttemptId: managed.attemptId,
1145
+ recipientWorkerAttemptGeneration: managed.workerAttemptGeneration,
1146
+ status: "indeterminate",
1147
+ diagnostic: toError(error).message,
1148
+ }).catch(() => {});
1149
+ throw error;
1150
+ }
1151
+ }
1152
+
1153
+ async #deliverNextRevision(managed: ManagedAttempt): Promise<boolean> {
1154
+ const next = managed.pendingRevisions[0];
1155
+ if (!next) return false;
1156
+ const projection = await this.#requiredRun(managed.requestKey);
1157
+ const worker = requiredWorker(projection, managed.workerId);
1158
+ const revision = worker.contractRevisions.find(
1159
+ (candidate) => candidate.revisionId === next.revisionId,
1160
+ );
1161
+ if (!revision) throw new Error(`Unknown queued contract revision ${next.revisionId}`);
1162
+ if (!correlationCanWake(worker, revision.correlation)) return false;
1163
+ managed.pendingRevisions.shift();
1164
+ await managed.child.writeLine(JSON.stringify({
1165
+ id: `revision-${next.revisionId}`,
1166
+ type: "prompt",
1167
+ message: next.prompt,
1168
+ }));
1169
+ managed.inferenceActive = true;
1170
+ await this.#recordRevisionDelivered(managed, next.revisionId);
1171
+ return true;
1172
+ }
1173
+
1174
+ async #recordRevisionDelivered(
1175
+ managed: ManagedAttempt,
1176
+ revisionId: ContractRevisionId,
1177
+ ): Promise<RunProjectionV2> {
1178
+ try {
1179
+ return await this.#store.recordContractRevisionDelivered(managed.requestKey, {
1180
+ workerId: managed.workerId,
1181
+ revisionId,
1182
+ boundary: "next-inference",
1183
+ });
1184
+ } catch (error) {
1185
+ await this.#store.recordWorkerNeedsAttention(managed.requestKey, {
1186
+ workerId: managed.workerId,
1187
+ attemptId: managed.attemptId,
1188
+ workerAttemptGeneration: managed.workerAttemptGeneration,
1189
+ reason: `Contract revision reached the worker but ledger delivery is uncertain: ${toError(error).message}`,
1190
+ ambiguous: true,
1191
+ }).catch(() => {});
1192
+ throw error;
1193
+ }
1194
+ }
1195
+
1196
+ async #enforceObservedBudgets(input: {
1197
+ readonly requestKey: string;
1198
+ readonly worker: WorkerProjection;
1199
+ readonly state: MonitorState;
1200
+ }): Promise<void> {
1201
+ const budget = input.worker.contract.budget;
1202
+ const tokenExceeded = budgetExceeded(input.state.tokens, budget.maxTokens);
1203
+ const costExceeded = budgetExceeded(input.state.costUsd, budget.maxCostUsd);
1204
+ const outputExceeded = budgetExceeded(input.state.outputBytes, budget.maxOutputBytes);
1205
+ if (!tokenExceeded && !costExceeded && !outputExceeded) return;
1206
+ const reason = tokenExceeded
1207
+ ? "worker token budget exhausted"
1208
+ : costExceeded
1209
+ ? "worker cost budget exhausted"
1210
+ : "worker output-byte budget exhausted";
1211
+ let current = await this.#requiredRun(input.requestKey);
1212
+ let attempt = currentAttempt(requiredWorker(current, input.worker.workerId));
1213
+ current = await this.#store.recordAttemptUsage(input.requestKey, {
1214
+ workerId: input.worker.workerId,
1215
+ attemptId: attempt.attemptId,
1216
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
1217
+ tokens: input.state.tokens,
1218
+ costUsd: input.state.costUsd,
1219
+ outputBytes: input.state.outputBytes,
1220
+ });
1221
+ attempt = currentAttempt(requiredWorker(current, input.worker.workerId));
1222
+ if (attempt.cancellation.requestedAt === null) {
1223
+ await this.cancelWorker(input.requestKey, {
1224
+ workerId: input.worker.workerId,
1225
+ reason,
1226
+ });
1227
+ }
1228
+ }
1229
+
1230
+ async #inspectAttempt(attempt: WorkerAttemptProjection): Promise<"live" | "dead" | "unknown"> {
1231
+ const pid = attempt.pidStartEvidence.pid;
1232
+ if (pid === null) return "unknown";
1233
+ const inspection = await this.#inspector.inspect(pid);
1234
+ // A PID lookup is lease/liveness evidence, not an authoritative child-exit event.
1235
+ // Absence can race platform process accounting and must stay fail-closed.
1236
+ if (inspection.kind === "absent") return "unknown";
1237
+ if (inspection.kind === "unknown") return "unknown";
1238
+ return processIdentityMatchesAttempt(inspection.identity, attempt)
1239
+ ? "live"
1240
+ : "dead";
1241
+ }
1242
+
1243
+ async #continuationSource(run: RunProjectionV2, workerId: WorkerId): Promise<{
1244
+ readonly priorWorkerId: WorkerId;
1245
+ readonly priorAttemptId: AttemptId;
1246
+ readonly priorLeaseGeneration: string;
1247
+ readonly sessionPath: string;
1248
+ readonly sessionFile: string;
1249
+ }> {
1250
+ const worker = requiredWorker(run, workerId);
1251
+ const attempt = currentAttempt(worker);
1252
+ if (worker.state !== "done" && worker.state !== "failed" && worker.state !== "cancelled") {
1253
+ throw new ResumeRefusedError(
1254
+ `Continuation prior worker is not terminal: ${worker.state}`,
1255
+ );
1256
+ }
1257
+ if (attempt.pidStartEvidence.observedExitAt === null
1258
+ || attempt.pidStartEvidence.exitStatus === null
1259
+ || attempt.pidStartEvidence.pid === null) {
1260
+ throw new ResumeRefusedError("Continuation prior process death is not durably observed");
1261
+ }
1262
+ const inspection = await this.#inspector.inspect(attempt.pidStartEvidence.pid);
1263
+ if (inspection.kind === "unknown") {
1264
+ throw new ResumeRefusedError(
1265
+ `Continuation refused because prior liveness is unknown: ${inspection.diagnostic}`,
1266
+ );
1267
+ }
1268
+ if (inspection.kind === "live" && processIdentityMatchesAttempt(inspection.identity, attempt)) {
1269
+ throw new ResumeRefusedError("Continuation refused because prior liveness is live");
1270
+ }
1271
+ if (attempt.session.file === null || attempt.session.leaseGeneration === null) {
1272
+ throw new ResumeRefusedError("Continuation prior session file and lease generation are required");
1273
+ }
1274
+ return {
1275
+ priorWorkerId: worker.workerId,
1276
+ priorAttemptId: attempt.attemptId,
1277
+ priorLeaseGeneration: attempt.session.leaseGeneration,
1278
+ sessionPath: attempt.session.path,
1279
+ sessionFile: attempt.session.file,
1280
+ };
1281
+ }
1282
+
1283
+ async #requiredRun(requestKey: string): Promise<RunProjectionV2> {
1284
+ const projection = await this.#store.readRun(requestKey);
1285
+ if (!projection) throw new Error(`Unknown run request ${requestKey}`);
1286
+ return projection;
1287
+ }
1288
+
1289
+ #sessionDirectory(runId: string, workerRequestKey: string): string {
1290
+ return join(
1291
+ this.#namespaceDirectory,
1292
+ "worker-sessions",
1293
+ sha256(`${runId}\0${workerRequestKey}`).slice("sha256:".length),
1294
+ );
1295
+ }
1296
+ }
1297
+
1298
+ export class NodePiProcessFactory implements PiProcessFactory {
1299
+ async spawn(command: PiProcessLaunchCommand): Promise<ManagedWorkerProcess> {
1300
+ const child = spawnChild(command.executable, [...command.args], {
1301
+ cwd: command.cwd,
1302
+ env: { ...command.env },
1303
+ shell: false,
1304
+ detached: process.platform !== "win32",
1305
+ stdio: ["pipe", "pipe", "pipe", "pipe"],
1306
+ });
1307
+ const exited = new Promise<ProcessExit>((resolve) => {
1308
+ child.once("exit", (code, signal) => resolve({ code, signal }));
1309
+ });
1310
+ await waitForSpawn(child);
1311
+ if (child.pid === undefined || child.stdout === null || child.stderr === null || child.stdin === null) {
1312
+ throw new Error("Pi child did not expose the required process pipes");
1313
+ }
1314
+ const pid = child.pid;
1315
+ return {
1316
+ pid,
1317
+ stdout: child.stdout,
1318
+ stderr: child.stderr,
1319
+ exited,
1320
+ writeLine: async (line) => writeToChild(child, `${line}\n`),
1321
+ endInput: async () => { child.stdin?.end(); },
1322
+ signalProcessGroup: async (signal) => signalGroup(child, pid, signal),
1323
+ closeParentLiveness: async () => { child.stdio[command.parentLivenessFd]?.destroy(); },
1324
+ };
1325
+ }
1326
+ }
1327
+
1328
+ class NodeWorkerTimerScheduler implements WorkerTimerScheduler {
1329
+ schedule(delayMs: number, callback: () => void): { cancel(): void } {
1330
+ const timer = setTimeout(callback, Math.max(0, delayMs));
1331
+ timer.unref();
1332
+ return { cancel: () => clearTimeout(timer) };
1333
+ }
1334
+ }
1335
+
1336
+ async function waitForSpawn(child: ChildProcess): Promise<void> {
1337
+ await new Promise<void>((resolve, reject) => {
1338
+ const onSpawn = (): void => { cleanup(); resolve(); };
1339
+ const onError = (error: Error): void => { cleanup(); reject(error); };
1340
+ const cleanup = (): void => {
1341
+ child.off("spawn", onSpawn);
1342
+ child.off("error", onError);
1343
+ };
1344
+ child.once("spawn", onSpawn);
1345
+ child.once("error", onError);
1346
+ });
1347
+ }
1348
+
1349
+ async function writeToChild(child: ChildProcess, value: string): Promise<void> {
1350
+ if (!child.stdin || child.stdin.destroyed) throw new Error("Pi child stdin is unavailable");
1351
+ await new Promise<void>((resolve, reject) => {
1352
+ child.stdin!.write(value, "utf8", (error) => error ? reject(error) : resolve());
1353
+ });
1354
+ }
1355
+
1356
+ async function signalGroup(child: ChildProcess, pid: number, signal: NodeJS.Signals): Promise<void> {
1357
+ try {
1358
+ if (process.platform === "win32") child.kill(signal);
1359
+ else process.kill(-pid, signal);
1360
+ } catch (error) {
1361
+ if (!isNodeError(error) || error.code !== "ESRCH") throw error;
1362
+ }
1363
+ }
1364
+
1365
+ function assertReadyEnvelope(
1366
+ value: Record<string, unknown>,
1367
+ worker: WorkerProjection,
1368
+ attempt: WorkerAttemptProjection,
1369
+ supervisorGeneration: string,
1370
+ ): void {
1371
+ assertProtocolIdentity(value, worker, attempt, supervisorGeneration);
1372
+ }
1373
+
1374
+ function assertProtocolIdentity(
1375
+ value: Record<string, unknown>,
1376
+ worker: WorkerProjection,
1377
+ attempt: WorkerAttemptProjection,
1378
+ supervisorGeneration: string,
1379
+ ): void {
1380
+ if (value["runId"] !== worker.contract.runId
1381
+ || value["workerId"] !== worker.workerId
1382
+ || value["attemptId"] !== attempt.attemptId
1383
+ || value["workerAttemptGeneration"] !== attempt.workerAttemptGeneration
1384
+ || value["supervisorGeneration"] !== supervisorGeneration) {
1385
+ throw new Error("Worker protocol envelope generation or identity mismatch");
1386
+ }
1387
+ }
1388
+
1389
+ function assistantText(value: unknown): string | null {
1390
+ if (typeof value === "string") return value;
1391
+ if (!Array.isArray(value)) return null;
1392
+ const text = value.flatMap((item) =>
1393
+ isRecord(item) && item["type"] === "text" && typeof item["text"] === "string"
1394
+ ? [item["text"]]
1395
+ : []).join("");
1396
+ return text.length === 0 ? null : text;
1397
+ }
1398
+
1399
+ function providerFallbackReason(message: Record<string, unknown>): string | null {
1400
+ return classifyProviderFallbackFailure(message["stopReason"], message["errorMessage"]);
1401
+ }
1402
+
1403
+ function currentAttempt(worker: WorkerProjection): WorkerAttemptProjection {
1404
+ const attempt = worker.attempts.at(-1);
1405
+ if (!attempt) throw new Error(`Worker ${worker.workerId} has no attempt`);
1406
+ return attempt;
1407
+ }
1408
+
1409
+ function processIdentityMatchesAttempt(
1410
+ identity: ProcessIdentity,
1411
+ attempt: WorkerAttemptProjection,
1412
+ ): boolean {
1413
+ const recorded = attempt.pidStartEvidence;
1414
+ return identity.processStartIdentity === recorded.processStartIdentity
1415
+ && identity.bootIdentity === recorded.bootIdentity
1416
+ && identity.executable === recorded.executable
1417
+ && identity.hostname === recorded.hostname;
1418
+ }
1419
+
1420
+ function requiredWorker(projection: RunProjectionV2, workerId: WorkerId): WorkerProjection {
1421
+ const worker = projection.workers[workerId];
1422
+ if (!worker) throw new Error(`Unknown worker ${workerId}`);
1423
+ return worker;
1424
+ }
1425
+
1426
+ function formatExit(exit: ProcessExit): string {
1427
+ return exit.signal ?? `code ${exit.code ?? "unknown"}`;
1428
+ }
1429
+
1430
+ function stripCarriageReturn(line: string): string {
1431
+ return line.endsWith("\r") ? line.slice(0, -1) : line;
1432
+ }
1433
+
1434
+ function isRecord(value: unknown): value is Record<string, unknown> {
1435
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1436
+ }
1437
+
1438
+ function isNodeError(error: unknown): error is NodeJS.ErrnoException {
1439
+ return error instanceof Error && "code" in error;
1440
+ }
1441
+
1442
+ function toError(error: unknown): Error {
1443
+ return error instanceof Error ? error : new Error(String(error));
1444
+ }
1445
+
1446
+ function protocolOutputLimit(): WorkerProtocolError {
1447
+ return new WorkerProtocolError(
1448
+ "protocol_output_limit",
1449
+ `Worker protocol JSONL line exceeds ${WORKER_PROTOCOL_LINE_LIMIT_BYTES} bytes`,
1450
+ WORKER_PROTOCOL_LINE_LIMIT_BYTES,
1451
+ );
1452
+ }
1453
+
1454
+ function protocolErrorDetails(error: Error): string {
1455
+ return error instanceof WorkerProtocolError
1456
+ ? JSON.stringify({
1457
+ type: "protocol_error",
1458
+ code: error.code,
1459
+ message: error.message,
1460
+ ...(error.limitBytes === undefined ? {} : { limitBytes: error.limitBytes }),
1461
+ })
1462
+ : error.message;
1463
+ }
1464
+
1465
+ function appendTail(current: Uint8Array, chunk: Uint8Array, limit: number): Uint8Array {
1466
+ if (chunk.byteLength >= limit) return chunk.slice(chunk.byteLength - limit);
1467
+ const keep = Math.min(current.byteLength, limit - chunk.byteLength);
1468
+ const result = new Uint8Array(keep + chunk.byteLength);
1469
+ result.set(current.slice(current.byteLength - keep), 0);
1470
+ result.set(chunk, keep);
1471
+ return result;
1472
+ }
1473
+
1474
+ function nonEmpty(value: string, name: string): string {
1475
+ if (value.length === 0) throw new TypeError(`${name} must not be empty`);
1476
+ return value;
1477
+ }
1478
+
1479
+ function boundedProtocolText(value: string, name: string, maxLength: number): string {
1480
+ if (value.trim().length === 0) throw new TypeError(`${name} must not be empty`);
1481
+ if (value.length > maxLength) throw new TypeError(`${name} exceeds ${maxLength} characters`);
1482
+ return value;
1483
+ }
1484
+
1485
+ function correlationCanWake(worker: WorkerProjection, correlation: string | null): boolean {
1486
+ if (worker.state === "running") return true;
1487
+ if (worker.state !== "blocked") return false;
1488
+ const active = worker.waits.filter((wait) => wait.status === "waiting");
1489
+ return active.length === 1
1490
+ && correlation !== null
1491
+ && active[0]!.correlation === correlation;
1492
+ }
1493
+
1494
+ function assertBlockedWakeCorrelation(
1495
+ worker: WorkerProjection,
1496
+ correlation: string | null,
1497
+ ): void {
1498
+ if (!correlationCanWake(worker, correlation)) {
1499
+ const expected = worker.waits.filter((wait) => wait.status === "waiting").at(-1)?.correlation;
1500
+ throw new Error(
1501
+ `Input correlation does not match typed wait${expected ? ` ${expected}` : ""}`,
1502
+ );
1503
+ }
1504
+ }