pi-amq 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/CHANGELOG.md +66 -0
  2. package/LICENSE +21 -0
  3. package/README.md +92 -0
  4. package/THIRD_PARTY_NOTICES.md +33 -0
  5. package/dist/extension/guidance.d.ts +25 -0
  6. package/dist/extension/guidance.d.ts.map +1 -0
  7. package/dist/extension/guidance.js +121 -0
  8. package/dist/extension/guidance.js.map +1 -0
  9. package/dist/extension/guide.d.ts +30 -0
  10. package/dist/extension/guide.d.ts.map +1 -0
  11. package/dist/extension/guide.js +128 -0
  12. package/dist/extension/guide.js.map +1 -0
  13. package/dist/extension/index.d.ts +4 -0
  14. package/dist/extension/index.d.ts.map +1 -0
  15. package/dist/extension/index.js +6 -0
  16. package/dist/extension/index.js.map +1 -0
  17. package/dist/extension/runtime.d.ts +48 -0
  18. package/dist/extension/runtime.d.ts.map +1 -0
  19. package/dist/extension/runtime.js +1095 -0
  20. package/dist/extension/runtime.js.map +1 -0
  21. package/dist/extension/tools.d.ts +12 -0
  22. package/dist/extension/tools.d.ts.map +1 -0
  23. package/dist/extension/tools.js +198 -0
  24. package/dist/extension/tools.js.map +1 -0
  25. package/dist/extension/worker-runtime.d.ts +34 -0
  26. package/dist/extension/worker-runtime.d.ts.map +1 -0
  27. package/dist/extension/worker-runtime.js +974 -0
  28. package/dist/extension/worker-runtime.js.map +1 -0
  29. package/dist/extension/worker-tools.d.ts +140 -0
  30. package/dist/extension/worker-tools.d.ts.map +1 -0
  31. package/dist/extension/worker-tools.js +217 -0
  32. package/dist/extension/worker-tools.js.map +1 -0
  33. package/dist/integration/confirmed-bind-onboarding.d.ts +44 -0
  34. package/dist/integration/confirmed-bind-onboarding.d.ts.map +1 -0
  35. package/dist/integration/confirmed-bind-onboarding.js +199 -0
  36. package/dist/integration/confirmed-bind-onboarding.js.map +1 -0
  37. package/dist/integration/fleet-view.d.ts +66 -0
  38. package/dist/integration/fleet-view.d.ts.map +1 -0
  39. package/dist/integration/fleet-view.js +238 -0
  40. package/dist/integration/fleet-view.js.map +1 -0
  41. package/dist/integration/fresh-session-dispatcher.d.ts +66 -0
  42. package/dist/integration/fresh-session-dispatcher.d.ts.map +1 -0
  43. package/dist/integration/fresh-session-dispatcher.js +103 -0
  44. package/dist/integration/fresh-session-dispatcher.js.map +1 -0
  45. package/dist/integration/launch-preview.d.ts +31 -0
  46. package/dist/integration/launch-preview.d.ts.map +1 -0
  47. package/dist/integration/launch-preview.js +325 -0
  48. package/dist/integration/launch-preview.js.map +1 -0
  49. package/dist/integration/transport-port.d.ts +39 -0
  50. package/dist/integration/transport-port.d.ts.map +1 -0
  51. package/dist/integration/transport-port.js +2 -0
  52. package/dist/integration/transport-port.js.map +1 -0
  53. package/dist/integration/worker-completion-notifier.d.ts +22 -0
  54. package/dist/integration/worker-completion-notifier.d.ts.map +1 -0
  55. package/dist/integration/worker-completion-notifier.js +81 -0
  56. package/dist/integration/worker-completion-notifier.js.map +1 -0
  57. package/dist/transport/amq-cli-adapter.d.ts +41 -0
  58. package/dist/transport/amq-cli-adapter.d.ts.map +1 -0
  59. package/dist/transport/amq-cli-adapter.js +291 -0
  60. package/dist/transport/amq-cli-adapter.js.map +1 -0
  61. package/dist/transport/amq-message.d.ts +47 -0
  62. package/dist/transport/amq-message.d.ts.map +1 -0
  63. package/dist/transport/amq-message.js +173 -0
  64. package/dist/transport/amq-message.js.map +1 -0
  65. package/dist/transport/amq-operations.d.ts +105 -0
  66. package/dist/transport/amq-operations.d.ts.map +1 -0
  67. package/dist/transport/amq-operations.js +320 -0
  68. package/dist/transport/amq-operations.js.map +1 -0
  69. package/dist/transport/atomic-json.d.ts +18 -0
  70. package/dist/transport/atomic-json.d.ts.map +1 -0
  71. package/dist/transport/atomic-json.js +90 -0
  72. package/dist/transport/atomic-json.js.map +1 -0
  73. package/dist/transport/binding-resolver.d.ts +46 -0
  74. package/dist/transport/binding-resolver.d.ts.map +1 -0
  75. package/dist/transport/binding-resolver.js +389 -0
  76. package/dist/transport/binding-resolver.js.map +1 -0
  77. package/dist/transport/bridge-journal.d.ts +100 -0
  78. package/dist/transport/bridge-journal.d.ts.map +1 -0
  79. package/dist/transport/bridge-journal.js +459 -0
  80. package/dist/transport/bridge-journal.js.map +1 -0
  81. package/dist/transport/dlq-retry-audit.d.ts +28 -0
  82. package/dist/transport/dlq-retry-audit.d.ts.map +1 -0
  83. package/dist/transport/dlq-retry-audit.js +66 -0
  84. package/dist/transport/dlq-retry-audit.js.map +1 -0
  85. package/dist/transport/inbox-pump.d.ts +81 -0
  86. package/dist/transport/inbox-pump.d.ts.map +1 -0
  87. package/dist/transport/inbox-pump.js +558 -0
  88. package/dist/transport/inbox-pump.js.map +1 -0
  89. package/dist/transport/index.d.ts +12 -0
  90. package/dist/transport/index.d.ts.map +1 -0
  91. package/dist/transport/index.js +12 -0
  92. package/dist/transport/index.js.map +1 -0
  93. package/dist/transport/presence-projector.d.ts +29 -0
  94. package/dist/transport/presence-projector.d.ts.map +1 -0
  95. package/dist/transport/presence-projector.js +73 -0
  96. package/dist/transport/presence-projector.js.map +1 -0
  97. package/dist/transport/priority-delivery-scheduler.d.ts +85 -0
  98. package/dist/transport/priority-delivery-scheduler.d.ts.map +1 -0
  99. package/dist/transport/priority-delivery-scheduler.js +210 -0
  100. package/dist/transport/priority-delivery-scheduler.js.map +1 -0
  101. package/dist/transport/receiver-lease.d.ts +102 -0
  102. package/dist/transport/receiver-lease.d.ts.map +1 -0
  103. package/dist/transport/receiver-lease.js +778 -0
  104. package/dist/transport/receiver-lease.js.map +1 -0
  105. package/dist/transport/runtime-status.d.ts +48 -0
  106. package/dist/transport/runtime-status.d.ts.map +1 -0
  107. package/dist/transport/runtime-status.js +71 -0
  108. package/dist/transport/runtime-status.js.map +1 -0
  109. package/dist/workflow/atomic-storage.d.ts +12 -0
  110. package/dist/workflow/atomic-storage.d.ts.map +1 -0
  111. package/dist/workflow/atomic-storage.js +104 -0
  112. package/dist/workflow/atomic-storage.js.map +1 -0
  113. package/dist/workflow/budget.d.ts +18 -0
  114. package/dist/workflow/budget.d.ts.map +1 -0
  115. package/dist/workflow/budget.js +74 -0
  116. package/dist/workflow/budget.js.map +1 -0
  117. package/dist/workflow/index.d.ts +17 -0
  118. package/dist/workflow/index.d.ts.map +1 -0
  119. package/dist/workflow/index.js +17 -0
  120. package/dist/workflow/index.js.map +1 -0
  121. package/dist/workflow/lifecycle-projection.d.ts +18 -0
  122. package/dist/workflow/lifecycle-projection.d.ts.map +1 -0
  123. package/dist/workflow/lifecycle-projection.js +22 -0
  124. package/dist/workflow/lifecycle-projection.js.map +1 -0
  125. package/dist/workflow/model-fallback.d.ts +2 -0
  126. package/dist/workflow/model-fallback.d.ts.map +1 -0
  127. package/dist/workflow/model-fallback.js +17 -0
  128. package/dist/workflow/model-fallback.js.map +1 -0
  129. package/dist/workflow/pi-session-fork.d.ts +18 -0
  130. package/dist/workflow/pi-session-fork.d.ts.map +1 -0
  131. package/dist/workflow/pi-session-fork.js +80 -0
  132. package/dist/workflow/pi-session-fork.js.map +1 -0
  133. package/dist/workflow/run-artifacts.d.ts +5 -0
  134. package/dist/workflow/run-artifacts.d.ts.map +1 -0
  135. package/dist/workflow/run-artifacts.js +25 -0
  136. package/dist/workflow/run-artifacts.js.map +1 -0
  137. package/dist/workflow/run-cleanup.d.ts +70 -0
  138. package/dist/workflow/run-cleanup.d.ts.map +1 -0
  139. package/dist/workflow/run-cleanup.js +222 -0
  140. package/dist/workflow/run-cleanup.js.map +1 -0
  141. package/dist/workflow/run-gate.d.ts +43 -0
  142. package/dist/workflow/run-gate.d.ts.map +1 -0
  143. package/dist/workflow/run-gate.js +70 -0
  144. package/dist/workflow/run-gate.js.map +1 -0
  145. package/dist/workflow/run-store.d.ts +208 -0
  146. package/dist/workflow/run-store.d.ts.map +1 -0
  147. package/dist/workflow/run-store.js +2965 -0
  148. package/dist/workflow/run-store.js.map +1 -0
  149. package/dist/workflow/spawn-confirmation.d.ts +44 -0
  150. package/dist/workflow/spawn-confirmation.d.ts.map +1 -0
  151. package/dist/workflow/spawn-confirmation.js +228 -0
  152. package/dist/workflow/spawn-confirmation.js.map +1 -0
  153. package/dist/workflow/system-process-inspector.d.ts +9 -0
  154. package/dist/workflow/system-process-inspector.d.ts.map +1 -0
  155. package/dist/workflow/system-process-inspector.js +129 -0
  156. package/dist/workflow/system-process-inspector.js.map +1 -0
  157. package/dist/workflow/worker-child-extension.d.ts +3 -0
  158. package/dist/workflow/worker-child-extension.d.ts.map +1 -0
  159. package/dist/workflow/worker-child-extension.js +182 -0
  160. package/dist/workflow/worker-child-extension.js.map +1 -0
  161. package/dist/workflow/worker-gate.d.ts +74 -0
  162. package/dist/workflow/worker-gate.d.ts.map +1 -0
  163. package/dist/workflow/worker-gate.js +165 -0
  164. package/dist/workflow/worker-gate.js.map +1 -0
  165. package/dist/workflow/worker-launch.d.ts +65 -0
  166. package/dist/workflow/worker-launch.d.ts.map +1 -0
  167. package/dist/workflow/worker-launch.js +338 -0
  168. package/dist/workflow/worker-launch.js.map +1 -0
  169. package/dist/workflow/worker-model-catalog.d.ts +36 -0
  170. package/dist/workflow/worker-model-catalog.d.ts.map +1 -0
  171. package/dist/workflow/worker-model-catalog.js +105 -0
  172. package/dist/workflow/worker-model-catalog.js.map +1 -0
  173. package/dist/workflow/worker-process-supervisor.d.ts +110 -0
  174. package/dist/workflow/worker-process-supervisor.d.ts.map +1 -0
  175. package/dist/workflow/worker-process-supervisor.js +1188 -0
  176. package/dist/workflow/worker-process-supervisor.js.map +1 -0
  177. package/dist/workflow/worker-prompt-capsule.d.ts +32 -0
  178. package/dist/workflow/worker-prompt-capsule.d.ts.map +1 -0
  179. package/dist/workflow/worker-prompt-capsule.js +52 -0
  180. package/dist/workflow/worker-prompt-capsule.js.map +1 -0
  181. package/dist/workflow/worker-role-presets.d.ts +38 -0
  182. package/dist/workflow/worker-role-presets.d.ts.map +1 -0
  183. package/dist/workflow/worker-role-presets.js +201 -0
  184. package/dist/workflow/worker-role-presets.js.map +1 -0
  185. package/dist/workflow/workflow-doctor.d.ts +57 -0
  186. package/dist/workflow/workflow-doctor.d.ts.map +1 -0
  187. package/dist/workflow/workflow-doctor.js +289 -0
  188. package/dist/workflow/workflow-doctor.js.map +1 -0
  189. package/dist/workflow/workflow-lease.d.ts +125 -0
  190. package/dist/workflow/workflow-lease.d.ts.map +1 -0
  191. package/dist/workflow/workflow-lease.js +689 -0
  192. package/dist/workflow/workflow-lease.js.map +1 -0
  193. package/dist/workflow/workflow-types.d.ts +335 -0
  194. package/dist/workflow/workflow-types.d.ts.map +1 -0
  195. package/dist/workflow/workflow-types.js +11 -0
  196. package/dist/workflow/workflow-types.js.map +1 -0
  197. package/dist/workflow/workspace-paths.d.ts +4 -0
  198. package/dist/workflow/workspace-paths.d.ts.map +1 -0
  199. package/dist/workflow/workspace-paths.js +22 -0
  200. package/dist/workflow/workspace-paths.js.map +1 -0
  201. package/docs/conformance.md +116 -0
  202. package/docs/demo-transcript.md +34 -0
  203. package/docs/design.md +1481 -0
  204. package/docs/operator-guide.md +100 -0
  205. package/extension/guidance.ts +162 -0
  206. package/extension/guide.ts +155 -0
  207. package/extension/index.ts +8 -0
  208. package/extension/runtime.ts +1224 -0
  209. package/extension/tools.ts +221 -0
  210. package/extension/worker-runtime.ts +1150 -0
  211. package/extension/worker-tools.ts +276 -0
  212. package/integration/confirmed-bind-onboarding.ts +296 -0
  213. package/integration/fleet-view.ts +286 -0
  214. package/integration/fresh-session-dispatcher.ts +166 -0
  215. package/integration/launch-preview.ts +396 -0
  216. package/integration/transport-port.ts +29 -0
  217. package/integration/worker-completion-notifier.ts +98 -0
  218. package/package.json +55 -0
  219. package/roles/oracle.md +9 -0
  220. package/roles/researcher.md +8 -0
  221. package/roles/reviewer.md +9 -0
  222. package/roles/scout.md +8 -0
  223. package/roles/worker.md +10 -0
  224. package/scripts/demo-two-terminal.sh +59 -0
  225. package/skills/coordination-patterns/SKILL.md +41 -0
  226. package/skills/coordination-patterns/references/supervised-worker-fan-out.md +12 -0
  227. package/skills/coordination-patterns/references/today-mail-team.md +36 -0
  228. package/transport/amq-cli-adapter.ts +381 -0
  229. package/transport/amq-message.ts +218 -0
  230. package/transport/amq-operations.ts +482 -0
  231. package/transport/atomic-json.ts +115 -0
  232. package/transport/binding-resolver.ts +565 -0
  233. package/transport/bridge-journal.ts +588 -0
  234. package/transport/dlq-retry-audit.ts +97 -0
  235. package/transport/inbox-pump.ts +729 -0
  236. package/transport/index.ts +11 -0
  237. package/transport/presence-projector.ts +104 -0
  238. package/transport/priority-delivery-scheduler.ts +328 -0
  239. package/transport/receiver-lease.ts +945 -0
  240. package/transport/runtime-status.ts +112 -0
  241. package/workflow/.gitkeep +1 -0
  242. package/workflow/atomic-storage.ts +120 -0
  243. package/workflow/budget.ts +90 -0
  244. package/workflow/index.ts +16 -0
  245. package/workflow/lifecycle-projection.ts +37 -0
  246. package/workflow/model-fallback.ts +17 -0
  247. package/workflow/pi-session-fork.ts +96 -0
  248. package/workflow/run-artifacts.ts +36 -0
  249. package/workflow/run-cleanup.ts +315 -0
  250. package/workflow/run-gate.ts +122 -0
  251. package/workflow/run-store.ts +3890 -0
  252. package/workflow/spawn-confirmation.ts +305 -0
  253. package/workflow/system-process-inspector.ts +142 -0
  254. package/workflow/worker-child-extension.ts +193 -0
  255. package/workflow/worker-gate.ts +238 -0
  256. package/workflow/worker-launch.ts +485 -0
  257. package/workflow/worker-model-catalog.ts +165 -0
  258. package/workflow/worker-process-supervisor.ts +1504 -0
  259. package/workflow/worker-prompt-capsule.ts +81 -0
  260. package/workflow/worker-role-presets.ts +241 -0
  261. package/workflow/workflow-doctor.ts +400 -0
  262. package/workflow/workflow-lease.ts +905 -0
  263. package/workflow/workflow-types.ts +367 -0
  264. package/workflow/workspace-paths.ts +32 -0
@@ -0,0 +1,1188 @@
1
+ import { spawn as spawnChild } from "node:child_process";
2
+ import { mkdir } from "node:fs/promises";
3
+ import { join } from "node:path";
4
+ import { sha256, writeWorkflowJson } from "./atomic-storage.js";
5
+ import { budgetExceeded, budgetValue } from "./budget.js";
6
+ import { NodePiSessionForker } from "./pi-session-fork.js";
7
+ import { classifyProviderFallbackFailure } from "./model-fallback.js";
8
+ import { effectiveWorkerContract, BudgetExceededError, WorkflowRunStore, } from "./run-store.js";
9
+ import { buildWorkerPrompt } from "./worker-prompt-capsule.js";
10
+ import { resolveWorkerLaunch, } from "./worker-launch.js";
11
+ import { WorkflowLeaseManager, } from "./workflow-lease.js";
12
+ export const WORKER_PROTOCOL_LINE_LIMIT_BYTES = 4 * 1024 * 1024;
13
+ export const WORKER_STDERR_TAIL_LIMIT_BYTES = 128 * 1024;
14
+ export class WorkerProtocolError extends Error {
15
+ code;
16
+ limitBytes;
17
+ constructor(code, message, limitBytes) {
18
+ super(message);
19
+ this.code = code;
20
+ this.limitBytes = limitBytes;
21
+ this.name = "WorkerProtocolError";
22
+ }
23
+ }
24
+ export { BudgetExceededError } from "./run-store.js";
25
+ export class ResumeRefusedError extends Error {
26
+ constructor(message) {
27
+ super(message);
28
+ this.name = "ResumeRefusedError";
29
+ }
30
+ }
31
+ class ModelFallbackEligibleError extends Error {
32
+ fallbackReason;
33
+ constructor(fallbackReason, message) {
34
+ super(message);
35
+ this.fallbackReason = fallbackReason;
36
+ this.name = "ModelFallbackEligibleError";
37
+ }
38
+ }
39
+ export class WorkerProcessSupervisor {
40
+ #namespaceDirectory;
41
+ #store;
42
+ #leaseManager;
43
+ #supervisorGeneration;
44
+ #inspector;
45
+ #processFactory;
46
+ #launchAuthority;
47
+ #timers;
48
+ #now;
49
+ #cancellationGraceMs;
50
+ #sessionForker;
51
+ #managed = new Map();
52
+ constructor(options) {
53
+ this.#namespaceDirectory = options.namespaceDirectory;
54
+ this.#store = options.store;
55
+ this.#leaseManager = options.leaseManager;
56
+ this.#supervisorGeneration = options.supervisorGeneration;
57
+ this.#inspector = options.inspector;
58
+ this.#processFactory = options.processFactory;
59
+ this.#launchAuthority = options.launchAuthority;
60
+ this.#timers = options.timerScheduler ?? new NodeWorkerTimerScheduler();
61
+ this.#now = options.now ?? (() => new Date());
62
+ this.#cancellationGraceMs = options.cancellationGraceMs ?? 2_000;
63
+ this.#sessionForker = options.sessionForker ?? new NodePiSessionForker();
64
+ }
65
+ updateLaunchAuthority(authority) {
66
+ this.#launchAuthority = authority;
67
+ }
68
+ async spawnWorker(requestKey, input) {
69
+ const run = await this.#requiredRun(requestKey);
70
+ if (input.continueWorkerId !== undefined && input.request.context === "fork") {
71
+ throw new TypeError("A worker cannot continue another worker session and fork driver context together");
72
+ }
73
+ const continuation = input.continueWorkerId === undefined
74
+ ? undefined
75
+ : await this.#continuationSource(run, input.continueWorkerId);
76
+ const sessionPath = continuation?.sessionPath
77
+ ?? this.#sessionDirectory(run.runId, input.request.requestKey);
78
+ const forkedSessionFile = input.request.context === "fork"
79
+ ? join(sessionPath, "session.jsonl")
80
+ : undefined;
81
+ const prompt = continuation === undefined
82
+ ? input.prompt
83
+ : buildWorkerPrompt({
84
+ ...input.request,
85
+ continuation: {
86
+ priorWorkerId: continuation.priorWorkerId,
87
+ priorAttemptId: continuation.priorAttemptId,
88
+ sessionFile: continuation.sessionFile,
89
+ },
90
+ });
91
+ // Pure preflight uses disposable runtime IDs. It proves launch authority before any ledger,
92
+ // lease, filesystem, or process effect.
93
+ resolveWorkerLaunch({
94
+ contract: {
95
+ ...input.request,
96
+ context: input.request.context ?? "fresh",
97
+ runId: run.runId,
98
+ workerId: "worker_preflight",
99
+ createdByDriverSessionId: run.source.driverSessionId,
100
+ },
101
+ attempt: {
102
+ attemptId: "attempt_preflight",
103
+ workerAttemptGeneration: "preflight-worker-generation",
104
+ supervisorGeneration: this.#supervisorGeneration,
105
+ },
106
+ depth: input.depth,
107
+ prompt,
108
+ sessionDirectory: sessionPath,
109
+ launchContractPath: join(this.#namespaceDirectory, "launch-contracts", "preflight.json"),
110
+ ...(continuation ? { continuedSessionFile: continuation.sessionFile } : {}),
111
+ ...(forkedSessionFile ? { forkedSessionFile } : {}),
112
+ modelOverride: input.request.model,
113
+ authority: this.#launchAuthority,
114
+ });
115
+ const prepared = await this.#store.prepareWorkerIntent(requestKey, {
116
+ createdByDriverSessionId: run.source.driverSessionId,
117
+ sessionPath,
118
+ request: input.request,
119
+ parentWorkerId: input.parentWorkerId ?? null,
120
+ depth: input.depth,
121
+ ...(continuation ? { continuation: {
122
+ priorWorkerId: continuation.priorWorkerId,
123
+ priorAttemptId: continuation.priorAttemptId,
124
+ priorLeaseGeneration: continuation.priorLeaseGeneration,
125
+ sessionFile: continuation.sessionFile,
126
+ } } : {}),
127
+ });
128
+ const attempt = currentAttempt(prepared.worker);
129
+ if (prepared.kind === "existing") {
130
+ const managed = this.#managed.get(prepared.worker.workerId);
131
+ if (managed?.attemptId === attempt.attemptId) {
132
+ return { kind: "existing", worker: prepared.worker, attempt };
133
+ }
134
+ return { kind: "needs-reconcile", worker: prepared.worker, attempt };
135
+ }
136
+ if (prepared.worker.contract.context === "fork") {
137
+ const forkSource = prepared.worker.contract.forkSource;
138
+ if (!forkSource || !forkedSessionFile)
139
+ throw new Error("Fork worker context is incomplete");
140
+ try {
141
+ await this.#sessionForker.fork({ ...forkSource, targetSessionFile: forkedSessionFile });
142
+ }
143
+ catch (error) {
144
+ await this.#store.recordAttemptExit(requestKey, {
145
+ workerId: prepared.worker.workerId,
146
+ attemptId: attempt.attemptId,
147
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
148
+ exitStatus: { code: 1, signal: null },
149
+ candidateOutcome: { valueOrRef: null, hash: null },
150
+ uncertainSideEffects: false,
151
+ reason: `Pi session fork failed before process creation: ${toError(error).message}`,
152
+ });
153
+ throw error;
154
+ }
155
+ }
156
+ return this.#startPreparedAttempt(requestKey, prepared.worker, attempt, prompt);
157
+ }
158
+ async resumeWorker(requestKey, input) {
159
+ let run = await this.#requiredRun(requestKey);
160
+ let worker = requiredWorker(run, input.workerId);
161
+ let attempt = currentAttempt(worker);
162
+ if (attempt.state === "ambiguous" || attempt.sideEffects.status === "uncertain") {
163
+ throw new ResumeRefusedError(`Worker ${input.workerId} has unresolved side-effect ambiguity`);
164
+ }
165
+ if (attempt.state === "running") {
166
+ const inspection = await this.#inspectAttempt(attempt);
167
+ if (inspection !== "dead") {
168
+ throw new ResumeRefusedError(`Worker ${input.workerId} cannot resume because prior liveness is ${inspection}`);
169
+ }
170
+ run = await this.#store.recordAttemptExit(requestKey, {
171
+ workerId: worker.workerId,
172
+ attemptId: attempt.attemptId,
173
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
174
+ exitStatus: { code: 1, signal: "proven-dead-before-resume" },
175
+ candidateOutcome: attempt.candidateOutcome,
176
+ uncertainSideEffects: false,
177
+ reason: "Prior worker process is proven dead before explicit resume",
178
+ });
179
+ worker = requiredWorker(run, input.workerId);
180
+ attempt = currentAttempt(worker);
181
+ }
182
+ if (attempt.state === "pending") {
183
+ throw new ResumeRefusedError(`Worker ${input.workerId} has an unreconciled spawn intent; replay is forbidden`);
184
+ }
185
+ const prepared = await this.#store.prepareReplacementAttempt(requestKey, {
186
+ workerId: input.workerId,
187
+ });
188
+ return this.#startPreparedAttempt(requestKey, prepared.worker, prepared.attempt, input.prompt);
189
+ }
190
+ async cancelWorker(requestKey, input) {
191
+ let projection = await this.#store.requestWorkerCancellation(requestKey, input);
192
+ const attempt = currentAttempt(requiredWorker(projection, input.workerId));
193
+ const managed = this.#managed.get(input.workerId);
194
+ if (!managed || managed.attemptId !== attempt.attemptId
195
+ || managed.workerAttemptGeneration !== attempt.workerAttemptGeneration) {
196
+ return {
197
+ kind: "indeterminate",
198
+ projection,
199
+ reason: "No current supervised process is attached to the durable cancellation intent",
200
+ };
201
+ }
202
+ if (attempt.launchContract?.channel === "rpc") {
203
+ await managed.child.writeLine(JSON.stringify({
204
+ id: `cancel-${attempt.attemptId}`,
205
+ type: "abort",
206
+ })).catch(() => { });
207
+ }
208
+ try {
209
+ await managed.child.signalProcessGroup("SIGTERM");
210
+ }
211
+ catch (error) {
212
+ projection = await this.#store.recordWorkerNeedsAttention(requestKey, {
213
+ workerId: input.workerId,
214
+ attemptId: attempt.attemptId,
215
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
216
+ reason: `Cancellation signal outcome is unknown: ${toError(error).message}`,
217
+ ambiguous: true,
218
+ });
219
+ return { kind: "indeterminate", projection, reason: toError(error).message };
220
+ }
221
+ projection = await this.#store.recordCancellationSignalSent(requestKey, {
222
+ workerId: input.workerId,
223
+ attemptId: attempt.attemptId,
224
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
225
+ signal: "SIGTERM",
226
+ });
227
+ managed.cancellationEscalation?.cancel();
228
+ managed.cancellationEscalation = this.#timers.schedule(this.#cancellationGraceMs, () => {
229
+ const current = this.#managed.get(input.workerId);
230
+ if (current?.attemptId === attempt.attemptId) {
231
+ void current.child.signalProcessGroup("SIGKILL").catch(() => { });
232
+ }
233
+ });
234
+ return { kind: "requested", projection };
235
+ }
236
+ async steerWorker(requestKey, input) {
237
+ const managed = this.#managed.get(input.workerId);
238
+ if (!managed)
239
+ throw new Error(`Worker ${input.workerId} has no attached RPC process`);
240
+ const projection = await this.#requiredRun(requestKey);
241
+ const worker = requiredWorker(projection, input.workerId);
242
+ if (effectiveWorkerContract(worker).channel !== "rpc") {
243
+ throw new Error("Only an RPC worker can be steered");
244
+ }
245
+ const message = nonEmpty(input.message, "worker steer message");
246
+ const correlation = input.correlation === undefined
247
+ ? null
248
+ : boundedProtocolText(input.correlation, "worker steer correlation", 256);
249
+ assertBlockedWakeCorrelation(worker, correlation);
250
+ await managed.child.writeLine(JSON.stringify({
251
+ id: `steer-${managed.attemptId}`,
252
+ type: "steer",
253
+ message,
254
+ }));
255
+ managed.inferenceActive = true;
256
+ try {
257
+ return await this.#store.recordSteerDelivered(requestKey, {
258
+ workerId: input.workerId,
259
+ messageHash: sha256(message),
260
+ ...(correlation === null ? {} : { correlation }),
261
+ });
262
+ }
263
+ catch (error) {
264
+ await this.#store.recordWorkerNeedsAttention(requestKey, {
265
+ workerId: worker.workerId,
266
+ attemptId: managed.attemptId,
267
+ workerAttemptGeneration: managed.workerAttemptGeneration,
268
+ reason: `Steer reached the worker but correlated delivery ledgering failed: ${toError(error).message}`,
269
+ ambiguous: true,
270
+ }).catch(() => { });
271
+ throw error;
272
+ }
273
+ }
274
+ async reviseWorkerContract(requestKey, input) {
275
+ const before = await this.#requiredRun(requestKey);
276
+ const workerBefore = requiredWorker(before, input.workerId);
277
+ if (effectiveWorkerContract(workerBefore).channel !== "rpc") {
278
+ throw new Error("Contract revision delivery requires an RPC worker");
279
+ }
280
+ let projection = await this.#store.requestContractRevision(requestKey, input);
281
+ const worker = requiredWorker(projection, input.workerId);
282
+ const revision = worker.contractRevisions.at(-1);
283
+ if (!revision || revision.status !== "pending") {
284
+ throw new Error("Contract revision was not recorded as pending");
285
+ }
286
+ const managed = this.#managed.get(input.workerId);
287
+ if (!managed)
288
+ return projection;
289
+ const prompt = [
290
+ `Apply immutable contract revision ${revision.revision} at this inference boundary.`,
291
+ buildWorkerPrompt(revision.contract),
292
+ ].join("\n\n");
293
+ if (managed.inferenceActive) {
294
+ managed.pendingRevisions.push({ revisionId: revision.revisionId, prompt });
295
+ return projection;
296
+ }
297
+ await managed.child.writeLine(JSON.stringify({
298
+ id: `revision-${revision.revisionId}`,
299
+ type: "prompt",
300
+ message: prompt,
301
+ }));
302
+ managed.inferenceActive = true;
303
+ projection = await this.#recordRevisionDelivered(managed, revision.revisionId);
304
+ return projection;
305
+ }
306
+ async reconcileRun(requestKey) {
307
+ let projection = await this.#requiredRun(requestKey);
308
+ const reconciledPeerMessages = new Set();
309
+ for (const worker of Object.values(projection.workers)) {
310
+ for (const peer of worker.peerMessages) {
311
+ if (peer.status !== "pending" || reconciledPeerMessages.has(peer.messageId))
312
+ continue;
313
+ reconciledPeerMessages.add(peer.messageId);
314
+ const recipient = requiredWorker(projection, peer.recipientWorkerId);
315
+ const recipientAttempt = currentAttempt(recipient);
316
+ projection = await this.#store.recordPeerMessageDelivery(requestKey, {
317
+ messageId: peer.messageId,
318
+ recipientWorkerId: recipient.workerId,
319
+ recipientAttemptId: recipientAttempt.attemptId,
320
+ recipientWorkerAttemptGeneration: recipientAttempt.workerAttemptGeneration,
321
+ status: "indeterminate",
322
+ diagnostic: "Pending peer-message body was not persisted and cannot be replayed after supervisor restart",
323
+ });
324
+ }
325
+ }
326
+ for (const worker of Object.values(projection.workers)) {
327
+ const attempt = currentAttempt(worker);
328
+ if (attempt.state === "pending") {
329
+ projection = await this.#store.recordWorkerNeedsAttention(requestKey, {
330
+ workerId: worker.workerId,
331
+ attemptId: attempt.attemptId,
332
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
333
+ reason: "Recovered spawn intent has no durable process-start evidence; no replay performed",
334
+ ambiguous: true,
335
+ });
336
+ continue;
337
+ }
338
+ if (attempt.state !== "running")
339
+ continue;
340
+ const liveness = await this.#inspectAttempt(attempt);
341
+ if (liveness === "dead") {
342
+ projection = await this.#store.recordAttemptExit(requestKey, {
343
+ workerId: worker.workerId,
344
+ attemptId: attempt.attemptId,
345
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
346
+ exitStatus: { code: 1, signal: "proven-dead-on-recovery" },
347
+ candidateOutcome: attempt.candidateOutcome,
348
+ uncertainSideEffects: attempt.sideEffects.policy === "external"
349
+ || attempt.sideEffects.policy === "reconcilable",
350
+ reason: "Worker was proven dead during restart reconciliation",
351
+ });
352
+ }
353
+ else {
354
+ projection = await this.#store.recordWorkerNeedsAttention(requestKey, {
355
+ workerId: worker.workerId,
356
+ attemptId: attempt.attemptId,
357
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
358
+ reason: liveness === "live"
359
+ ? "Worker remains live but its control pipe was not reattached"
360
+ : "Worker liveness could not be verified after restart",
361
+ ambiguous: liveness === "unknown",
362
+ });
363
+ }
364
+ }
365
+ return projection;
366
+ }
367
+ async #startPreparedAttempt(requestKey, worker, attempt, prompt) {
368
+ const launchContractPath = join(this.#namespaceDirectory, "launch-contracts", worker.workerId, `${attempt.attemptId}.json`);
369
+ const resolved = resolveWorkerLaunch({
370
+ contract: worker.contract,
371
+ attempt: {
372
+ attemptId: attempt.attemptId,
373
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
374
+ supervisorGeneration: attempt.supervisorGeneration,
375
+ },
376
+ depth: worker.depth,
377
+ prompt,
378
+ sessionDirectory: attempt.session.path,
379
+ launchContractPath,
380
+ ...(worker.continuation
381
+ ? { continuedSessionFile: worker.continuation.sessionFile }
382
+ : {}),
383
+ ...(worker.contract.context === "fork"
384
+ ? { forkedSessionFile: join(attempt.session.path, "session.jsonl") }
385
+ : {}),
386
+ modelOverride: attempt.modelSelection?.model ?? worker.contract.model,
387
+ authority: this.#launchAuthority,
388
+ });
389
+ await mkdir(attempt.session.path, { recursive: true, mode: 0o700 });
390
+ await mkdir(join(this.#namespaceDirectory, "launch-contracts", worker.workerId), {
391
+ recursive: true,
392
+ mode: 0o700,
393
+ });
394
+ await writeWorkflowJson(launchContractPath, resolved.contract, { createOnly: true });
395
+ const reservation = await this.#leaseManager.reserveWorkerSession({
396
+ sessionPath: attempt.session.path,
397
+ runId: worker.contract.runId,
398
+ workerId: worker.workerId,
399
+ attemptId: attempt.attemptId,
400
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
401
+ supervisorGeneration: this.#supervisorGeneration,
402
+ });
403
+ if (reservation.kind !== "acquired") {
404
+ await this.#store.recordWorkerNeedsAttention(requestKey, {
405
+ workerId: worker.workerId,
406
+ attemptId: attempt.attemptId,
407
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
408
+ reason: reservation.kind === "held"
409
+ ? `Worker session is held by generation ${reservation.owner.generation}`
410
+ : reservation.diagnostic,
411
+ ambiguous: reservation.kind !== "held",
412
+ });
413
+ throw new ResumeRefusedError("Worker session could not be reserved before process creation");
414
+ }
415
+ let child;
416
+ try {
417
+ child = await this.#processFactory.spawn(resolved.command);
418
+ }
419
+ catch (error) {
420
+ await this.#leaseManager.releaseWorkerSession({
421
+ sessionPath: attempt.session.path,
422
+ generation: reservation.owner.generation,
423
+ supervisorGeneration: this.#supervisorGeneration,
424
+ });
425
+ await this.#store.recordAttemptExit(requestKey, {
426
+ workerId: worker.workerId,
427
+ attemptId: attempt.attemptId,
428
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
429
+ exitStatus: { code: 127, signal: null },
430
+ candidateOutcome: { valueOrRef: null, hash: null },
431
+ uncertainSideEffects: false,
432
+ reason: `Pi process creation failed: ${toError(error).message}`,
433
+ });
434
+ throw error;
435
+ }
436
+ const inspection = await this.#inspector.inspect(child.pid);
437
+ if (inspection.kind !== "live") {
438
+ await child.signalProcessGroup("SIGKILL").catch(() => { });
439
+ const postKillInspection = await this.#inspector.inspect(child.pid);
440
+ if (postKillInspection.kind === "absent") {
441
+ await this.#leaseManager.releaseWorkerSession({
442
+ sessionPath: attempt.session.path,
443
+ generation: reservation.owner.generation,
444
+ supervisorGeneration: this.#supervisorGeneration,
445
+ });
446
+ }
447
+ await this.#store.recordWorkerNeedsAttention(requestKey, {
448
+ workerId: worker.workerId,
449
+ attemptId: attempt.attemptId,
450
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
451
+ reason: postKillInspection.kind === "unknown"
452
+ ? postKillInspection.diagnostic
453
+ : "Spawned child disappeared before process identity could be recorded",
454
+ ambiguous: true,
455
+ });
456
+ throw new Error("Spawned worker identity could not be proven");
457
+ }
458
+ const activeLease = await this.#leaseManager.activateWorkerSession({
459
+ sessionPath: attempt.session.path,
460
+ generation: reservation.owner.generation,
461
+ supervisorGeneration: this.#supervisorGeneration,
462
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
463
+ processIdentity: inspection.identity,
464
+ });
465
+ let projection = await this.#store.recordAttemptStarted(requestKey, {
466
+ workerId: worker.workerId,
467
+ attemptId: attempt.attemptId,
468
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
469
+ leaseGeneration: activeLease.generation,
470
+ processIdentity: inspection.identity,
471
+ launchContract: resolved.contract,
472
+ });
473
+ worker = requiredWorker(projection, worker.workerId);
474
+ attempt = currentAttempt(worker);
475
+ let resolveReady;
476
+ let rejectReady;
477
+ const ready = new Promise((resolve, reject) => {
478
+ resolveReady = resolve;
479
+ rejectReady = reject;
480
+ });
481
+ const state = {
482
+ outputBytes: 0,
483
+ tokens: 0,
484
+ costUsd: 0,
485
+ candidateValue: null,
486
+ candidateHash: null,
487
+ ready: false,
488
+ stderrTail: new Uint8Array(),
489
+ fallbackReason: null,
490
+ };
491
+ const wallClockMs = budgetValue(worker.contract.budget.maxWallClockMs);
492
+ const deadline = wallClockMs === undefined ? { cancel() { } } : this.#timers.schedule(wallClockMs, () => {
493
+ void this.cancelWorker(requestKey, {
494
+ workerId: worker.workerId,
495
+ reason: "worker wall-clock budget exhausted",
496
+ }).catch(() => { });
497
+ });
498
+ const managedPlaceholder = {};
499
+ const monitored = this.#monitorAttempt({
500
+ requestKey,
501
+ worker,
502
+ attempt,
503
+ child,
504
+ leaseGeneration: activeLease.generation,
505
+ state,
506
+ resolveReady,
507
+ rejectReady,
508
+ });
509
+ const managed = Object.assign(managedPlaceholder, {
510
+ requestKey,
511
+ workerId: worker.workerId,
512
+ attemptId: attempt.attemptId,
513
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
514
+ leaseGeneration: activeLease.generation,
515
+ child,
516
+ deadline,
517
+ inferenceActive: false,
518
+ pendingRevisions: [],
519
+ pendingPeerMessages: [],
520
+ });
521
+ this.#managed.set(worker.workerId, managed);
522
+ try {
523
+ await Promise.race([
524
+ ready,
525
+ child.exited.then((exit) => {
526
+ throw new Error(`Worker exited before launch parity was ready: ${formatExit(exit)}`);
527
+ }),
528
+ ]);
529
+ }
530
+ catch (error) {
531
+ const result = await monitored.catch(() => undefined);
532
+ if (result?.fallbackReason && this.#nextFallbackModel(result.projection, worker.workerId, attempt)) {
533
+ return this.#startModelFallback(requestKey, result.projection, worker.workerId, attempt, result.fallbackReason, prompt);
534
+ }
535
+ throw error;
536
+ }
537
+ if (worker.contract.channel === "rpc") {
538
+ await child.writeLine(JSON.stringify({
539
+ id: `prompt-${attempt.attemptId}`,
540
+ type: "prompt",
541
+ message: prompt,
542
+ }));
543
+ managed.inferenceActive = true;
544
+ }
545
+ else {
546
+ await child.endInput();
547
+ }
548
+ projection = await this.#requiredRun(requestKey);
549
+ worker = requiredWorker(projection, worker.workerId);
550
+ attempt = currentAttempt(worker);
551
+ const settlement = monitored.then((result) => this.#settleModelFallback(requestKey, result, worker.workerId, attempt, prompt));
552
+ return { kind: "started", worker, attempt, settlement };
553
+ }
554
+ async #settleModelFallback(requestKey, result, workerId, attempt, prompt) {
555
+ if (!result.fallbackReason)
556
+ return result.projection;
557
+ const next = this.#nextFallbackModel(result.projection, workerId, attempt);
558
+ if (!next)
559
+ return result.projection;
560
+ const fallback = await this.#startModelFallback(requestKey, result.projection, workerId, attempt, result.fallbackReason, prompt);
561
+ return fallback.kind === "started"
562
+ ? fallback.settlement
563
+ : this.#requiredRun(requestKey);
564
+ }
565
+ async #startModelFallback(requestKey, projection, workerId, attempt, reason, prompt) {
566
+ const next = this.#nextFallbackModel(projection, workerId, attempt);
567
+ if (!next)
568
+ throw new Error(`Model fallback chain exhausted after ${reason}`);
569
+ const prepared = await this.#store.prepareReplacementAttempt(requestKey, {
570
+ workerId,
571
+ model: next,
572
+ fallbackReason: reason,
573
+ });
574
+ return this.#startPreparedAttempt(requestKey, prepared.worker, prepared.attempt, prompt);
575
+ }
576
+ #nextFallbackModel(projection, workerId, attempt) {
577
+ const worker = requiredWorker(projection, workerId);
578
+ const chain = [worker.contract.model, ...(worker.contract.modelFallbacks ?? [])];
579
+ const attempted = attempt.modelSelection?.model ?? attempt.launchContract?.model ?? worker.contract.model;
580
+ const index = chain.findIndex((model) => model.provider === attempted.provider && model.id === attempted.id);
581
+ return index < 0 ? undefined : chain[index + 1];
582
+ }
583
+ async #monitorAttempt(input) {
584
+ let stdoutError;
585
+ const stdout = this.#consumeStdout(input).catch((error) => {
586
+ stdoutError = toError(error);
587
+ if (stdoutError instanceof ModelFallbackEligibleError) {
588
+ input.state.fallbackReason = stdoutError.fallbackReason;
589
+ input.rejectReady(stdoutError);
590
+ return;
591
+ }
592
+ input.rejectReady(stdoutError);
593
+ return this.cancelWorker(input.requestKey, {
594
+ workerId: input.worker.workerId,
595
+ reason: `worker protocol failed: ${stdoutError.message}`,
596
+ }).then(() => { }).catch(() => { });
597
+ });
598
+ const stderr = this.#consumeStderr(input);
599
+ const exit = await input.child.exited;
600
+ await Promise.all([stdout, stderr]);
601
+ const managed = this.#managed.get(input.worker.workerId);
602
+ managed?.deadline.cancel();
603
+ managed?.cancellationEscalation?.cancel();
604
+ await input.child.closeParentLiveness().catch(() => { });
605
+ let projection = await this.#store.recordAttemptUsage(input.requestKey, {
606
+ workerId: input.worker.workerId,
607
+ attemptId: input.attempt.attemptId,
608
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
609
+ tokens: input.state.tokens,
610
+ costUsd: input.state.costUsd,
611
+ outputBytes: input.state.outputBytes,
612
+ });
613
+ const uncertain = input.worker.contract.sideEffectPolicy === "external"
614
+ || input.worker.contract.sideEffectPolicy === "reconcilable";
615
+ projection = await this.#store.recordAttemptExit(input.requestKey, {
616
+ workerId: input.worker.workerId,
617
+ attemptId: input.attempt.attemptId,
618
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
619
+ exitStatus: stdoutError ? { code: 1, signal: exit.signal } : exit,
620
+ candidateOutcome: {
621
+ valueOrRef: input.state.candidateValue,
622
+ hash: input.state.candidateHash,
623
+ },
624
+ uncertainSideEffects: uncertain && (exit.code !== 0 || stdoutError !== undefined),
625
+ reason: stdoutError
626
+ ? `Worker protocol failed: ${protocolErrorDetails(stdoutError)}`
627
+ : `Worker process exited: ${formatExit(exit)}${exit.code === 0 || input.state.stderrTail.byteLength === 0
628
+ ? ""
629
+ : `; stderr_tail=${JSON.stringify(new TextDecoder().decode(input.state.stderrTail))}`}`,
630
+ });
631
+ await this.#leaseManager.releaseWorkerSession({
632
+ sessionPath: input.attempt.session.path,
633
+ generation: input.leaseGeneration,
634
+ supervisorGeneration: this.#supervisorGeneration,
635
+ });
636
+ if (this.#managed.get(input.worker.workerId)?.attemptId === input.attempt.attemptId) {
637
+ this.#managed.delete(input.worker.workerId);
638
+ }
639
+ return { projection, fallbackReason: input.state.fallbackReason };
640
+ }
641
+ async #consumeStdout(input) {
642
+ const decoder = new TextDecoder();
643
+ let buffer = "";
644
+ for await (const chunk of input.child.stdout) {
645
+ input.state.outputBytes += chunk.byteLength;
646
+ buffer += decoder.decode(chunk, { stream: true });
647
+ if (!buffer.includes("\n") && Buffer.byteLength(buffer, "utf8") > WORKER_PROTOCOL_LINE_LIMIT_BYTES) {
648
+ throw protocolOutputLimit();
649
+ }
650
+ await this.#consumeLines(buffer, false, async (line, rest) => {
651
+ buffer = rest;
652
+ await this.#handleProtocolLine(input, line);
653
+ });
654
+ await this.#enforceObservedBudgets(input);
655
+ }
656
+ buffer += decoder.decode();
657
+ if (buffer.length > 0)
658
+ await this.#handleProtocolLine(input, stripCarriageReturn(buffer));
659
+ }
660
+ async #consumeStderr(input) {
661
+ for await (const chunk of input.child.stderr) {
662
+ input.state.outputBytes += chunk.byteLength;
663
+ input.state.stderrTail = appendTail(input.state.stderrTail, chunk, WORKER_STDERR_TAIL_LIMIT_BYTES);
664
+ await this.#enforceObservedBudgets(input);
665
+ }
666
+ }
667
+ async #consumeLines(initial, flush, consume) {
668
+ let buffer = initial;
669
+ while (true) {
670
+ const index = buffer.indexOf("\n");
671
+ if (index < 0)
672
+ break;
673
+ const line = stripCarriageReturn(buffer.slice(0, index));
674
+ const rest = buffer.slice(index + 1);
675
+ await consume(line, rest);
676
+ buffer = rest;
677
+ }
678
+ if (flush && buffer.length > 0)
679
+ await consume(stripCarriageReturn(buffer), "");
680
+ }
681
+ async #handleProtocolLine(input, line) {
682
+ if (line.length === 0)
683
+ return;
684
+ if (Buffer.byteLength(line, "utf8") > WORKER_PROTOCOL_LINE_LIMIT_BYTES) {
685
+ throw protocolOutputLimit();
686
+ }
687
+ let value;
688
+ try {
689
+ value = JSON.parse(line);
690
+ }
691
+ catch (error) {
692
+ throw new WorkerProtocolError("protocol_invalid_json", `Worker protocol line is invalid JSON: ${toError(error).message}`);
693
+ }
694
+ if (!isRecord(value))
695
+ throw new Error("Worker protocol line must be a JSON object");
696
+ if (value["type"] === "pi_amq_worker_refused") {
697
+ const diagnostic = String(value["error"] ?? "unknown reason");
698
+ throw new ModelFallbackEligibleError(`launch-validation: ${diagnostic}`, `Child launch refused: ${diagnostic}`);
699
+ }
700
+ if (value["type"] === "pi_amq_worker_provider_failure") {
701
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
702
+ const reason = typeof value["reason"] === "string" ? value["reason"] : "";
703
+ if (!/^provider-(auth|quota|capacity): /.test(reason)) {
704
+ throw new Error("Worker provider-failure envelope has an invalid reason");
705
+ }
706
+ throw new ModelFallbackEligibleError(reason, `Provider failure: ${reason}`);
707
+ }
708
+ if (value["type"] === "pi_amq_worker_wait") {
709
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
710
+ const reason = typeof value["reason"] === "string"
711
+ ? boundedProtocolText(value["reason"], "worker wait reason", 2_048)
712
+ : "";
713
+ const correlation = typeof value["correlation"] === "string"
714
+ ? boundedProtocolText(value["correlation"], "worker wait correlation", 256)
715
+ : "";
716
+ await this.#store.recordWorkerWaitDeclared(input.requestKey, {
717
+ workerId: input.worker.workerId,
718
+ attemptId: input.attempt.attemptId,
719
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
720
+ reason,
721
+ correlation,
722
+ });
723
+ return;
724
+ }
725
+ if (value["type"] === "pi_amq_worker_peer_message") {
726
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
727
+ const target = typeof value["target"] === "string" ? nonEmpty(value["target"], "peer target") : "";
728
+ const message = typeof value["message"] === "string" ? nonEmpty(value["message"], "peer message") : "";
729
+ const correlation = typeof value["correlation"] === "string"
730
+ ? nonEmpty(value["correlation"], "peer correlation")
731
+ : "";
732
+ if (message.length > 32_768 || target.length > 256 || correlation.length > 256) {
733
+ throw new Error("Peer message exceeds its bounded protocol fields");
734
+ }
735
+ await this.#routePeerMessage(input.requestKey, input.worker.workerId, target, message, correlation);
736
+ return;
737
+ }
738
+ if (value["type"] === "pi_amq_worker_ready") {
739
+ assertReadyEnvelope(value, input.worker, input.attempt, this.#supervisorGeneration);
740
+ const sessionFile = value["sessionFile"];
741
+ if (typeof sessionFile !== "string") {
742
+ throw new Error("Worker ready envelope requires a session file");
743
+ }
744
+ await this.#store.recordWorkerReady(input.requestKey, {
745
+ workerId: input.worker.workerId,
746
+ attemptId: input.attempt.attemptId,
747
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
748
+ sessionFile,
749
+ });
750
+ input.state.ready = true;
751
+ input.resolveReady();
752
+ return;
753
+ }
754
+ if (value["type"] === "message_end") {
755
+ const message = isRecord(value["message"]) ? value["message"] : undefined;
756
+ if (!message || message["role"] !== "assistant")
757
+ return;
758
+ const usage = isRecord(message["usage"]) ? message["usage"] : undefined;
759
+ const tokens = usage?.["totalTokens"];
760
+ if (typeof tokens === "number" && Number.isFinite(tokens) && tokens >= 0) {
761
+ input.state.tokens += Math.floor(tokens);
762
+ }
763
+ const cost = isRecord(usage?.["cost"]) ? usage["cost"]["total"] : undefined;
764
+ if (typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
765
+ input.state.costUsd += cost;
766
+ }
767
+ const text = assistantText(message["content"]);
768
+ if (text !== null) {
769
+ input.state.candidateValue = text;
770
+ input.state.candidateHash = sha256(text);
771
+ }
772
+ await this.#store.recordAttemptUsage(input.requestKey, {
773
+ workerId: input.worker.workerId,
774
+ attemptId: input.attempt.attemptId,
775
+ workerAttemptGeneration: input.attempt.workerAttemptGeneration,
776
+ tokens: input.state.tokens,
777
+ costUsd: input.state.costUsd,
778
+ outputBytes: input.state.outputBytes,
779
+ });
780
+ const providerFailure = providerFallbackReason(message);
781
+ if (providerFailure) {
782
+ throw new ModelFallbackEligibleError(providerFailure, `Provider failure: ${providerFailure}`);
783
+ }
784
+ return;
785
+ }
786
+ if (value["type"] === "pi_amq_worker_settled") {
787
+ assertProtocolIdentity(value, input.worker, input.attempt, this.#supervisorGeneration);
788
+ const managed = this.#managed.get(input.worker.workerId);
789
+ if (managed?.attemptId === input.attempt.attemptId) {
790
+ managed.inferenceActive = false;
791
+ const revisionDelivered = await this.#deliverNextRevision(managed);
792
+ if (!revisionDelivered)
793
+ await this.#deliverNextPeerMessage(managed);
794
+ }
795
+ }
796
+ }
797
+ async #routePeerMessage(requestKey, senderWorkerId, target, message, correlation) {
798
+ const projection = await this.#requiredRun(requestKey);
799
+ const sender = requiredWorker(projection, senderWorkerId);
800
+ const grant = `message_worker:${target}`;
801
+ if (!effectiveWorkerContract(sender).capabilities.includes(grant)) {
802
+ throw new Error(`Peer message target is not directionally granted: ${target}`);
803
+ }
804
+ const candidates = target.startsWith("worker_")
805
+ ? [projection.workers[target]].filter((worker) => worker !== undefined)
806
+ : Object.values(projection.workers).filter((worker) => effectiveWorkerContract(worker).role?.name === target);
807
+ if (candidates.length !== 1) {
808
+ throw new Error(`Peer message target ${target} resolved to ${candidates.length} workers`);
809
+ }
810
+ const recipient = candidates[0];
811
+ if (recipient.workerId === senderWorkerId)
812
+ throw new Error("Peer message cannot target its sender");
813
+ const recipientAttempt = currentAttempt(recipient);
814
+ const managed = this.#managed.get(recipient.workerId);
815
+ if (!managed
816
+ || managed.attemptId !== recipientAttempt.attemptId
817
+ || managed.workerAttemptGeneration !== recipientAttempt.workerAttemptGeneration) {
818
+ throw new Error(`Peer message recipient ${recipient.workerId} has no current attached process`);
819
+ }
820
+ const requested = await this.#store.recordPeerMessageRequested(requestKey, {
821
+ senderWorkerId,
822
+ recipientWorkerId: recipient.workerId,
823
+ grant,
824
+ correlation,
825
+ messageHash: sha256(message),
826
+ });
827
+ const pending = {
828
+ messageId: requested.message.messageId,
829
+ senderWorkerId,
830
+ message,
831
+ correlation,
832
+ };
833
+ if (managed.inferenceActive) {
834
+ managed.pendingPeerMessages.push(pending);
835
+ return;
836
+ }
837
+ await this.#deliverPeerMessage(managed, pending);
838
+ }
839
+ async #deliverNextPeerMessage(managed) {
840
+ const projection = await this.#requiredRun(managed.requestKey);
841
+ const worker = requiredWorker(projection, managed.workerId);
842
+ const index = managed.pendingPeerMessages.findIndex((peer) => correlationCanWake(worker, peer.correlation));
843
+ if (index < 0)
844
+ return false;
845
+ const [next] = managed.pendingPeerMessages.splice(index, 1);
846
+ if (!next)
847
+ return false;
848
+ await this.#deliverPeerMessage(managed, next);
849
+ return true;
850
+ }
851
+ async #deliverPeerMessage(managed, peer) {
852
+ const prompt = [
853
+ "<peer-message>",
854
+ JSON.stringify({
855
+ senderWorkerId: peer.senderWorkerId,
856
+ correlation: peer.correlation,
857
+ message: peer.message,
858
+ }),
859
+ "</peer-message>",
860
+ ].join("\n");
861
+ try {
862
+ await managed.child.writeLine(JSON.stringify({
863
+ id: `peer-${peer.messageId}`,
864
+ type: "prompt",
865
+ message: prompt,
866
+ }));
867
+ managed.inferenceActive = true;
868
+ await this.#store.recordPeerMessageDelivery(managed.requestKey, {
869
+ messageId: peer.messageId,
870
+ recipientWorkerId: managed.workerId,
871
+ recipientAttemptId: managed.attemptId,
872
+ recipientWorkerAttemptGeneration: managed.workerAttemptGeneration,
873
+ status: "delivered",
874
+ });
875
+ }
876
+ catch (error) {
877
+ await this.#store.recordPeerMessageDelivery(managed.requestKey, {
878
+ messageId: peer.messageId,
879
+ recipientWorkerId: managed.workerId,
880
+ recipientAttemptId: managed.attemptId,
881
+ recipientWorkerAttemptGeneration: managed.workerAttemptGeneration,
882
+ status: "indeterminate",
883
+ diagnostic: toError(error).message,
884
+ }).catch(() => { });
885
+ throw error;
886
+ }
887
+ }
888
+ async #deliverNextRevision(managed) {
889
+ const next = managed.pendingRevisions[0];
890
+ if (!next)
891
+ return false;
892
+ const projection = await this.#requiredRun(managed.requestKey);
893
+ const worker = requiredWorker(projection, managed.workerId);
894
+ const revision = worker.contractRevisions.find((candidate) => candidate.revisionId === next.revisionId);
895
+ if (!revision)
896
+ throw new Error(`Unknown queued contract revision ${next.revisionId}`);
897
+ if (!correlationCanWake(worker, revision.correlation))
898
+ return false;
899
+ managed.pendingRevisions.shift();
900
+ await managed.child.writeLine(JSON.stringify({
901
+ id: `revision-${next.revisionId}`,
902
+ type: "prompt",
903
+ message: next.prompt,
904
+ }));
905
+ managed.inferenceActive = true;
906
+ await this.#recordRevisionDelivered(managed, next.revisionId);
907
+ return true;
908
+ }
909
+ async #recordRevisionDelivered(managed, revisionId) {
910
+ try {
911
+ return await this.#store.recordContractRevisionDelivered(managed.requestKey, {
912
+ workerId: managed.workerId,
913
+ revisionId,
914
+ boundary: "next-inference",
915
+ });
916
+ }
917
+ catch (error) {
918
+ await this.#store.recordWorkerNeedsAttention(managed.requestKey, {
919
+ workerId: managed.workerId,
920
+ attemptId: managed.attemptId,
921
+ workerAttemptGeneration: managed.workerAttemptGeneration,
922
+ reason: `Contract revision reached the worker but ledger delivery is uncertain: ${toError(error).message}`,
923
+ ambiguous: true,
924
+ }).catch(() => { });
925
+ throw error;
926
+ }
927
+ }
928
+ async #enforceObservedBudgets(input) {
929
+ const budget = input.worker.contract.budget;
930
+ const tokenExceeded = budgetExceeded(input.state.tokens, budget.maxTokens);
931
+ const costExceeded = budgetExceeded(input.state.costUsd, budget.maxCostUsd);
932
+ const outputExceeded = budgetExceeded(input.state.outputBytes, budget.maxOutputBytes);
933
+ if (!tokenExceeded && !costExceeded && !outputExceeded)
934
+ return;
935
+ const reason = tokenExceeded
936
+ ? "worker token budget exhausted"
937
+ : costExceeded
938
+ ? "worker cost budget exhausted"
939
+ : "worker output-byte budget exhausted";
940
+ let current = await this.#requiredRun(input.requestKey);
941
+ let attempt = currentAttempt(requiredWorker(current, input.worker.workerId));
942
+ current = await this.#store.recordAttemptUsage(input.requestKey, {
943
+ workerId: input.worker.workerId,
944
+ attemptId: attempt.attemptId,
945
+ workerAttemptGeneration: attempt.workerAttemptGeneration,
946
+ tokens: input.state.tokens,
947
+ costUsd: input.state.costUsd,
948
+ outputBytes: input.state.outputBytes,
949
+ });
950
+ attempt = currentAttempt(requiredWorker(current, input.worker.workerId));
951
+ if (attempt.cancellation.requestedAt === null) {
952
+ await this.cancelWorker(input.requestKey, {
953
+ workerId: input.worker.workerId,
954
+ reason,
955
+ });
956
+ }
957
+ }
958
+ async #inspectAttempt(attempt) {
959
+ const pid = attempt.pidStartEvidence.pid;
960
+ if (pid === null)
961
+ return "unknown";
962
+ const inspection = await this.#inspector.inspect(pid);
963
+ // A PID lookup is lease/liveness evidence, not an authoritative child-exit event.
964
+ // Absence can race platform process accounting and must stay fail-closed.
965
+ if (inspection.kind === "absent")
966
+ return "unknown";
967
+ if (inspection.kind === "unknown")
968
+ return "unknown";
969
+ return processIdentityMatchesAttempt(inspection.identity, attempt)
970
+ ? "live"
971
+ : "dead";
972
+ }
973
+ async #continuationSource(run, workerId) {
974
+ const worker = requiredWorker(run, workerId);
975
+ const attempt = currentAttempt(worker);
976
+ if (worker.state !== "done" && worker.state !== "failed" && worker.state !== "cancelled") {
977
+ throw new ResumeRefusedError(`Continuation prior worker is not terminal: ${worker.state}`);
978
+ }
979
+ if (attempt.pidStartEvidence.observedExitAt === null
980
+ || attempt.pidStartEvidence.exitStatus === null
981
+ || attempt.pidStartEvidence.pid === null) {
982
+ throw new ResumeRefusedError("Continuation prior process death is not durably observed");
983
+ }
984
+ const inspection = await this.#inspector.inspect(attempt.pidStartEvidence.pid);
985
+ if (inspection.kind === "unknown") {
986
+ throw new ResumeRefusedError(`Continuation refused because prior liveness is unknown: ${inspection.diagnostic}`);
987
+ }
988
+ if (inspection.kind === "live" && processIdentityMatchesAttempt(inspection.identity, attempt)) {
989
+ throw new ResumeRefusedError("Continuation refused because prior liveness is live");
990
+ }
991
+ if (attempt.session.file === null || attempt.session.leaseGeneration === null) {
992
+ throw new ResumeRefusedError("Continuation prior session file and lease generation are required");
993
+ }
994
+ return {
995
+ priorWorkerId: worker.workerId,
996
+ priorAttemptId: attempt.attemptId,
997
+ priorLeaseGeneration: attempt.session.leaseGeneration,
998
+ sessionPath: attempt.session.path,
999
+ sessionFile: attempt.session.file,
1000
+ };
1001
+ }
1002
+ async #requiredRun(requestKey) {
1003
+ const projection = await this.#store.readRun(requestKey);
1004
+ if (!projection)
1005
+ throw new Error(`Unknown run request ${requestKey}`);
1006
+ return projection;
1007
+ }
1008
+ #sessionDirectory(runId, workerRequestKey) {
1009
+ return join(this.#namespaceDirectory, "worker-sessions", sha256(`${runId}\0${workerRequestKey}`).slice("sha256:".length));
1010
+ }
1011
+ }
1012
+ export class NodePiProcessFactory {
1013
+ async spawn(command) {
1014
+ const child = spawnChild(command.executable, [...command.args], {
1015
+ cwd: command.cwd,
1016
+ env: { ...command.env },
1017
+ shell: false,
1018
+ detached: process.platform !== "win32",
1019
+ stdio: ["pipe", "pipe", "pipe", "pipe"],
1020
+ });
1021
+ const exited = new Promise((resolve) => {
1022
+ child.once("exit", (code, signal) => resolve({ code, signal }));
1023
+ });
1024
+ await waitForSpawn(child);
1025
+ if (child.pid === undefined || child.stdout === null || child.stderr === null || child.stdin === null) {
1026
+ throw new Error("Pi child did not expose the required process pipes");
1027
+ }
1028
+ const pid = child.pid;
1029
+ return {
1030
+ pid,
1031
+ stdout: child.stdout,
1032
+ stderr: child.stderr,
1033
+ exited,
1034
+ writeLine: async (line) => writeToChild(child, `${line}\n`),
1035
+ endInput: async () => { child.stdin?.end(); },
1036
+ signalProcessGroup: async (signal) => signalGroup(child, pid, signal),
1037
+ closeParentLiveness: async () => { child.stdio[command.parentLivenessFd]?.destroy(); },
1038
+ };
1039
+ }
1040
+ }
1041
+ class NodeWorkerTimerScheduler {
1042
+ schedule(delayMs, callback) {
1043
+ const timer = setTimeout(callback, Math.max(0, delayMs));
1044
+ timer.unref();
1045
+ return { cancel: () => clearTimeout(timer) };
1046
+ }
1047
+ }
1048
+ async function waitForSpawn(child) {
1049
+ await new Promise((resolve, reject) => {
1050
+ const onSpawn = () => { cleanup(); resolve(); };
1051
+ const onError = (error) => { cleanup(); reject(error); };
1052
+ const cleanup = () => {
1053
+ child.off("spawn", onSpawn);
1054
+ child.off("error", onError);
1055
+ };
1056
+ child.once("spawn", onSpawn);
1057
+ child.once("error", onError);
1058
+ });
1059
+ }
1060
+ async function writeToChild(child, value) {
1061
+ if (!child.stdin || child.stdin.destroyed)
1062
+ throw new Error("Pi child stdin is unavailable");
1063
+ await new Promise((resolve, reject) => {
1064
+ child.stdin.write(value, "utf8", (error) => error ? reject(error) : resolve());
1065
+ });
1066
+ }
1067
+ async function signalGroup(child, pid, signal) {
1068
+ try {
1069
+ if (process.platform === "win32")
1070
+ child.kill(signal);
1071
+ else
1072
+ process.kill(-pid, signal);
1073
+ }
1074
+ catch (error) {
1075
+ if (!isNodeError(error) || error.code !== "ESRCH")
1076
+ throw error;
1077
+ }
1078
+ }
1079
+ function assertReadyEnvelope(value, worker, attempt, supervisorGeneration) {
1080
+ assertProtocolIdentity(value, worker, attempt, supervisorGeneration);
1081
+ }
1082
+ function assertProtocolIdentity(value, worker, attempt, supervisorGeneration) {
1083
+ if (value["runId"] !== worker.contract.runId
1084
+ || value["workerId"] !== worker.workerId
1085
+ || value["attemptId"] !== attempt.attemptId
1086
+ || value["workerAttemptGeneration"] !== attempt.workerAttemptGeneration
1087
+ || value["supervisorGeneration"] !== supervisorGeneration) {
1088
+ throw new Error("Worker protocol envelope generation or identity mismatch");
1089
+ }
1090
+ }
1091
+ function assistantText(value) {
1092
+ if (typeof value === "string")
1093
+ return value;
1094
+ if (!Array.isArray(value))
1095
+ return null;
1096
+ const text = value.flatMap((item) => isRecord(item) && item["type"] === "text" && typeof item["text"] === "string"
1097
+ ? [item["text"]]
1098
+ : []).join("");
1099
+ return text.length === 0 ? null : text;
1100
+ }
1101
+ function providerFallbackReason(message) {
1102
+ return classifyProviderFallbackFailure(message["stopReason"], message["errorMessage"]);
1103
+ }
1104
+ function currentAttempt(worker) {
1105
+ const attempt = worker.attempts.at(-1);
1106
+ if (!attempt)
1107
+ throw new Error(`Worker ${worker.workerId} has no attempt`);
1108
+ return attempt;
1109
+ }
1110
+ function processIdentityMatchesAttempt(identity, attempt) {
1111
+ const recorded = attempt.pidStartEvidence;
1112
+ return identity.processStartIdentity === recorded.processStartIdentity
1113
+ && identity.bootIdentity === recorded.bootIdentity
1114
+ && identity.executable === recorded.executable
1115
+ && identity.hostname === recorded.hostname;
1116
+ }
1117
+ function requiredWorker(projection, workerId) {
1118
+ const worker = projection.workers[workerId];
1119
+ if (!worker)
1120
+ throw new Error(`Unknown worker ${workerId}`);
1121
+ return worker;
1122
+ }
1123
+ function formatExit(exit) {
1124
+ return exit.signal ?? `code ${exit.code ?? "unknown"}`;
1125
+ }
1126
+ function stripCarriageReturn(line) {
1127
+ return line.endsWith("\r") ? line.slice(0, -1) : line;
1128
+ }
1129
+ function isRecord(value) {
1130
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1131
+ }
1132
+ function isNodeError(error) {
1133
+ return error instanceof Error && "code" in error;
1134
+ }
1135
+ function toError(error) {
1136
+ return error instanceof Error ? error : new Error(String(error));
1137
+ }
1138
+ function protocolOutputLimit() {
1139
+ return new WorkerProtocolError("protocol_output_limit", `Worker protocol JSONL line exceeds ${WORKER_PROTOCOL_LINE_LIMIT_BYTES} bytes`, WORKER_PROTOCOL_LINE_LIMIT_BYTES);
1140
+ }
1141
+ function protocolErrorDetails(error) {
1142
+ return error instanceof WorkerProtocolError
1143
+ ? JSON.stringify({
1144
+ type: "protocol_error",
1145
+ code: error.code,
1146
+ message: error.message,
1147
+ ...(error.limitBytes === undefined ? {} : { limitBytes: error.limitBytes }),
1148
+ })
1149
+ : error.message;
1150
+ }
1151
+ function appendTail(current, chunk, limit) {
1152
+ if (chunk.byteLength >= limit)
1153
+ return chunk.slice(chunk.byteLength - limit);
1154
+ const keep = Math.min(current.byteLength, limit - chunk.byteLength);
1155
+ const result = new Uint8Array(keep + chunk.byteLength);
1156
+ result.set(current.slice(current.byteLength - keep), 0);
1157
+ result.set(chunk, keep);
1158
+ return result;
1159
+ }
1160
+ function nonEmpty(value, name) {
1161
+ if (value.length === 0)
1162
+ throw new TypeError(`${name} must not be empty`);
1163
+ return value;
1164
+ }
1165
+ function boundedProtocolText(value, name, maxLength) {
1166
+ if (value.trim().length === 0)
1167
+ throw new TypeError(`${name} must not be empty`);
1168
+ if (value.length > maxLength)
1169
+ throw new TypeError(`${name} exceeds ${maxLength} characters`);
1170
+ return value;
1171
+ }
1172
+ function correlationCanWake(worker, correlation) {
1173
+ if (worker.state === "running")
1174
+ return true;
1175
+ if (worker.state !== "blocked")
1176
+ return false;
1177
+ const active = worker.waits.filter((wait) => wait.status === "waiting");
1178
+ return active.length === 1
1179
+ && correlation !== null
1180
+ && active[0].correlation === correlation;
1181
+ }
1182
+ function assertBlockedWakeCorrelation(worker, correlation) {
1183
+ if (!correlationCanWake(worker, correlation)) {
1184
+ const expected = worker.waits.filter((wait) => wait.status === "waiting").at(-1)?.correlation;
1185
+ throw new Error(`Input correlation does not match typed wait${expected ? ` ${expected}` : ""}`);
1186
+ }
1187
+ }
1188
+ //# sourceMappingURL=worker-process-supervisor.js.map