world-model-optimizer 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. llm_waterfall/LICENSE +21 -0
  2. llm_waterfall/__init__.py +53 -0
  3. llm_waterfall/adapters/__init__.py +36 -0
  4. llm_waterfall/adapters/anthropic.py +105 -0
  5. llm_waterfall/adapters/aws_mantle.py +47 -0
  6. llm_waterfall/adapters/azure_openai.py +71 -0
  7. llm_waterfall/adapters/base.py +51 -0
  8. llm_waterfall/adapters/bedrock.py +309 -0
  9. llm_waterfall/adapters/openai.py +130 -0
  10. llm_waterfall/classify.py +184 -0
  11. llm_waterfall/pricing.py +110 -0
  12. llm_waterfall/py.typed +0 -0
  13. llm_waterfall/types.py +295 -0
  14. llm_waterfall/waterfall.py +255 -0
  15. wmo/__init__.py +38 -0
  16. wmo/agents/__init__.py +7 -0
  17. wmo/agents/default.py +29 -0
  18. wmo/agents/meta.py +55 -0
  19. wmo/agents/optimizer.py +55 -0
  20. wmo/agents/project.py +928 -0
  21. wmo/cli/__init__.py +5 -0
  22. wmo/cli/agent_session.py +1123 -0
  23. wmo/cli/app.py +2489 -0
  24. wmo/cli/e2b_cmds.py +212 -0
  25. wmo/cli/eval_closed_loop.py +207 -0
  26. wmo/cli/harness_app.py +1147 -0
  27. wmo/cli/harness_distill.py +659 -0
  28. wmo/cli/hosted_session.py +880 -0
  29. wmo/cli/ingest_cmd.py +165 -0
  30. wmo/cli/model_roles.py +82 -0
  31. wmo/cli/platform_cmds.py +372 -0
  32. wmo/cli/route_app.py +274 -0
  33. wmo/cli/session_state.py +243 -0
  34. wmo/cli/ui.py +1107 -0
  35. wmo/cli/workspace_sync.py +504 -0
  36. wmo/config/__init__.py +60 -0
  37. wmo/config/card.py +129 -0
  38. wmo/config/config.py +367 -0
  39. wmo/config/dotenv.py +67 -0
  40. wmo/config/settings.py +128 -0
  41. wmo/config/store.py +177 -0
  42. wmo/conftest.py +19 -0
  43. wmo/connect/__init__.py +88 -0
  44. wmo/connect/apps.py +78 -0
  45. wmo/connect/brave.py +284 -0
  46. wmo/connect/connector.py +79 -0
  47. wmo/connect/credentials.py +164 -0
  48. wmo/connect/github.py +321 -0
  49. wmo/connect/google.py +627 -0
  50. wmo/connect/notion.py +790 -0
  51. wmo/connect/oauth.py +461 -0
  52. wmo/connect/slack.py +555 -0
  53. wmo/connect/store.py +199 -0
  54. wmo/connect/types.py +156 -0
  55. wmo/core/__init__.py +21 -0
  56. wmo/core/parsing.py +281 -0
  57. wmo/core/render.py +271 -0
  58. wmo/core/text.py +40 -0
  59. wmo/core/types.py +116 -0
  60. wmo/distill/__init__.py +14 -0
  61. wmo/distill/agents.py +140 -0
  62. wmo/distill/config.py +1006 -0
  63. wmo/distill/cost.py +437 -0
  64. wmo/distill/data.py +921 -0
  65. wmo/distill/deadlines.py +254 -0
  66. wmo/distill/fake_tinker.py +734 -0
  67. wmo/distill/gate.py +122 -0
  68. wmo/distill/loop.py +3499 -0
  69. wmo/distill/renderers.py +399 -0
  70. wmo/distill/rendering.py +620 -0
  71. wmo/distill/rollouts.py +726 -0
  72. wmo/distill/samples.py +195 -0
  73. wmo/distill/store.py +829 -0
  74. wmo/distill/teacher.py +714 -0
  75. wmo/distill/tokens.py +535 -0
  76. wmo/distill/tracking.py +552 -0
  77. wmo/distill/tripwire.py +411 -0
  78. wmo/distill/xtoken/byte_offsets.py +152 -0
  79. wmo/distill/xtoken/chunks.py +457 -0
  80. wmo/distill/xtoken/prompt_logprobs.py +475 -0
  81. wmo/distill/xtoken/teacher_render.py +346 -0
  82. wmo/engine/__init__.py +28 -0
  83. wmo/engine/autoconfig.py +367 -0
  84. wmo/engine/build.py +346 -0
  85. wmo/engine/demo.py +77 -0
  86. wmo/engine/eval_suites.py +245 -0
  87. wmo/engine/grounding.py +491 -0
  88. wmo/engine/knowledge.py +291 -0
  89. wmo/engine/loader.py +36 -0
  90. wmo/engine/play.py +92 -0
  91. wmo/engine/prompts.py +99 -0
  92. wmo/engine/replay.py +443 -0
  93. wmo/engine/reporting.py +58 -0
  94. wmo/engine/workspace.py +468 -0
  95. wmo/engine/world_model.py +568 -0
  96. wmo/env/__init__.py +22 -0
  97. wmo/env/base.py +121 -0
  98. wmo/env/closed_loop.py +229 -0
  99. wmo/env/episode.py +107 -0
  100. wmo/env/llm_agent.py +93 -0
  101. wmo/env/scenarios.py +73 -0
  102. wmo/evals/__init__.py +52 -0
  103. wmo/evals/agreement.py +110 -0
  104. wmo/evals/base.py +45 -0
  105. wmo/evals/closed_loop.py +480 -0
  106. wmo/evals/failover.py +96 -0
  107. wmo/evals/gold.py +127 -0
  108. wmo/evals/grid.py +394 -0
  109. wmo/evals/grid_plot.py +205 -0
  110. wmo/evals/harbor/__init__.py +27 -0
  111. wmo/evals/harbor/agent.py +573 -0
  112. wmo/evals/harbor/ctrf.py +171 -0
  113. wmo/evals/harbor/e2b_environment.py +587 -0
  114. wmo/evals/harbor/e2b_template_policy.py +144 -0
  115. wmo/evals/harbor/scorer.py +875 -0
  116. wmo/evals/harbor/tasks.py +140 -0
  117. wmo/evals/open_loop.py +194 -0
  118. wmo/evals/tasks.py +53 -0
  119. wmo/harness/__init__.py +51 -0
  120. wmo/harness/code_runtime.py +288 -0
  121. wmo/harness/create.py +1191 -0
  122. wmo/harness/delta.py +220 -0
  123. wmo/harness/doc.py +556 -0
  124. wmo/harness/e2b_ledger.py +342 -0
  125. wmo/harness/e2b_reap.py +476 -0
  126. wmo/harness/e2b_sandbox.py +350 -0
  127. wmo/harness/environment.py +35 -0
  128. wmo/harness/live_session.py +543 -0
  129. wmo/harness/mutate.py +343 -0
  130. wmo/harness/pi_e2b.py +1710 -0
  131. wmo/harness/pi_entry/entry.ts +268 -0
  132. wmo/harness/pi_entry/runner_frames.ts +92 -0
  133. wmo/harness/pi_entry/runner_live.ts +587 -0
  134. wmo/harness/pi_entry/runner_service.ts +270 -0
  135. wmo/harness/pi_entry/runner_stdio.ts +374 -0
  136. wmo/harness/pi_entry/runner_termination.ts +142 -0
  137. wmo/harness/pi_local.py +262 -0
  138. wmo/harness/pi_runtime.py +495 -0
  139. wmo/harness/pi_vendor.py +65 -0
  140. wmo/harness/population.py +509 -0
  141. wmo/harness/project_proposer.py +569 -0
  142. wmo/harness/proposer.py +977 -0
  143. wmo/harness/runner_link.py +619 -0
  144. wmo/harness/runtime.py +389 -0
  145. wmo/harness/scoring.py +247 -0
  146. wmo/harness/skills.py +116 -0
  147. wmo/harness/source_tree.py +319 -0
  148. wmo/harness/store.py +176 -0
  149. wmo/harness/tools.py +105 -0
  150. wmo/harness/vendor/manifest.sha256 +58 -0
  151. wmo/harness/vendor/pi-agent/CHANGELOG.md +556 -0
  152. wmo/harness/vendor/pi-agent/LICENSE +21 -0
  153. wmo/harness/vendor/pi-agent/README.md +488 -0
  154. wmo/harness/vendor/pi-agent/VENDOR.md +39 -0
  155. wmo/harness/vendor/pi-agent/docs/agent-harness.md +486 -0
  156. wmo/harness/vendor/pi-agent/docs/durable-harness.md +212 -0
  157. wmo/harness/vendor/pi-agent/docs/hooks.md +445 -0
  158. wmo/harness/vendor/pi-agent/docs/models.md +966 -0
  159. wmo/harness/vendor/pi-agent/docs/observability.md +376 -0
  160. wmo/harness/vendor/pi-agent/package.json +60 -0
  161. wmo/harness/vendor/pi-agent/src/agent-loop.ts +748 -0
  162. wmo/harness/vendor/pi-agent/src/agent.ts +575 -0
  163. wmo/harness/vendor/pi-agent/src/harness/agent-harness.ts +1029 -0
  164. wmo/harness/vendor/pi-agent/src/harness/compaction/branch-summarization.ts +261 -0
  165. wmo/harness/vendor/pi-agent/src/harness/compaction/compaction.ts +747 -0
  166. wmo/harness/vendor/pi-agent/src/harness/compaction/utils.ts +144 -0
  167. wmo/harness/vendor/pi-agent/src/harness/env/nodejs.ts +550 -0
  168. wmo/harness/vendor/pi-agent/src/harness/messages.ts +164 -0
  169. wmo/harness/vendor/pi-agent/src/harness/prompt-templates.ts +267 -0
  170. wmo/harness/vendor/pi-agent/src/harness/session/jsonl-repo.ts +177 -0
  171. wmo/harness/vendor/pi-agent/src/harness/session/jsonl-storage.ts +293 -0
  172. wmo/harness/vendor/pi-agent/src/harness/session/memory-repo.ts +50 -0
  173. wmo/harness/vendor/pi-agent/src/harness/session/memory-storage.ts +131 -0
  174. wmo/harness/vendor/pi-agent/src/harness/session/repo-utils.ts +51 -0
  175. wmo/harness/vendor/pi-agent/src/harness/session/session.ts +267 -0
  176. wmo/harness/vendor/pi-agent/src/harness/session/uuid.ts +54 -0
  177. wmo/harness/vendor/pi-agent/src/harness/skills.ts +375 -0
  178. wmo/harness/vendor/pi-agent/src/harness/system-prompt.ts +34 -0
  179. wmo/harness/vendor/pi-agent/src/harness/types.ts +836 -0
  180. wmo/harness/vendor/pi-agent/src/harness/utils/shell-output.ts +135 -0
  181. wmo/harness/vendor/pi-agent/src/harness/utils/truncate.ts +344 -0
  182. wmo/harness/vendor/pi-agent/src/index.ts +44 -0
  183. wmo/harness/vendor/pi-agent/src/node.ts +2 -0
  184. wmo/harness/vendor/pi-agent/src/proxy.ts +367 -0
  185. wmo/harness/vendor/pi-agent/src/types.ts +428 -0
  186. wmo/harness/vendor/pi-agent/test/agent-loop.test.ts +1351 -0
  187. wmo/harness/vendor/pi-agent/test/agent.test.ts +699 -0
  188. wmo/harness/vendor/pi-agent/test/e2e.test.ts +404 -0
  189. wmo/harness/vendor/pi-agent/test/harness/agent-harness-stream.test.ts +213 -0
  190. wmo/harness/vendor/pi-agent/test/harness/agent-harness.test.ts +608 -0
  191. wmo/harness/vendor/pi-agent/test/harness/compaction.test.ts +655 -0
  192. wmo/harness/vendor/pi-agent/test/harness/nodejs-env.test.ts +321 -0
  193. wmo/harness/vendor/pi-agent/test/harness/prompt-templates.test.ts +90 -0
  194. wmo/harness/vendor/pi-agent/test/harness/repo.test.ts +68 -0
  195. wmo/harness/vendor/pi-agent/test/harness/resource-formatting.test.ts +24 -0
  196. wmo/harness/vendor/pi-agent/test/harness/session-test-utils.ts +55 -0
  197. wmo/harness/vendor/pi-agent/test/harness/session-uuid.test.ts +50 -0
  198. wmo/harness/vendor/pi-agent/test/harness/session.test.ts +156 -0
  199. wmo/harness/vendor/pi-agent/test/harness/skills.test.ts +116 -0
  200. wmo/harness/vendor/pi-agent/test/harness/storage.test.ts +299 -0
  201. wmo/harness/vendor/pi-agent/test/harness/system-prompt.test.ts +66 -0
  202. wmo/harness/vendor/pi-agent/test/harness/truncate.test.ts +169 -0
  203. wmo/harness/vendor/pi-agent/test/scratch/simple.ts +72 -0
  204. wmo/harness/vendor/pi-agent/test/utils/calculate.ts +32 -0
  205. wmo/harness/vendor/pi-agent/test/utils/get-current-time.ts +46 -0
  206. wmo/harness/vendor/pi-agent/tsconfig.build.json +13 -0
  207. wmo/harness/vendor/pi-agent/vitest.config.ts +19 -0
  208. wmo/harness/vendor/pi-agent/vitest.harness.config.ts +28 -0
  209. wmo/harness/vendor/vendor_pi.sh +59 -0
  210. wmo/harness/workspace_patch.py +270 -0
  211. wmo/ingest/__init__.py +47 -0
  212. wmo/ingest/adapter.py +72 -0
  213. wmo/ingest/base.py +114 -0
  214. wmo/ingest/braintrust.py +339 -0
  215. wmo/ingest/detect.py +126 -0
  216. wmo/ingest/langfuse.py +291 -0
  217. wmo/ingest/langsmith.py +444 -0
  218. wmo/ingest/mastra.py +330 -0
  219. wmo/ingest/messages.py +170 -0
  220. wmo/ingest/normalize.py +679 -0
  221. wmo/ingest/otel_genai.py +69 -0
  222. wmo/ingest/otel_writer.py +100 -0
  223. wmo/ingest/phoenix.py +150 -0
  224. wmo/ingest/postgres.py +246 -0
  225. wmo/ingest/posthog.py +320 -0
  226. wmo/ingest/quality.py +28 -0
  227. wmo/ingest/stream.py +209 -0
  228. wmo/ingest/testdata/sample_otlp.json +60 -0
  229. wmo/ingest/testdata/sample_spans.jsonl +3 -0
  230. wmo/optimize/__init__.py +25 -0
  231. wmo/optimize/base.py +143 -0
  232. wmo/optimize/gepa.py +806 -0
  233. wmo/optimize/judge.py +262 -0
  234. wmo/optimize/judge_quality.py +359 -0
  235. wmo/optimize/knn.py +468 -0
  236. wmo/optimize/numeric.py +152 -0
  237. wmo/optimize/outcomes.py +103 -0
  238. wmo/optimize/policy.py +669 -0
  239. wmo/optimize/report.py +231 -0
  240. wmo/optimize/reward.py +129 -0
  241. wmo/optimize/routing.py +373 -0
  242. wmo/platform/__init__.py +6 -0
  243. wmo/platform/auth.py +115 -0
  244. wmo/platform/client.py +551 -0
  245. wmo/platform/credentials.py +126 -0
  246. wmo/platform/transfer.py +158 -0
  247. wmo/providers/__init__.py +40 -0
  248. wmo/providers/_bedrock_chat.py +155 -0
  249. wmo/providers/_openai_common.py +182 -0
  250. wmo/providers/_responses_common.py +472 -0
  251. wmo/providers/anthropic.py +134 -0
  252. wmo/providers/azure_openai.py +296 -0
  253. wmo/providers/base.py +300 -0
  254. wmo/providers/bedrock.py +312 -0
  255. wmo/providers/models.py +205 -0
  256. wmo/providers/openai.py +143 -0
  257. wmo/providers/openai_responses.py +240 -0
  258. wmo/providers/pool.py +170 -0
  259. wmo/providers/registry.py +73 -0
  260. wmo/providers/retry.py +151 -0
  261. wmo/providers/tinker.py +936 -0
  262. wmo/providers/waterfall.py +336 -0
  263. wmo/research/__init__.py +81 -0
  264. wmo/research/ablation.py +133 -0
  265. wmo/research/concurrency_plot.py +523 -0
  266. wmo/research/concurrency_run.py +240 -0
  267. wmo/research/concurrency_scaling.py +270 -0
  268. wmo/research/gepa_scaling.py +274 -0
  269. wmo/research/pipeline.py +198 -0
  270. wmo/research/scaling_split.py +82 -0
  271. wmo/research/scenario_fidelity.py +198 -0
  272. wmo/research/scenario_recovery.py +92 -0
  273. wmo/research/seed_stability.py +90 -0
  274. wmo/research/trace_scaling.py +348 -0
  275. wmo/retrieval/__init__.py +6 -0
  276. wmo/retrieval/embedders.py +105 -0
  277. wmo/retrieval/leakfree.py +52 -0
  278. wmo/retrieval/retriever.py +173 -0
  279. wmo/scenarios/__init__.py +58 -0
  280. wmo/scenarios/builder.py +152 -0
  281. wmo/scenarios/mining/__init__.py +27 -0
  282. wmo/scenarios/mining/clustering.py +171 -0
  283. wmo/scenarios/mining/facets.py +226 -0
  284. wmo/scenarios/mining/selection.py +220 -0
  285. wmo/scenarios/synthesis/__init__.py +6 -0
  286. wmo/scenarios/synthesis/scenario_set.py +63 -0
  287. wmo/scenarios/synthesis/synthesizer.py +85 -0
  288. wmo/scenarios/verification/__init__.py +17 -0
  289. wmo/scenarios/verification/judge.py +97 -0
  290. wmo/scenarios/verification/verify.py +135 -0
  291. wmo/serving/__init__.py +5 -0
  292. wmo/serving/builds.py +451 -0
  293. wmo/serving/chat.py +878 -0
  294. wmo/serving/endpoint_config.py +64 -0
  295. wmo/serving/savings.py +250 -0
  296. wmo/serving/server.py +553 -0
  297. wmo/serving/traces_source.py +206 -0
  298. wmo/telemetry.py +213 -0
  299. wmo/tracking/__init__.py +36 -0
  300. wmo/tracking/clock.py +24 -0
  301. wmo/tracking/metered.py +125 -0
  302. wmo/tracking/pricing.py +99 -0
  303. wmo/tracking/store.py +31 -0
  304. wmo/tracking/tracker.py +149 -0
  305. world_model_optimizer-0.2.0.dist-info/METADATA +203 -0
  306. world_model_optimizer-0.2.0.dist-info/RECORD +308 -0
  307. world_model_optimizer-0.2.0.dist-info/WHEEL +4 -0
  308. world_model_optimizer-0.2.0.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,270 @@
1
+ /**
2
+ * Persistent pi runner for the RunnerLink transport.
3
+ *
4
+ * Dials the control-plane host once (outbound — no inbound port, no reverse tunnel), sends `hello`,
5
+ * then runs one pi episode per `episode_start` frame the host pushes. The worker LLM and the
6
+ * environment tools never touch the network directly:
7
+ * - the worker LLM goes through a tiny in-process localhost bridge that pi's openai-completions
8
+ * transport POSTs to; the bridge frames the request as `llm_request` and renders the host's
9
+ * returned completion object back as SSE (so pi-ai's own codec does the OpenAI translation and
10
+ * no model credentials ever reach this process),
11
+ * - each env tool `execute` sends a `tool_request` frame and awaits the observation,
12
+ * - `submit` (or, failing that, pi's last assistant message) sends `done`.
13
+ *
14
+ * Run: PI_LINK_ADDR=host:port node --experimental-strip-types runner_service.ts
15
+ *
16
+ * NOTE (step 3): pi source is loaded from ./src next to this file (statically imported). Per-episode
17
+ * source materialization from episode_start.files + child-process isolation per episode is a later
18
+ * migration step; today one runner serves episodes sequentially over the one connection.
19
+ */
20
+ import fs from "node:fs";
21
+ import http from "node:http";
22
+ import net from "node:net";
23
+ import path from "node:path";
24
+ import { pathToFileURL } from "node:url";
25
+ import type { Model } from "@earendil-works/pi-ai";
26
+ import { Agent as StaticAgent } from "./src/agent.ts";
27
+ import type { AgentTool, AgentToolResult } from "./src/types.ts";
28
+ import { FrameConn, type Frame } from "./runner_frames.ts";
29
+ import {
30
+ classifyEnd,
31
+ newTurnSignal,
32
+ nudgeFor,
33
+ observeCompletion,
34
+ shouldNudge,
35
+ type DoneReason,
36
+ type TurnSignal,
37
+ } from "./runner_termination.ts";
38
+
39
+ const [HOST, PORT] = (process.env.PI_LINK_ADDR ?? "127.0.0.1:8900").split(":");
40
+ const AGENT_MODEL = process.env.PI_AGENT_MODEL ?? "worker";
41
+ const configuredMaxTurns = Number(process.env.PI_MAX_TURNS ?? "20");
42
+ const DEFAULT_MAX_TURNS =
43
+ Number.isInteger(configuredMaxTurns) && configuredMaxTurns >= 1 ? configuredMaxTurns : 20;
44
+ // Last-resort model context window when episode_start carries none. The host resolves the REAL
45
+ // served window (provider/SDK model info) and sends it as context_window; never assume a size here.
46
+ const DEFAULT_CONTEXT_WINDOW = 128000;
47
+
48
+ function assistantText(msg: any): string {
49
+ if (!msg || msg.role !== "assistant" || !Array.isArray(msg.content)) return "";
50
+ return msg.content
51
+ .filter((c: any) => c?.type === "text")
52
+ .map((c: any) => String(c.text ?? ""))
53
+ .join("")
54
+ .trim();
55
+ }
56
+
57
+ interface Bridge {
58
+ url: string;
59
+ close: () => void;
60
+ }
61
+
62
+ /** Localhost HTTP endpoint pi's openai-completions transport POSTs to; frames each request to the
63
+ * host and streams the returned completion object back as the SSE pi's parser expects. */
64
+ function startLlmBridge(conn: FrameConn, signal: TurnSignal): Promise<Bridge> {
65
+ return new Promise((resolve) => {
66
+ const server = http.createServer((req, res) => {
67
+ const chunks: Buffer[] = [];
68
+ req.on("data", (c: Buffer) => chunks.push(c));
69
+ req.on("end", async () => {
70
+ res.writeHead(200, { "Content-Type": "text/event-stream", Connection: "close" });
71
+ try {
72
+ const body = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}");
73
+ const reply = await conn.request("llm_request", { openai_body: body });
74
+ observeCompletion(signal, reply);
75
+ if (reply.error) {
76
+ res.end(`data: ${JSON.stringify({ error: { message: reply.error } })}\n\ndata: [DONE]\n\n`);
77
+ return;
78
+ }
79
+ const choice = reply.completion?.choices?.[0] ?? {};
80
+ const msg = choice.message ?? {};
81
+ const delta: any = { role: "assistant", content: msg.content ?? "" };
82
+ if (msg.reasoning_details) delta.reasoning_details = msg.reasoning_details;
83
+ if (msg.tool_calls) {
84
+ // Keep `function` explicitly nested (the streaming OpenAI shape the pi parser
85
+ // expects); index each call.
86
+ delta.tool_calls = msg.tool_calls.map((tc: any, i: number) => ({
87
+ index: i,
88
+ id: tc.id,
89
+ type: tc.type ?? "function",
90
+ function: tc.function ?? {},
91
+ }));
92
+ }
93
+ const first = { choices: [{ index: 0, delta, finish_reason: null }] };
94
+ const last = {
95
+ choices: [{ index: 0, delta: {}, finish_reason: choice.finish_reason ?? "stop" }],
96
+ // Pi uses the latest assistant usage to estimate occupied context. Without this,
97
+ // it falls back to chars/4 and can prematurely clamp the next output budget.
98
+ usage: reply.completion?.usage,
99
+ };
100
+ res.write(`data: ${JSON.stringify(first)}\n\n`);
101
+ res.write(`data: ${JSON.stringify(last)}\n\n`);
102
+ res.end("data: [DONE]\n\n");
103
+ } catch (e) {
104
+ res.end(`data: ${JSON.stringify({ error: { message: String(e) } })}\n\ndata: [DONE]\n\n`);
105
+ }
106
+ });
107
+ });
108
+ server.listen(0, "127.0.0.1", () => {
109
+ const addr = server.address() as net.AddressInfo;
110
+ resolve({ url: `http://127.0.0.1:${addr.port}/v1`, close: () => server.close() });
111
+ });
112
+ });
113
+ }
114
+
115
+ /**
116
+ * Load pi's Agent for this episode. If episode_start carries `files` (the doc's code surfaces), they
117
+ * are materialized into a fresh per-episode dir under cwd (~/pi-run, so node_modules resolves
118
+ * upward) and the Agent is dynamically imported from there — a distinct module URL per episode, so a
119
+ * searched code mutation actually takes effect. With no files, the statically-imported Agent runs
120
+ * (dev / prompt-only searches). Returns [AgentCtor, cleanup].
121
+ */
122
+ async function loadAgent(start: Frame): Promise<[any, () => void]> {
123
+ const files: Record<string, string> = start.files ?? {};
124
+ if (Object.keys(files).length === 0) return [StaticAgent, () => {}];
125
+ const base = path.join(process.cwd(), `ep-${start.episode_id}`);
126
+ for (const [rel, content] of Object.entries(files)) {
127
+ const dst = path.join(base, rel);
128
+ fs.mkdirSync(path.dirname(dst), { recursive: true });
129
+ fs.writeFileSync(dst, content);
130
+ }
131
+ const mod = await import(pathToFileURL(path.join(base, "src/agent.ts")).href);
132
+ return [mod.Agent, () => fs.rmSync(base, { recursive: true, force: true })];
133
+ }
134
+
135
+ async function runEpisode(conn: FrameConn, start: Frame): Promise<void> {
136
+ const signal = newTurnSignal();
137
+ const bridge = await startLlmBridge(conn, signal);
138
+ const [AgentCtor, cleanupSrc] = await loadAgent(start);
139
+ const episodeId = start.episode_id;
140
+ const maxTurns =
141
+ Number.isInteger(start.max_turns) && start.max_turns >= 1
142
+ ? start.max_turns
143
+ : DEFAULT_MAX_TURNS;
144
+ const maxOutputTokens =
145
+ Number.isInteger(start.max_output_tokens) && start.max_output_tokens >= 1
146
+ ? start.max_output_tokens
147
+ : 4096;
148
+ const contextWindow =
149
+ Number.isInteger(start.context_window) && start.context_window >= 1024
150
+ ? start.context_window
151
+ : DEFAULT_CONTEXT_WINDOW;
152
+ let doneSent = false;
153
+ let lastAssistantText = "";
154
+
155
+ const model: Model<"openai-completions"> = {
156
+ id: AGENT_MODEL,
157
+ name: AGENT_MODEL,
158
+ api: "openai-completions",
159
+ provider: "link", // non-builtin -> uses model.baseUrl (our localhost bridge) directly
160
+ baseUrl: bridge.url,
161
+ reasoning: false,
162
+ input: ["text"],
163
+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
164
+ contextWindow,
165
+ maxTokens: maxOutputTokens,
166
+ };
167
+
168
+ const envTools: AgentTool<any>[] = (start.tools ?? [])
169
+ .filter((t: any) => t.name !== "submit")
170
+ .map(
171
+ (t: any): AgentTool<any> => ({
172
+ name: t.name,
173
+ label: t.name,
174
+ description: t.description,
175
+ parameters: t.parameters,
176
+ execute: async (_id, params): Promise<AgentToolResult<any>> => {
177
+ const r = await conn.request("tool_request", { name: t.name, arguments: params });
178
+ return {
179
+ content: [{ type: "text", text: String(r.content ?? "") }],
180
+ details: r,
181
+ terminate: false,
182
+ };
183
+ },
184
+ }),
185
+ );
186
+ const submit: AgentTool<any> = {
187
+ name: "submit",
188
+ label: "submit",
189
+ description: "Submit the final answer and finish the task.",
190
+ parameters: { type: "object", properties: { answer: { type: "string" } }, required: ["answer"] },
191
+ execute: async (_id, params: { answer: string }): Promise<AgentToolResult<any>> => {
192
+ doneSent = true;
193
+ conn.send({
194
+ type: "done",
195
+ episode_id: episodeId,
196
+ reason: "submit",
197
+ answer: params.answer ?? "",
198
+ });
199
+ return { content: [{ type: "text", text: "submitted" }], details: {}, terminate: true };
200
+ },
201
+ };
202
+
203
+ const agent = new AgentCtor({
204
+ initialState: { systemPrompt: start.system ?? "", model, tools: [...envTools, submit] },
205
+ getApiKey: () => "x",
206
+ });
207
+ let turns = 0;
208
+ let hitTurnCap = false;
209
+ agent.subscribe((event: any) => {
210
+ if (event.type === "turn_end" || event.type === "message_end") {
211
+ const t = assistantText(event.message);
212
+ if (t) lastAssistantText = t;
213
+ }
214
+ if (event.type === "turn_end") {
215
+ turns += 1;
216
+ if (turns >= maxTurns) {
217
+ hitTurnCap = true;
218
+ agent.abort();
219
+ }
220
+ }
221
+ });
222
+
223
+ try {
224
+ // A turn without tool calls is NOT a completion: nudge (bounded) before reporting done, and
225
+ // always report WHY. See runner_termination.ts.
226
+ await agent.prompt(start.instruction);
227
+ let reason: DoneReason = classifyEnd(signal, {
228
+ hitTurnCap,
229
+ agentError: String(agent.state?.errorMessage ?? ""),
230
+ });
231
+ let consecutiveNonAction = 1;
232
+ while (!doneSent && shouldNudge(reason, consecutiveNonAction, turns, maxTurns)) {
233
+ const before = signal.toolCallTurns;
234
+ await agent.prompt(nudgeFor(reason, signal, maxOutputTokens));
235
+ consecutiveNonAction = signal.toolCallTurns > before ? 1 : consecutiveNonAction + 1;
236
+ reason = classifyEnd(signal, {
237
+ hitTurnCap,
238
+ agentError: String(agent.state?.errorMessage ?? ""),
239
+ });
240
+ }
241
+ if (!doneSent) {
242
+ conn.send({ type: "done", episode_id: episodeId, reason, answer: lastAssistantText });
243
+ }
244
+ } catch (e) {
245
+ if (!doneSent) conn.send({ type: "episode_error", episode_id: episodeId, note: String(e) });
246
+ } finally {
247
+ bridge.close();
248
+ cleanupSrc();
249
+ }
250
+ }
251
+
252
+ function main(): void {
253
+ const sock = net.connect(Number(PORT), HOST);
254
+ const conn = new FrameConn(sock);
255
+ sock.on("connect", () => {
256
+ conn.send({ type: "hello", node_version: process.version, pi_version: "0.80.3", max_concurrent: 1 });
257
+ process.stderr.write(`[runner] connected ${HOST}:${PORT}\n`);
258
+ });
259
+ conn.on("episode_start", (start) => {
260
+ runEpisode(conn, start).catch((e) => process.stderr.write(`[runner] episode fatal ${e}\n`));
261
+ });
262
+ conn.on("hello_ack", () => {});
263
+ sock.on("close", () => process.exit(0));
264
+ sock.on("error", (e: Error) => {
265
+ process.stderr.write(`[runner] socket error ${e}\n`);
266
+ process.exit(1);
267
+ });
268
+ }
269
+
270
+ main();
@@ -0,0 +1,374 @@
1
+ /**
2
+ * Stdio pi runner: the in-sandbox peer of wmo/harness/pi_e2b.py (E2BStdioChannel).
3
+ *
4
+ * Same per-episode contract as runner_service.ts — materialize episode_start.files into
5
+ * ./ep-<episode_id>/, bridge pi's LLM calls through an ephemeral localhost SSE server that frames
6
+ * them as `llm_request` (model credentials stay on the host), route env tools as `tool_request`,
7
+ * finish with `done`/`episode_error` — but the transport is this process's own stdin/stdout:
8
+ * one base64(JSON) frame per line, because the E2B command channel is a text stream (the TCP
9
+ * length-prefixed framing of runner_frames.ts does not apply here).
10
+ *
11
+ * stdout is the frame stream and NOTHING else may write to it: the first statements below rebind
12
+ * console.log/info/warn/debug to stderr before any agent code can load. The host collects stderr
13
+ * for diagnostics.
14
+ *
15
+ * Run (inside the sandbox, workdir holding node_modules + package.json {"type":"module"}):
16
+ * cd /home/user/pi-run && node --experimental-strip-types runner_stdio.ts
17
+ *
18
+ * Unlike runner_service.ts there is NO static ./src/agent.ts fallback — a fresh sandbox has no
19
+ * checkout — so an episode_start without files is answered with episode_error. Episode logic is
20
+ * deliberately a small controlled mirror of runner_service.ts (that file boots a TCP client at
21
+ * import time, so its episode helpers cannot be imported without dialing out).
22
+ */
23
+ import fs from "node:fs";
24
+ import http from "node:http";
25
+ import path from "node:path";
26
+ import util from "node:util";
27
+ import { pathToFileURL } from "node:url";
28
+ import type { AddressInfo } from "node:net";
29
+ import {
30
+ classifyEnd,
31
+ newTurnSignal,
32
+ nudgeFor,
33
+ observeCompletion,
34
+ shouldNudge,
35
+ type DoneReason,
36
+ type TurnSignal,
37
+ } from "./runner_termination.ts";
38
+
39
+ // CRITICAL — FIRST statements: every console channel that defaults to stdout is rebound to stderr
40
+ // before anything else runs (especially dynamically imported agent code), so stray prints can
41
+ // never corrupt the frame stream.
42
+ const toStderr = (...args: unknown[]): void => {
43
+ process.stderr.write(util.format(...args) + "\n");
44
+ };
45
+ console.log = toStderr;
46
+ console.info = toStderr;
47
+ console.warn = toStderr;
48
+ console.debug = toStderr;
49
+
50
+ const AGENT_MODEL = process.env.PI_AGENT_MODEL ?? "worker";
51
+ const configuredMaxTurns = Number(process.env.PI_MAX_TURNS ?? "20");
52
+ const DEFAULT_MAX_TURNS =
53
+ Number.isInteger(configuredMaxTurns) && configuredMaxTurns >= 1 ? configuredMaxTurns : 20;
54
+ // Last-resort model context window when episode_start carries none. The host resolves the REAL
55
+ // served window (provider/SDK model info) and sends it as context_window; never assume a size here.
56
+ const DEFAULT_CONTEXT_WINDOW = 128000;
57
+ const TRANSPORT_KEEPALIVE_MS = 30_000;
58
+
59
+ type Frame = Record<string, any>;
60
+
61
+ /** Encode one frame as the base64(JSON) + "\n" line pi_e2b.py's reader decodes. */
62
+ function encodeFrame(frame: Frame): string {
63
+ return Buffer.from(JSON.stringify(frame), "utf8").toString("base64") + "\n";
64
+ }
65
+
66
+ /** Decode one base64(JSON) line into a frame; null for blank or undecodable lines. */
67
+ function decodeFrame(line: string): Frame | null {
68
+ const text = line.trim();
69
+ if (!text) return null;
70
+ try {
71
+ const frame = JSON.parse(Buffer.from(text, "base64").toString("utf8"));
72
+ return frame && typeof frame === "object" && !Array.isArray(frame) ? (frame as Frame) : null;
73
+ } catch {
74
+ return null;
75
+ }
76
+ }
77
+
78
+ /**
79
+ * The stdio twin of runner_frames.FrameConn (same waiter/handler semantics, different wire):
80
+ * `request` sends a frame with a fresh req_id and resolves on the matching response frame;
81
+ * host-pushed frames (episode_start, shutdown) fire registered handlers.
82
+ */
83
+ class StdioConn {
84
+ private buf = "";
85
+ private waiters = new Map<number, (f: Frame) => void>();
86
+ private handlers = new Map<string, (f: Frame) => void>();
87
+ private reqSeq = 0;
88
+
89
+ constructor() {
90
+ process.stdin.setEncoding("utf8");
91
+ process.stdin.on("data", (chunk: string) => this.onData(chunk));
92
+ // stdin EOF = the host side is gone; nobody is left to answer llm/tool requests.
93
+ process.stdin.on("end", () => process.exit(0));
94
+ }
95
+
96
+ on(type: string, handler: (f: Frame) => void): void {
97
+ this.handlers.set(type, handler);
98
+ }
99
+
100
+ send(frame: Frame): void {
101
+ process.stdout.write(encodeFrame(frame));
102
+ }
103
+
104
+ request(type: string, payload: Frame): Promise<Frame> {
105
+ const req_id = ++this.reqSeq;
106
+ return new Promise((resolve) => {
107
+ this.waiters.set(req_id, resolve);
108
+ this.send({ type, req_id, ...payload });
109
+ });
110
+ }
111
+
112
+ /** Keep the E2B command stream and its pooled sandbox lease alive during one active episode. */
113
+ startTransportKeepalive(): () => void {
114
+ const timer = setInterval(
115
+ () => this.send({ type: "transport_keepalive" }),
116
+ TRANSPORT_KEEPALIVE_MS,
117
+ );
118
+ timer.unref();
119
+ return () => clearInterval(timer);
120
+ }
121
+
122
+ private onData(chunk: string): void {
123
+ this.buf += chunk;
124
+ let nl = this.buf.indexOf("\n");
125
+ while (nl >= 0) {
126
+ const line = this.buf.slice(0, nl);
127
+ this.buf = this.buf.slice(nl + 1);
128
+ const frame = decodeFrame(line);
129
+ if (frame) this.dispatch(frame);
130
+ nl = this.buf.indexOf("\n");
131
+ }
132
+ }
133
+
134
+ private dispatch(frame: Frame): void {
135
+ const rid = frame.req_id;
136
+ if (typeof rid === "number" && this.waiters.has(rid)) {
137
+ const resolve = this.waiters.get(rid);
138
+ this.waiters.delete(rid);
139
+ resolve?.(frame);
140
+ return;
141
+ }
142
+ const handler = this.handlers.get(frame.type);
143
+ if (handler) handler(frame);
144
+ }
145
+ }
146
+
147
+ function assistantText(msg: any): string {
148
+ if (!msg || msg.role !== "assistant" || !Array.isArray(msg.content)) return "";
149
+ return msg.content
150
+ .filter((c: any) => c?.type === "text")
151
+ .map((c: any) => String(c.text ?? ""))
152
+ .join("")
153
+ .trim();
154
+ }
155
+
156
+ interface Bridge {
157
+ url: string;
158
+ close: () => void;
159
+ }
160
+
161
+ /** Localhost HTTP endpoint pi's openai-completions transport POSTs to; frames each request to the
162
+ * host and streams the returned completion object back as the SSE pi's parser expects.
163
+ * (Mirror of runner_service.ts's bridge, over StdioConn.) */
164
+ function startLlmBridge(conn: StdioConn, signal: TurnSignal): Promise<Bridge> {
165
+ return new Promise((resolve) => {
166
+ const server = http.createServer((req, res) => {
167
+ const chunks: Buffer[] = [];
168
+ req.on("data", (c: Buffer) => chunks.push(c));
169
+ req.on("end", async () => {
170
+ res.writeHead(200, { "Content-Type": "text/event-stream", Connection: "close" });
171
+ try {
172
+ const body = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}");
173
+ const reply = await conn.request("llm_request", { openai_body: body });
174
+ observeCompletion(signal, reply);
175
+ if (reply.error) {
176
+ res.end(`data: ${JSON.stringify({ error: { message: reply.error } })}\n\ndata: [DONE]\n\n`);
177
+ return;
178
+ }
179
+ const choice = reply.completion?.choices?.[0] ?? {};
180
+ const msg = choice.message ?? {};
181
+ const delta: any = { role: "assistant", content: msg.content ?? "" };
182
+ if (msg.reasoning_details) delta.reasoning_details = msg.reasoning_details;
183
+ if (msg.tool_calls) {
184
+ // Keep `function` explicitly nested (the streaming OpenAI shape the pi parser
185
+ // expects); index each call.
186
+ delta.tool_calls = msg.tool_calls.map((tc: any, i: number) => ({
187
+ index: i,
188
+ id: tc.id,
189
+ type: tc.type ?? "function",
190
+ function: tc.function ?? {},
191
+ }));
192
+ }
193
+ const first = { choices: [{ index: 0, delta, finish_reason: null }] };
194
+ const last = {
195
+ choices: [{ index: 0, delta: {}, finish_reason: choice.finish_reason ?? "stop" }],
196
+ // Pi uses the latest assistant usage to estimate occupied context. Without this,
197
+ // it falls back to chars/4 and can prematurely clamp the next output budget.
198
+ usage: reply.completion?.usage,
199
+ };
200
+ res.write(`data: ${JSON.stringify(first)}\n\n`);
201
+ res.write(`data: ${JSON.stringify(last)}\n\n`);
202
+ res.end("data: [DONE]\n\n");
203
+ } catch (e) {
204
+ res.end(`data: ${JSON.stringify({ error: { message: String(e) } })}\n\ndata: [DONE]\n\n`);
205
+ }
206
+ });
207
+ });
208
+ server.listen(0, "127.0.0.1", () => {
209
+ const addr = server.address() as AddressInfo;
210
+ resolve({ url: `http://127.0.0.1:${addr.port}/v1`, close: () => server.close() });
211
+ });
212
+ });
213
+ }
214
+
215
+ /**
216
+ * Materialize episode_start.files (the doc's code surfaces) into a fresh per-episode dir under
217
+ * cwd (so node_modules resolves upward from the workdir) and dynamically import the Agent from
218
+ * there — a distinct module URL per episode, so a searched code mutation actually takes effect.
219
+ * Returns [AgentCtor, cleanup].
220
+ */
221
+ async function loadAgent(start: Frame): Promise<[any, () => void]> {
222
+ const files: Record<string, string> = start.files ?? {};
223
+ if (!files["src/agent.ts"]) {
224
+ throw new Error("episode_start carried no src/agent.ts (the stdio runner has no static fallback)");
225
+ }
226
+ const base = path.join(process.cwd(), `ep-${start.episode_id}`);
227
+ for (const [rel, content] of Object.entries(files)) {
228
+ const dst = path.join(base, rel);
229
+ fs.mkdirSync(path.dirname(dst), { recursive: true });
230
+ fs.writeFileSync(dst, content);
231
+ }
232
+ const mod = await import(pathToFileURL(path.join(base, "src/agent.ts")).href);
233
+ return [mod.Agent, () => fs.rmSync(base, { recursive: true, force: true })];
234
+ }
235
+
236
+ async function runEpisode(conn: StdioConn, start: Frame): Promise<void> {
237
+ const episodeId = start.episode_id;
238
+ const maxTurns =
239
+ Number.isInteger(start.max_turns) && start.max_turns >= 1
240
+ ? start.max_turns
241
+ : DEFAULT_MAX_TURNS;
242
+ const maxOutputTokens =
243
+ Number.isInteger(start.max_output_tokens) && start.max_output_tokens >= 1
244
+ ? start.max_output_tokens
245
+ : 4096;
246
+ const contextWindow =
247
+ Number.isInteger(start.context_window) && start.context_window >= 1024
248
+ ? start.context_window
249
+ : DEFAULT_CONTEXT_WINDOW;
250
+ let doneSent = false;
251
+ let lastAssistantText = "";
252
+ let bridge: Bridge | null = null;
253
+ let cleanupSrc: () => void = () => {};
254
+ const signal = newTurnSignal();
255
+ const stopTransportKeepalive = conn.startTransportKeepalive();
256
+
257
+ try {
258
+ const [AgentCtor, cleanup] = await loadAgent(start);
259
+ cleanupSrc = cleanup;
260
+ bridge = await startLlmBridge(conn, signal);
261
+
262
+ const model = {
263
+ id: AGENT_MODEL,
264
+ name: AGENT_MODEL,
265
+ api: "openai-completions",
266
+ provider: "link", // non-builtin -> uses model.baseUrl (our localhost bridge) directly
267
+ baseUrl: bridge.url,
268
+ reasoning: false,
269
+ input: ["text"],
270
+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
271
+ contextWindow,
272
+ maxTokens: maxOutputTokens,
273
+ };
274
+
275
+ const envTools: any[] = (start.tools ?? [])
276
+ .filter((t: any) => t.name !== "submit")
277
+ .map((t: any) => ({
278
+ name: t.name,
279
+ label: t.name,
280
+ description: t.description,
281
+ parameters: t.parameters,
282
+ execute: async (_id: string, params: any) => {
283
+ const r = await conn.request("tool_request", { name: t.name, arguments: params });
284
+ return {
285
+ content: [{ type: "text", text: String(r.content ?? "") }],
286
+ details: r,
287
+ terminate: false,
288
+ };
289
+ },
290
+ }));
291
+ const submit = {
292
+ name: "submit",
293
+ label: "submit",
294
+ description: "Submit the final answer and finish the task.",
295
+ parameters: { type: "object", properties: { answer: { type: "string" } }, required: ["answer"] },
296
+ execute: async (_id: string, params: { answer: string }) => {
297
+ doneSent = true;
298
+ conn.send({
299
+ type: "done",
300
+ episode_id: episodeId,
301
+ reason: "submit",
302
+ answer: params.answer ?? "",
303
+ });
304
+ return { content: [{ type: "text", text: "submitted" }], details: {}, terminate: true };
305
+ },
306
+ };
307
+
308
+ const agent = new AgentCtor({
309
+ initialState: { systemPrompt: start.system ?? "", model, tools: [...envTools, submit] },
310
+ getApiKey: () => "x",
311
+ });
312
+ let turns = 0;
313
+ let hitTurnCap = false;
314
+ agent.subscribe((event: any) => {
315
+ if (event.type === "turn_end" || event.type === "message_end") {
316
+ const t = assistantText(event.message);
317
+ if (t) lastAssistantText = t;
318
+ }
319
+ if (event.type === "turn_end") {
320
+ turns += 1;
321
+ if (turns >= maxTurns) {
322
+ hitTurnCap = true;
323
+ agent.abort();
324
+ }
325
+ }
326
+ });
327
+
328
+ // A turn without tool calls is NOT a completion. Salvage what the host could, then feed the
329
+ // reason back and ask the model to act or submit, up to MAX_NONACTION_TURNS consecutive
330
+ // times, before reporting WHY this episode really ended.
331
+ await agent.prompt(start.instruction);
332
+ let reason: DoneReason = classifyEnd(signal, {
333
+ hitTurnCap,
334
+ agentError: String(agent.state?.errorMessage ?? ""),
335
+ });
336
+ let consecutiveNonAction = 1;
337
+ while (!doneSent && shouldNudge(reason, consecutiveNonAction, turns, maxTurns)) {
338
+ const before = signal.toolCallTurns;
339
+ await agent.prompt(nudgeFor(reason, signal, maxOutputTokens));
340
+ consecutiveNonAction = signal.toolCallTurns > before ? 1 : consecutiveNonAction + 1;
341
+ reason = classifyEnd(signal, {
342
+ hitTurnCap,
343
+ agentError: String(agent.state?.errorMessage ?? ""),
344
+ });
345
+ }
346
+ if (!doneSent) {
347
+ conn.send({ type: "done", episode_id: episodeId, reason, answer: lastAssistantText });
348
+ }
349
+ } catch (e) {
350
+ if (!doneSent) conn.send({ type: "episode_error", episode_id: episodeId, note: String(e) });
351
+ } finally {
352
+ stopTransportKeepalive();
353
+ bridge?.close();
354
+ cleanupSrc();
355
+ }
356
+ }
357
+
358
+ function main(): void {
359
+ const conn = new StdioConn();
360
+ conn.on("shutdown", () => process.exit(0));
361
+ conn.on("episode_start", (start) => {
362
+ runEpisode(conn, start).catch((e) => process.stderr.write(`[runner-stdio] episode fatal ${e}\n`));
363
+ });
364
+ conn.send({
365
+ type: "hello",
366
+ node_version: process.version,
367
+ pi_version: "0.80.3",
368
+ max_concurrent: 1,
369
+ transport: "stdio",
370
+ });
371
+ process.stderr.write("[runner-stdio] ready\n");
372
+ }
373
+
374
+ main();