nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,608 @@
1
+ """τ³-bench's driver for the Nullius standard (JSON lines on stdin and stdout).
2
+
3
+ An agent benchmark of conversations: a customer, played by a model, calls a
4
+ service whose system the agent works through tools. Everything the
5
+ benchmark's authors decide is their code, ``tau2-bench`` at the release tag
6
+ ``standard.json`` pins, installed as this driver's environment
7
+ (``driver.install``: the upstream as a project, with its own lock file) and
8
+ imported - never copied:
9
+
10
+ units - the domain's tasks in the dataset file's order (the upstream's
11
+ ``load_file`` and ``Task`` model), those of the split the upstream
12
+ runs by default (``TextRunConfig.task_split_name``, ``base``) when
13
+ its registry has splits for the domain: the file may hold more
14
+ tasks than the upstream's task set runs.
15
+ A dataset with no file (banking_knowledge, its tasks one file each) is
16
+ the upstream's own task loader (``registry.get_tasks_loader``)
17
+ setup - the agent's instructions and the domain's policy as the upstream's
18
+ agent is given them (``LLMAgent.system_prompt``: its
19
+ ``SYSTEM_PROMPT`` and ``AGENT_INSTRUCTION`` over
20
+ ``Environment.get_policy``), the environment built for the task as the
21
+ upstream's runner builds it (``_build_env_kwargs``: a knowledge domain's
22
+ retrieval config, the run's ``retrieval`` option); the domain's tools as the MCP server
23
+ ``mcp_server.py`` (``mcp_servers`` in ``standard.json``), which
24
+ executes nothing itself: it passes each call to this driver
25
+ turn - the conversation is the upstream's: its batch runner,
26
+ ``runner.batch.run_tasks``, run on the unit's one task in a thread of
27
+ this driver, with a ``TextRunConfig`` at the upstream's defaults (max
28
+ steps, max errors, no timeout, one trial, its seed 300, three
29
+ retries, its user simulator). It draws the trial's seed from the
30
+ base seed as it does for every run: the unit's first attempt is
31
+ trial 0 of a one-trial run, its attempt N (``--repeat``) trial N-1
32
+ at the seed run_tasks draws for that trial (``trial``), and each
33
+ attempt is its ``run_single_task``: an ``Orchestrator`` built by its
34
+ ``build_text_orchestrator`` at that seed, run by its
35
+ ``run_simulation``; an attempt that raises is made again by its
36
+ ``run_with_retry``, and after the last the runner records
37
+ ``INFRASTRUCTURE_ERROR``. The agent in it is
38
+ ``AcpAgent``, an upstream ``HalfDuplexAgent`` registered with the
39
+ upstream's registry: each of the ACP agent's tool calls, arriving
40
+ from the MCP server, is the agent's next message (one ``ToolCall``),
41
+ which the Orchestrator executes in its own environment, counts and
42
+ answers - the answer goes back to the ACP agent through the server;
43
+ its final message in a turn is the agent's next message to the user.
44
+ A ``turn`` hands that message to the Orchestrator and answers with
45
+ what the upstream's user simulator says next, or ``done`` when the
46
+ Orchestrator ended the conversation - by its rules: the user's stop,
47
+ its step and error limits, an agent or user error
48
+ answer - the original: the upstream's own agent (``LLMAgent``, or the
49
+ variant the ``upstream_agent`` option names) asked as the run's
50
+ model through the unit's relay, in the same batch runner - no
51
+ ACP agent, no MCP server; the user simulator and the judge as for
52
+ an ACP agent's run
53
+ score - what ``run_tasks`` returned: the Orchestrator's own
54
+ ``SimulationRun``, evaluated by the upstream's ``evaluate_simulation``
55
+ (``ALL``) inside ``run_simulation``; a run the batch runner recorded
56
+ as ``INFRASTRUCTURE_ERROR`` is the unit's error, never a score, as
57
+ the upstream's metrics leave it out. The NL-assertion judge, when a
58
+ task asks it, is the upstream's evaluator at its own model name,
59
+ reached through the relay's judge route (litellm's own ``api_base``
60
+ and ``api_key``, which it reads before ``OPENAI_BASE_URL``: that one
61
+ is a search option's route, pinning embeddings and reranking
62
+ separately for the upstream's OpenAI clients)
63
+ oracle - the reference: the task's gold actions made through the upstream's
64
+ ``Environment.get_response``, from the task's initial state set by
65
+ its ``Environment.set_state`` as the Orchestrator sets it, then a message carrying every fact the
66
+ task requires; scored by the upstream's evaluator as
67
+ ``ALL_IGNORE_BASIS`` (every check but the NL assertions, which the
68
+ reference, having no conversation, is not asked). It and the null
69
+ control hold no conversation, so their ``SimulationRun`` is put
70
+ together here, the only one that is
71
+
72
+ What is ours, glue only: ``AcpAgent`` (a call in, a ``ToolCall`` message
73
+ out; a ``ToolMessage`` in, its text back to the call; the ACP agent's final
74
+ text in, an ``AssistantMessage`` out), the socket the MCP server reaches it
75
+ by, the routes (the simulator's as ``llm_user``/``llm_args_user``, the
76
+ judge's as litellm's environment), and the lines of the agent's first prompt
77
+ that say how its turns reach the user and where its tools are, after the
78
+ upstream's instructions. A call that arrives after the Orchestrator ended the
79
+ conversation is not executed: it is answered as an error saying so, so the
80
+ ACP agent's turn can end.
81
+
82
+ An attempt the batch runner makes again starts a new conversation: a new
83
+ Orchestrator, environment and agent. The ACP agent is told so by the next
84
+ ``turn``'s answer, ``restart`` with the user's first message, and Nullius
85
+ gives it a new session; a call or a final message of the ACP agent's that
86
+ belongs to the attempt before is not made, and answered as an error saying the
87
+ conversation was started again.
88
+ """
89
+ import contextlib
90
+ import json
91
+ import os
92
+ import queue
93
+ import random
94
+ import shutil
95
+ import socket
96
+ import sys
97
+ import tempfile
98
+ import threading
99
+ import time
100
+ import uuid
101
+
102
+ import litellm
103
+
104
+ os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") # no download at import: the relay is the only way out
105
+ PROTOCOL, sys.stdout = sys.stdout, sys.stderr # stdout is the protocol's, for good: whatever the upstream prints,
106
+ # from any thread, goes to the log
107
+ from loguru import logger
108
+ from tau2 import config
109
+ from tau2.agent.base_agent import HalfDuplexAgent
110
+ from tau2.agent.llm_agent import LLMAgent
111
+ from tau2.data_model.message import AssistantMessage, MultiToolMessage, ToolCall, ToolMessage, UserMessage
112
+ from tau2.data_model.simulation import SimulationRun, TerminationReason, TextRunConfig
113
+ from tau2.data_model.tasks import RewardType, Task
114
+ from tau2.evaluator.evaluator import EvaluationType, evaluate_simulation
115
+ from tau2.orchestrator.orchestrator import DEFAULT_FIRST_AGENT_MESSAGE
116
+ from tau2.registry import registry
117
+ from tau2.runner.batch import run_single_task, run_tasks
118
+ from tau2.runner.build import _build_env_kwargs
119
+ from tau2.runner.progress import run_with_retry
120
+ from tau2.utils import load_file
121
+
122
+ from nullius.standard.jsonl import serve
123
+ from search_relay import route_qwen
124
+
125
+ route_qwen(os.environ.get("NULLIUS_OPTION_RETRIEVAL"))
126
+
127
+ logger.remove()
128
+ logger.add(sys.stderr, level=config.DEFAULT_LOG_LEVEL)
129
+
130
+ AGENT = "nullius_acp" # the name AcpAgent is registered by, in the upstream's registry
131
+ HOW = """
132
+
133
+ How this session works: the tools of the MCP server named {domain}, given to this session, are your tools. \
134
+ Your final message in each turn is sent to the user, and their reply comes as your next message.
135
+
136
+ You said to the user: {greeting}
137
+ The user says:"""
138
+ ENDED_CALL = "Error: the conversation has ended ({why}); this call was not made."
139
+ RESTARTED_CALL = "Error: the conversation was started again; this call was not made."
140
+ WAIT = 3600 # the most a turn waits on the conversation thread: the run's own limits come first
141
+ STATE = {}
142
+ _TASKS = {}
143
+
144
+
145
+ class Aborted(BaseException):
146
+ """The unit was left before its conversation ended. Not an ``Exception``:
147
+ the upstream's run_with_retry retries those, and a unit left is not tried
148
+ again."""
149
+
150
+
151
+ class AcpAgent(HalfDuplexAgent):
152
+ """The ACP agent as the upstream's Orchestrator sees an agent: each of its
153
+ next messages is a tool call it made over MCP, or its final text in a
154
+ turn. Its stop rule is the base class's, as the upstream's LLMAgent's is."""
155
+
156
+ def __init__(self, tools, domain_policy, conversation):
157
+ super().__init__(tools=tools, domain_policy=domain_policy)
158
+ self.c = conversation
159
+ self.attempt = conversation.new_attempt() # the batch runner's attempt this agent was built for
160
+
161
+ def get_init_state(self, message_history=None):
162
+ return None
163
+
164
+ def generate_next_message(self, message, state):
165
+ c = self.c
166
+ if isinstance(message, (ToolMessage, MultiToolMessage)):
167
+ for m in (message.tool_messages if isinstance(message, MultiToolMessage) else [message]):
168
+ c.answer(m.id, m.content or "", bool(m.error))
169
+ elif isinstance(message, UserMessage):
170
+ c.out.put(("say", message.content, self.attempt))
171
+ while True:
172
+ what, value, attempt = c.inbox.get()
173
+ if what == "abort":
174
+ raise Aborted("the unit was left before the conversation ended")
175
+ if attempt == self.attempt:
176
+ break
177
+ if what == "call": # the ACP agent's, from an attempt before: not made
178
+ value[2](RESTARTED_CALL, True)
179
+ if what == "call":
180
+ name, arguments, reply = value
181
+ tc = ToolCall(id="call_" + uuid.uuid4().hex[:16], name=str(name),
182
+ arguments=arguments if isinstance(arguments, dict) else {}, requestor="assistant")
183
+ c.pending[tc.id] = reply
184
+ return AssistantMessage(role="assistant", content=None, tool_calls=[tc]), state
185
+ return AssistantMessage(role="assistant", content=value), state # "said": its final text in the turn
186
+
187
+
188
+ def _make_agent(tools=None, domain_policy=None, **kw):
189
+ return AcpAgent(tools, domain_policy, _CURRENT[0])
190
+
191
+
192
+ _CURRENT = [None] # the conversation running (one at a time: a driver holds one unit at a time)
193
+ registry.register_agent_factory(_make_agent, AGENT)
194
+
195
+
196
+ class Conversation:
197
+ """One unit's conversation: the upstream's Orchestrator in a thread, a
198
+ socket the MCP server passes calls through, and the queues between them."""
199
+
200
+ def __init__(self):
201
+ self.inbox = queue.Queue() # to AcpAgent: ("call", (name, arguments, reply), attempt), ("said", text, attempt),
202
+ # ("abort", None, None)
203
+ self.out = queue.Queue() # to a turn: ("say", text, attempt), ("done", None, None)
204
+ self.pending = {} # tool call id -> its reply
205
+ self.attempt = 0 # the batch runner's attempt running (1, 2, ...)
206
+ self.given = 0 # the attempt the ACP agent was last given a message of
207
+ self.lock = threading.Lock()
208
+ self.over = None # why the conversation ended, once it has
209
+ self.result = None # the SimulationRun run_tasks returned
210
+ self.error = None
211
+ self.config = None # the run's TextRunConfig, once started
212
+ self.thread = None
213
+ self.dir = tempfile.mkdtemp(prefix="nullius-tau-")
214
+ os.chmod(self.dir, 0o700)
215
+ self.at = os.path.join(self.dir, "s")
216
+ self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
217
+ self.sock.bind(self.at)
218
+ self.sock.listen(16)
219
+ threading.Thread(target=self._accept, daemon=True).start()
220
+
221
+ # -- the MCP server's side: one connection per call, a JSON line each way
222
+ def _accept(self):
223
+ while True:
224
+ try:
225
+ conn, _ = self.sock.accept()
226
+ except OSError:
227
+ return
228
+ threading.Thread(target=self._call, args=(conn,), daemon=True).start()
229
+
230
+ def _call(self, conn):
231
+ try:
232
+ f = conn.makefile("rwb")
233
+ line = f.readline()
234
+ req = json.loads(line or b"{}")
235
+ except (OSError, ValueError):
236
+ conn.close()
237
+ return
238
+
239
+ def reply(text, error):
240
+ try:
241
+ f.write((json.dumps({"text": text, "isError": error}) + "\n").encode())
242
+ f.flush()
243
+ except OSError:
244
+ pass
245
+ finally:
246
+ with contextlib.suppress(OSError):
247
+ conn.close()
248
+
249
+ with self.lock:
250
+ if self.over is None and self.thread is not None:
251
+ if self.given == self.attempt:
252
+ self.inbox.put(("call", (req.get("name"), req.get("arguments"), reply), self.given))
253
+ return
254
+ why = None # an attempt the ACP agent has not been told of yet
255
+ else:
256
+ why = self.over or "not started"
257
+ reply(RESTARTED_CALL if why is None else ENDED_CALL.format(why=why), True)
258
+
259
+ def new_attempt(self) -> int:
260
+ """The batch runner builds an agent: a new attempt. The calls the
261
+ attempt before left waiting are answered, never made."""
262
+ with self.lock:
263
+ self.attempt += 1
264
+ left = list(self.pending.values())
265
+ self.pending.clear()
266
+ for reply in left:
267
+ reply(RESTARTED_CALL, True)
268
+ return self.attempt
269
+
270
+ def answer(self, call_id, text, error):
271
+ reply = self.pending.pop(call_id, None)
272
+ if reply is not None:
273
+ reply(text, error)
274
+
275
+ # -- the conversation: the upstream's batch runner, on the one task, in a thread
276
+ def start(self, cfg, task, attempt=None):
277
+ self.config = cfg
278
+ self.thread = threading.Thread(target=self._run, args=(cfg, task, attempt), daemon=True)
279
+ self.thread.start()
280
+
281
+ def _run(self, cfg, task, attempt):
282
+ _CURRENT[0] = self
283
+ try:
284
+ self.result = trial(cfg, task, attempt)
285
+ except BaseException as e: # the runner raised (the unit was left): no SimulationRun
286
+ self.error = e
287
+ with self.lock:
288
+ why = self.result.termination_reason if self.result is not None else None
289
+ self.over = getattr(why, "value", None) or (str(why) if why else "stopped")
290
+ left = list(self.pending.values())
291
+ self.pending.clear()
292
+ while True:
293
+ try:
294
+ what, value, _ = self.inbox.get_nowait()
295
+ except queue.Empty:
296
+ break
297
+ if what == "call":
298
+ left.append(value[2])
299
+ for reply in left: # never made: the Orchestrator was done
300
+ reply(ENDED_CALL.format(why=self.over), True)
301
+ self.out.put(("done", None, None))
302
+
303
+ def said(self, text):
304
+ """The ACP agent's final message in its turn - for the attempt it was
305
+ last given a message of: one the batch runner has since left goes nowhere."""
306
+ with self.lock:
307
+ if self.given == self.attempt:
308
+ self.inbox.put(("said", text, self.given))
309
+
310
+ def next(self):
311
+ """What the user says next - in a new attempt, with ``restart`` - or done."""
312
+ try:
313
+ what, value, attempt = self.out.get(timeout=WAIT)
314
+ except queue.Empty:
315
+ raise TimeoutError(f"the conversation did not answer in {WAIT} s")
316
+ if what == "done":
317
+ if self.error is not None and not isinstance(self.error, Aborted):
318
+ raise RuntimeError(f"the upstream's run raised {type(self.error).__name__}: {self.error}")
319
+ return {"done": True}
320
+ with self.lock:
321
+ again, self.given = self.given not in (0, attempt), attempt
322
+ return {"message": value, "restart": True} if again else {"message": value}
323
+
324
+ def close(self):
325
+ if self.thread is not None and self.thread.is_alive():
326
+ self.inbox.put(("abort", None, None))
327
+ self.thread.join(timeout=10)
328
+ with contextlib.suppress(OSError):
329
+ self.sock.close()
330
+ shutil.rmtree(self.dir, ignore_errors=True)
331
+
332
+
333
+ def trial_seed(base, attempt):
334
+ """The seed the upstream's batch runner gives trial ``attempt - 1``: run_tasks draws one per trial
335
+ from its base seed (batch.py, "Seed management": ``random.seed(config.seed)``, then
336
+ ``random.randint(0, 1000000)`` for each trial in order) - the same draws, from a generator of
337
+ their own so nothing else's randomness moves."""
338
+ r = random.Random(base)
339
+ return [r.randint(0, 1000000) for _ in range(attempt)][-1]
340
+
341
+
342
+ def trial(cfg, t, attempt=None):
343
+ """One unit's run: trial ``attempt - 1`` of the upstream's batch runner on the unit's task. The first
344
+ attempt is run_tasks itself (a one-trial run: trial 0 at its seed), as every run was before attempts;
345
+ a later one is what run_tasks does for that trial of a run of as many (its ``_run_tracked``): its
346
+ run_single_task at the trial's seed, made again by its run_with_retry, which records
347
+ INFRASTRUCTURE_ERROR after the last retry."""
348
+ n = int(attempt or 1)
349
+ if n <= 1:
350
+ return run_tasks(cfg, [t], evaluation_type=EvaluationType.ALL).simulations[0]
351
+ seed = trial_seed(cfg.seed, n)
352
+ return run_with_retry(lambda: run_single_task(cfg, t, seed=seed, evaluation_type=EvaluationType.ALL),
353
+ task=t, trial=n - 1, seed=seed, max_retries=cfg.max_retries, retry_delay=cfg.retry_delay)
354
+
355
+
356
+ def domain():
357
+ return os.environ.get("NULLIUS_DATASET_ID") or ""
358
+
359
+
360
+ def tasks():
361
+ if not _TASKS:
362
+ if not os.environ.get("NULLIUS_DATASET"): # a dataset with no file: the upstream's own task loader at its pin
363
+ for task in registry.get_tasks_loader(domain())():
364
+ _TASKS[str(task.id)] = task
365
+ return _TASKS
366
+ splits = registry.get_task_splits_loader(domain())
367
+ if splits and split() not in splits(): # as the domain's own get_tasks refuses it
368
+ raise ValueError(f"Invalid task split name: {split()}. Valid splits are: {splits().keys()}")
369
+ keep = set(splits()[split()]) if splits else None
370
+ for t in load_file(os.environ["NULLIUS_DATASET"]):
371
+ task = Task.model_validate(t)
372
+ if keep is None or task.id in keep:
373
+ _TASKS[str(task.id)] = task
374
+ return _TASKS
375
+
376
+
377
+ def task(unit):
378
+ t = tasks().get(unit)
379
+ if t is None:
380
+ raise ValueError(f"no task {unit!r} in {domain()}")
381
+ return t
382
+
383
+
384
+ def split():
385
+ """The task split run, the upstream's --task-split-name: the run's ``task_split`` option, else the
386
+ upstream's default (``TextRunConfig.task_split_name``, ``base``)."""
387
+ return os.environ.get("NULLIUS_OPTION_TASK_SPLIT") or TextRunConfig.model_fields["task_split_name"].default
388
+
389
+
390
+ def effort_args(option):
391
+ """A model's arguments with a reasoning effort, as ``tau2 run --agent-llm-args`` / ``--user-llm-args``
392
+ '{"reasoning_effort": "low"}' passes them (the JSON replaces the default, temperature 0); None at
393
+ ``default``, the upstream's own arguments."""
394
+ v = os.environ.get("NULLIUS_OPTION_" + option.upper()) or "default"
395
+ return None if v == "default" else {"reasoning_effort": v}
396
+
397
+
398
+ def upstream_agent():
399
+ """The upstream's agent an original runs, its --agent: the run's ``upstream_agent`` option."""
400
+ return os.environ.get("NULLIUS_OPTION_UPSTREAM_AGENT") or config.DEFAULT_AGENT_IMPLEMENTATION
401
+
402
+
403
+ def retrieval():
404
+ """The search a knowledge domain's agent is given, the upstream's --retrieval-config: the run's
405
+ ``retrieval`` option (standard.json, its datasets banking_knowledge), set for that dataset only."""
406
+ return os.environ.get("NULLIUS_OPTION_RETRIEVAL") or None
407
+
408
+
409
+ def run_config(**kw):
410
+ """The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one."""
411
+ return TextRunConfig(domain=domain(), task_split_name=split(),
412
+ **({"retrieval_config": retrieval()} if retrieval() else {}), **kw)
413
+
414
+
415
+ def user_config(simulator):
416
+ """The user simulator's model and arguments: the simulator's route, at the upstream's default
417
+ arguments (temperature 0) or the run's ``user_reasoning_effort``, as --user-llm-args gives one."""
418
+ llm, at = route(simulator)
419
+ return {"llm_user": llm, "llm_args_user": dict(effort_args("user_reasoning_effort") or config.DEFAULT_LLM_ARGS_USER, **at)}
420
+
421
+
422
+ def use_judge(t, judge, st):
423
+ """The NL-assertion judge's route, as litellm's own api_base and api_key, when the task asks one."""
424
+ if asks_judge(t):
425
+ if not judge:
426
+ raise ValueError("this task's NL assertions are graded by a judge model, and none was given")
427
+ litellm.api_base, litellm.api_key = judge["base_url"], judge["key"]
428
+ st["judge"] = judge["model"]
429
+
430
+
431
+ def env_kwargs(t):
432
+ """The environment's arguments for one task, as the upstream's runner builds them (_build_env_kwargs)."""
433
+ return _build_env_kwargs(run_config(agent=AGENT), t)
434
+
435
+
436
+ def environment(t):
437
+ return registry.get_env_constructor(domain())(**env_kwargs(t))
438
+
439
+
440
+ def route(r):
441
+ """litellm's arguments for a relay route: the model as an OpenAI-compatible one at the route's address."""
442
+ return "openai/" + r["model"], {"api_base": r["base_url"], "api_key": r["key"]}
443
+
444
+
445
+ def asks_judge(t):
446
+ return bool(RewardType.NL_ASSERTION in (t.evaluation_criteria.reward_basis or []) and t.evaluation_criteria.nl_assertions)
447
+
448
+
449
+ def _state(unit):
450
+ if unit not in STATE:
451
+ raise ValueError(f"{unit} was not set up")
452
+ return STATE[unit]
453
+
454
+
455
+ def _leave():
456
+ for st in STATE.values():
457
+ if st.get("conversation") is not None:
458
+ st["conversation"].close()
459
+ STATE.clear()
460
+ _CURRENT[0] = None
461
+
462
+
463
+ def units(dataset, setup=None):
464
+ """The tasks run; an original's upstream agent may keep only some (its registered task filter, as
465
+ run_domain applies it: llm_agent_gt and llm_agent_solo take only the tasks they can run)."""
466
+ keep = registry.get_agent_task_filter(upstream_agent()) if setup == "original" else None
467
+ return {"units": [u for u, t in tasks().items() if keep is None or keep(t)]}
468
+
469
+
470
+ def setup(unit, dir, attempt=None):
471
+ t = task(unit)
472
+ _leave() # one unit at a time: an earlier one left unfinished is ended
473
+ env = environment(t)
474
+ agent = LLMAgent(tools=env.get_tools(), domain_policy=env.get_policy(), llm=config.DEFAULT_LLM_AGENT)
475
+ c = Conversation()
476
+ STATE[unit] = {"conversation": c, "dir": dir, "oracle": None}
477
+ return {"prompt": agent.system_prompt + HOW.format(domain=domain(), greeting=DEFAULT_FIRST_AGENT_MESSAGE.content),
478
+ "mcp": [{"name": domain(), "server": "domain", "env": {"TAU_DOMAIN": domain(), "TAU_AT": c.at,
479
+ **({"TAU_RETRIEVAL": retrieval()} if retrieval() else {})}}]}
480
+
481
+
482
+ def turn(unit, dir, transcript, simulator=None, judge=None, attempt=None):
483
+ """The user's next message, or done. With nothing said yet, the upstream's
484
+ Orchestrator starts: its first agent message, then the user's answer - in
485
+ trial ``attempt - 1`` of the upstream's batch runner (``trial``)."""
486
+ st, t = _state(unit), task(unit)
487
+ c = st["conversation"]
488
+ if not transcript:
489
+ if not simulator:
490
+ raise ValueError("the user is played by a model, and none was given")
491
+ use_judge(t, judge, st)
492
+ c.start(run_config(agent=AGENT, **user_config(simulator)), t, attempt)
493
+ return c.next()
494
+ if c.thread is None:
495
+ raise ValueError("the conversation was not started: the first turn has an empty transcript")
496
+ if c.over is not None:
497
+ return {"done": True}
498
+ c.said((transcript[-1].get("text") or "") if transcript[-1].get("role") == "agent" else "")
499
+ return c.next()
500
+
501
+
502
+ def answer(unit, dir, setup, model, base_url, key, simulator=None, judge=None, attempt=None):
503
+ """The original: the upstream's own agent (its --agent, the ``upstream_agent`` option: LLMAgent by
504
+ default) asked as ``model`` through the unit's relay, at the upstream's agent arguments
505
+ (temperature 0) or the run's ``agent_reasoning_effort``, as --agent-llm-args gives one; the
506
+ conversation run by the upstream's batch runner exactly as for an ACP agent (``trial``), its user
507
+ simulator and judge on their own routes. llm_agent_solo talks to no one: its user is the upstream's
508
+ dummy_user, as its docs pair them."""
509
+ if setup != "original":
510
+ raise ValueError(f"τ³-bench has no {setup} setup")
511
+ t = task(unit)
512
+ _leave()
513
+ st = STATE[unit] = {"conversation": None, "dir": dir, "oracle": None}
514
+ solo = registry.get_agent_metadata(upstream_agent(), "solo_mode", default=False)
515
+ if not solo and not simulator:
516
+ raise ValueError("the user is played by a model, and none was given: the run passes the simulator's route to answer")
517
+ use_judge(t, judge, st)
518
+ llm, at = route({"model": model, "base_url": base_url, "key": key})
519
+ cfg = run_config(agent=upstream_agent(), llm_agent=llm,
520
+ llm_args_agent=dict(effort_args("agent_reasoning_effort") or config.DEFAULT_LLM_ARGS_AGENT, **at),
521
+ **({"user": "dummy_user"} if solo else user_config(simulator)))
522
+ st.update(original=trial(cfg, t, attempt), config=cfg)
523
+ said = [m.content for m in st["original"].messages if m.role == "assistant" and m.content]
524
+ return {"answer": said[-1] if said else ""}
525
+
526
+
527
+ def score(unit, dir, transcript, answer, judge=None, setup=None):
528
+ st, t = _state(unit), task(unit)
529
+ c = st["conversation"]
530
+ if st.get("original") is not None: # the original: what its batch runner returned
531
+ run, kind = st["original"], EvaluationType.ALL
532
+ if run.termination_reason == TerminationReason.INFRASTRUCTURE_ERROR:
533
+ info = run.info or {}
534
+ _keep(dir, run)
535
+ raise ValueError(f"the upstream's batch runner recorded INFRASTRUCTURE_ERROR after {info.get('failed_after_attempts')} "
536
+ f"attempts: {info.get('error_type')}: {info.get('error')}")
537
+ elif st["oracle"] is not None:
538
+ messages = st["oracle"] + [AssistantMessage(role="assistant", content=answer)]
539
+ run = _assembled(t, messages, TerminationReason.AGENT_STOP)
540
+ kind = EvaluationType.ALL_IGNORE_BASIS
541
+ run.reward_info = evaluate_simulation(run, t, kind, solo_mode=False, domain=domain(), env_kwargs=env_kwargs(t))
542
+ elif c.thread is None: # the null control: nothing was said, an empty message
543
+ run, kind = _assembled(t, [], TerminationReason.AGENT_ERROR), EvaluationType.ALL
544
+ run.reward_info = evaluate_simulation(run, t, kind, solo_mode=False, domain=domain(), env_kwargs=env_kwargs(t))
545
+ else:
546
+ if c.over is None:
547
+ raise ValueError("the conversation has not ended: the run's turns ran out before the upstream's limits")
548
+ if c.error is not None:
549
+ raise ValueError(f"the upstream's run raised {type(c.error).__name__}: {c.error}")
550
+ run, kind = c.result, EvaluationType.ALL
551
+ if run.termination_reason == TerminationReason.INFRASTRUCTURE_ERROR:
552
+ info = run.info or {}
553
+ _keep(dir, run)
554
+ raise ValueError(f"the upstream's batch runner recorded INFRASTRUCTURE_ERROR after {info.get('failed_after_attempts')} "
555
+ f"attempts: {info.get('error_type')}: {info.get('error')}")
556
+ calls = [m for m in run.messages if isinstance(m, ToolMessage)]
557
+ detail = {"domain": domain(), "termination": getattr(run.termination_reason, "value", run.termination_reason),
558
+ "evaluation": kind.value, "tool_calls": len(calls), "tool_errors": sum(1 for m in calls if m.error),
559
+ "user_turns": sum(1 for m in run.messages if m.role == "user"),
560
+ "reward_info": run.reward_info.model_dump(mode="json", exclude_none=True)}
561
+ cfg = st.get("config") or (c.config if c is not None else None)
562
+ if cfg is not None:
563
+ detail.update(max_steps=cfg.effective_max_steps, max_errors=cfg.max_errors, seed=run.seed, trial=run.trial,
564
+ split=cfg.task_split_name, **({"attempts": c.attempt} if c is not None else {"agent": cfg.agent}))
565
+ if st.get("judge") and kind == EvaluationType.ALL:
566
+ detail["judge"] = st["judge"]
567
+ _keep(dir, run)
568
+ if c is not None:
569
+ c.close()
570
+ return {"reward": run.reward_info.reward, "passed": run.reward_info.reward == 1.0, "detail": detail}
571
+
572
+
573
+ def _assembled(t, messages, ended):
574
+ """A control's SimulationRun: it held no conversation for the Orchestrator to record."""
575
+ now = time.time()
576
+ return SimulationRun(id=str(uuid.uuid4()), task_id=t.id, start_time=str(now), end_time=str(now), duration=0.0,
577
+ termination_reason=ended, messages=messages)
578
+
579
+
580
+ def _keep(dir, run):
581
+ """The SimulationRun as the upstream writes it, beside the trial's verdict."""
582
+ out = os.path.join(os.path.dirname(os.path.abspath(dir)), "verifier")
583
+ with contextlib.suppress(OSError):
584
+ os.makedirs(out, exist_ok=True)
585
+ with open(os.path.join(out, "simulation.json"), "w") as f:
586
+ f.write(run.model_dump_json(indent=2))
587
+
588
+
589
+ def oracle(unit, dir):
590
+ """The gold actions, made through the upstream's environment, recorded as
591
+ the Orchestrator records a call and its answer."""
592
+ st, t = _state(unit), task(unit)
593
+ env = environment(t)
594
+ init = t.initial_state # the task's starting state, set as the Orchestrator and evaluator set it
595
+ if init is not None:
596
+ env.set_state(initialization_data=init.initialization_data, initialization_actions=init.initialization_actions,
597
+ message_history=[])
598
+ messages = []
599
+ for i, a in enumerate(t.evaluation_criteria.actions or []):
600
+ tc = ToolCall(id=f"oracle_{i}", name=a.name, arguments=a.arguments, requestor=a.requestor)
601
+ messages += [AssistantMessage(role="assistant", content=None, tool_calls=[tc]), env.get_response(tc)]
602
+ st["oracle"] = messages
603
+ facts = t.evaluation_criteria.communicate_info or []
604
+ return {"answer": " ".join(facts) if facts else "Your request is handled."}
605
+
606
+
607
+ if __name__ == "__main__":
608
+ serve({"units": units, "setup": setup, "turn": turn, "answer": answer, "score": score, "oracle": oracle}, out=PROTOCOL)