nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,114 @@
1
+ """τ³-bench's domain tools as a stdio MCP server: the upstream's tools, the
2
+ upstream's Orchestrator executing them.
3
+
4
+ The tools it declares are the upstream's (``tau2-bench`` at the release tag
5
+ ``standard.json`` pins, installed as the driver's environment,
6
+ ``driver.install``): the domain's ``Environment`` from the upstream's
7
+ registry, each tool's schema its ``Tool.openai_schema``. It executes
8
+ nothing: each call goes to the driver (``TAU_AT``, the unit's socket on this
9
+ machine), where it becomes the agent's next message to the upstream's
10
+ ``Orchestrator``, which executes it in its own environment - counting its
11
+ steps and errors, ending the conversation by its rules - and the answer it
12
+ gives (the result, or ``Error: <what was raised>``) comes back as the
13
+ call's result. The MCP side is the official ``mcp`` SDK (its low-level
14
+ server, so the schemas go out as the upstream writes them and a call is
15
+ never refused before the upstream sees it).
16
+
17
+ What is ours, glue only: each schema's ``$ref``s written out in place
18
+ (``inline``), since not every MCP client follows a reference - the schema
19
+ says the same thing; and the pass-through, one connection to the driver per
20
+ call, a JSON line each way.
21
+
22
+ ``TAU_DOMAIN`` names the domain (``airline``, ``retail``, ...), ``TAU_RETRIEVAL`` a knowledge domain's retrieval config. It runs on the
23
+ machine in the driver's environment, never in the agent's container
24
+ (``mcp_servers`` in ``standard.json``; ``nullius.standard.mcp_machine``).
25
+ """
26
+ import contextlib
27
+ import copy
28
+ import json
29
+ import os
30
+ import socket
31
+ import sys
32
+
33
+ from search_relay import route_qwen
34
+
35
+ os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") # no download at import: the server calls no model
36
+
37
+ with contextlib.redirect_stdout(sys.stderr): # stdout carries the protocol, nothing else
38
+ from loguru import logger
39
+ from tau2.registry import registry
40
+
41
+ import anyio
42
+ import mcp.types as types
43
+ from mcp.server.lowlevel import Server
44
+ from mcp.server.stdio import stdio_server
45
+
46
+ logger.remove()
47
+ logger.add(sys.stderr, level="ERROR") # the upstream's own default (config.DEFAULT_LOG_LEVEL)
48
+
49
+
50
+ def inline(schema):
51
+ """A JSON schema with each ``#/$defs/...`` reference replaced by what it names."""
52
+ defs = schema.get("$defs") or {}
53
+
54
+ def walk(x):
55
+ if isinstance(x, dict):
56
+ ref = x.get("$ref")
57
+ if isinstance(ref, str) and ref.startswith("#/$defs/") and ref[len("#/$defs/"):] in defs:
58
+ return walk(copy.deepcopy(defs[ref[len("#/$defs/"):]]))
59
+ return {k: walk(v) for k, v in x.items() if k != "$defs"}
60
+ if isinstance(x, list):
61
+ return [walk(v) for v in x]
62
+ return x
63
+ return walk(schema)
64
+
65
+
66
+ def tools_of(env):
67
+ """The domain's tools as MCP declares them: the upstream's schemas."""
68
+ out = []
69
+ for t in env.get_tools():
70
+ f = t.openai_schema["function"]
71
+ out.append({"name": f["name"], "description": f["description"], "inputSchema": inline(f["parameters"])})
72
+ return out
73
+
74
+
75
+ def call(at, name, arguments):
76
+ """One call, through the driver: (text, error)."""
77
+ try:
78
+ with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
79
+ s.connect(at)
80
+ f = s.makefile("rwb")
81
+ f.write((json.dumps({"name": name, "arguments": arguments}) + "\n").encode())
82
+ f.flush()
83
+ got = json.loads(f.readline() or b"{}")
84
+ return str(got["text"]), bool(got["isError"])
85
+ except (OSError, ValueError, KeyError) as e:
86
+ return f"Error: the benchmark could not be reached ({e})", True
87
+
88
+
89
+ def main():
90
+ domain, at = os.environ["TAU_DOMAIN"], os.environ["TAU_AT"]
91
+ # a knowledge domain's tools follow its retrieval config (the run's retrieval option): the same search the driver's environment has
92
+ retrieval = os.environ.get("TAU_RETRIEVAL")
93
+ route_qwen(retrieval)
94
+ declared = tools_of(registry.get_env_constructor(domain)(**({"retrieval_variant": retrieval} if retrieval else {})))
95
+ server = Server(domain)
96
+
97
+ @server.list_tools()
98
+ async def list_tools():
99
+ return [types.Tool(**t) for t in declared]
100
+
101
+ @server.call_tool(validate_input=False) # the upstream answers a malformed call itself
102
+ async def call_tool(name, arguments):
103
+ text, error = await anyio.to_thread.run_sync(call, at, name, {} if arguments is None else arguments)
104
+ return types.CallToolResult(content=[types.TextContent(type="text", text=text)], isError=error)
105
+
106
+ async def serve():
107
+ async with stdio_server() as (read, write):
108
+ await server.run(read, write, server.create_initialization_options())
109
+
110
+ anyio.run(serve)
111
+
112
+
113
+ if __name__ == "__main__":
114
+ main()
@@ -0,0 +1,19 @@
1
+ """Point the pinned upstream's Qwen client at the run's search relay."""
2
+ import os
3
+
4
+
5
+ def route_qwen(retrieval):
6
+ if retrieval not in ("qwen_embeddings", "qwen_embeddings_grep", "qwen_embeddings_reranker",
7
+ "qwen_embeddings_reranker_grep", "alltools-qwen"):
8
+ return
9
+ base, key = os.environ.get("OPENAI_BASE_URL"), os.environ.get("OPENAI_API_KEY")
10
+ if not base or not key:
11
+ raise ValueError("Qwen search needs the run's relay URL and placeholder key")
12
+ from openai import OpenAI
13
+ from tau2.knowledge.embedders import openrouter_embedder
14
+
15
+ def client(**kwargs):
16
+ return OpenAI(**{**kwargs, "base_url": base, "api_key": key})
17
+
18
+ # Only this upstream module's constructor: its embedder, instructions and cache stay upstream's.
19
+ openrouter_embedder.OpenAI = client
@@ -0,0 +1,547 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "tau3-bench",
4
+ "label": "\u03c4\u00b3-bench",
5
+ "summary": "Each task is a customer's call, held by the upstream's own batch runner and orchestrator: its user simulator plays the customer from the task's scenario, the agent answers through the domain's tools (served over MCP, each call executed by the orchestrator's environment), and the upstream's evaluator scores the database the agent left and what it said.",
6
+ "family": "agent",
7
+ "needs": [
8
+ "agent.tools/v1",
9
+ "agent.conversation/v1"
10
+ ],
11
+ "upstream": {
12
+ "repository": "https://github.com/sierra-research/tau2-bench",
13
+ "tag": "v1.0.1",
14
+ "tested": "fc0055dc4e0a316c3f83133267fbd6faaa770992",
15
+ "tested_date": "2026-10-13",
16
+ "license": "MIT"
17
+ },
18
+ "driver": {
19
+ "command": [
20
+ "{install}/.venv/bin/python",
21
+ "{here}/driver.py"
22
+ ],
23
+ "install": {
24
+ "python": "3.12",
25
+ "project": ".",
26
+ "extras": [
27
+ "knowledge"
28
+ ],
29
+ "locked": true,
30
+ "pip": [
31
+ "mcp==1.30.0"
32
+ ]
33
+ }
34
+ },
35
+ "mcp_servers": {
36
+ "domain": {
37
+ "command": [
38
+ "{install}/.venv/bin/python",
39
+ "{here}/mcp_server.py"
40
+ ],
41
+ "summary": "The domain's tools: the upstream's own schemas, served over MCP by the official SDK; each call passed to the driver, where the upstream's Orchestrator executes it in its environment and answers."
42
+ }
43
+ },
44
+ "datasets": [
45
+ {
46
+ "id": "airline",
47
+ "label": "Airline",
48
+ "path": "data/tau2/domains/airline/tasks.json",
49
+ "sha256": "ccd8ba737b4cc371415af70151187788f728d6108d0916e73bb4317b40542052",
50
+ "units": 50
51
+ },
52
+ {
53
+ "id": "retail",
54
+ "label": "Retail",
55
+ "path": "data/tau2/domains/retail/tasks.json",
56
+ "sha256": "8e03ebce7901bd6218e7a7dc3105faa9324091a68058f7fe61c65262868812e8",
57
+ "units": 114
58
+ },
59
+ {
60
+ "id": "telecom",
61
+ "label": "Telecom",
62
+ "path": "data/tau2/domains/telecom/tasks.json",
63
+ "sha256": "37e562e1ae3242577407e1303b1548bc64e7ea68e37d36173e6747990ceaf8a4",
64
+ "units": 114
65
+ },
66
+ {
67
+ "id": "telecom-workflow",
68
+ "label": "Telecom (workflow policy)",
69
+ "path": "data/tau2/domains/telecom/tasks.json",
70
+ "sha256": "37e562e1ae3242577407e1303b1548bc64e7ea68e37d36173e6747990ceaf8a4",
71
+ "units": 114,
72
+ "summary": "The telecom tasks, with a tech-support workflow as the agent's policy."
73
+ },
74
+ {
75
+ "id": "banking_knowledge",
76
+ "label": "Banking knowledge",
77
+ "summary": "97 tasks. How the agent searches the bank's documents is an option, bm25 by default.",
78
+ "units": 97
79
+ }
80
+ ],
81
+ "units": {
82
+ "noun": "task",
83
+ "nouns": "tasks"
84
+ },
85
+ "simulator": {
86
+ "model": "gpt-4.1-2025-04-14",
87
+ "says": "the customer: the upstream's own user simulator, built by its build_text_orchestrator (its guidelines and the task's scenario), its default model at its default arguments (temperature 0)"
88
+ },
89
+ "leaderboard": {
90
+ "says": "the \u03c4-bench leaderboard (taubench.com), a standard submission: the upstream's docs/leaderboard-submission.md recommends gpt-5.2 as the user simulator, its submissions run it at reasoning effort low, and it asks for 4 trials per domain",
91
+ "simulator": "gpt-5.2",
92
+ "bench_options": {
93
+ "user_reasoning_effort": "low"
94
+ },
95
+ "repeat": 4
96
+ },
97
+ "controls": {
98
+ "nop": {
99
+ "source": "nullius",
100
+ "from": "Nullius: no agent runs and nothing is said; there is no conversation for the upstream's orchestrator to record, and an empty one, ended by the agent's error, is scored 0 by the upstream's evaluator."
101
+ },
102
+ "oracle": {
103
+ "source": "nullius",
104
+ "from": "Nullius, from the authors' answers: the task's gold actions (evaluation_criteria.actions), made through the upstream's environment, and a message carrying each fact in its communicate_info; scored by the upstream's evaluator as ALL_IGNORE_BASIS (database, actions, facts said), without the NL assertions: the reference has no conversation to ask them of."
105
+ }
106
+ },
107
+ "score": {
108
+ "kind": "pass",
109
+ "says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200, max errors 10, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)",
110
+ "judge": {
111
+ "model": "gpt-4.1-2025-04-14",
112
+ "says": "the upstream's NL-assertion evaluator (evaluator_nl_assertions.py, its prompt, its call and its own model name), reached through the judge's route, at temperature 0, asked only for a task whose reward_basis names NL_ASSERTION and that has assertions"
113
+ }
114
+ },
115
+ "limits": {
116
+ "turn_seconds": 600,
117
+ "turns": 100,
118
+ "attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --repeat 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it."
119
+ },
120
+ "setups": {
121
+ "original": {
122
+ "summary": "The upstream's own agent (LLMAgent at its defaults, temperature 0) as the run's model, in the upstream's batch runner exactly as tau2 run runs it: its Orchestrator, user simulator, evaluator and judge, nothing of Nullius's in the conversation - the leaderboard's standard submission, which ranks models.",
123
+ "providers": {
124
+ "openai": "",
125
+ "openrouter": "",
126
+ "custom": ""
127
+ },
128
+ "turn_seconds": 7200,
129
+ "paper": "the \u03c4\u00b2-bench paper (arXiv 2506.07982)",
130
+ "reference_agent": "\u03c4\u00b3-bench LLM agent"
131
+ }
132
+ },
133
+ "options": {
134
+ "retrieval": {
135
+ "summary": "How the agent searches the bank's documents in banking_knowledge tasks.",
136
+ "datasets": [
137
+ "banking_knowledge"
138
+ ],
139
+ "values": [
140
+ "bm25",
141
+ "bm25_grep",
142
+ "grep_only",
143
+ "no_knowledge",
144
+ "full_kb",
145
+ "golden_retrieval",
146
+ "openai_embeddings",
147
+ "openai_embeddings_grep",
148
+ "alltools",
149
+ "terminal_use",
150
+ "terminal_use_write",
151
+ "bm25_reranker",
152
+ "bm25_reranker_grep",
153
+ "openai_embeddings_reranker",
154
+ "openai_embeddings_reranker_grep",
155
+ "qwen_embeddings",
156
+ "qwen_embeddings_grep",
157
+ "qwen_embeddings_reranker",
158
+ "qwen_embeddings_reranker_grep",
159
+ "alltools-qwen"
160
+ ],
161
+ "default": "bm25",
162
+ "about": {
163
+ "bm25": "Word matching over the documents, one search tool. Offline: no key, nothing to install.",
164
+ "bm25_grep": "Word matching and a grep tool over the documents. Offline.",
165
+ "grep_only": "Only a grep tool over the documents. Offline.",
166
+ "no_knowledge": "No search and no documents: the agent has its policy only - the upstream's lower bound. Offline.",
167
+ "full_kb": "Every document written into the agent's policy, no search tool. Offline; a very long prompt.",
168
+ "golden_retrieval": "The documents the task needs written into the agent's policy, no search tool - the upstream's upper bound. Offline.",
169
+ "openai_embeddings": "Meaning search with OpenAI's text-embedding-3-large, one search tool. Its embedding calls go through the run's relay, on OpenAI or OpenRouter.",
170
+ "openai_embeddings_grep": "Meaning search with text-embedding-3-large and a grep tool. Its embedding calls go through the run's relay, on OpenAI or OpenRouter.",
171
+ "alltools": "The upstream's default and the leaderboard's: word search, meaning search and a read-only shell over the documents. Needs OpenAI or OpenRouter and the sandbox tools on the machine.",
172
+ "terminal_use": "Only a read-only shell over the documents: the agent greps and reads them itself. Needs the sandbox tools on the machine.",
173
+ "terminal_use_write": "A shell over the documents that may also write (a copy of them). Needs the sandbox tools on the machine.",
174
+ "bm25_reranker": "Word matching, reranked by the upstream's gpt-5.2. Every model call goes through the run's relay.",
175
+ "bm25_reranker_grep": "Word matching, reranked by the upstream's gpt-5.2 and a grep tool. Every model call goes through the run's relay.",
176
+ "openai_embeddings_reranker": "Meaning search with text-embedding-3-large, reranked by the upstream's gpt-5.2. Every model call goes through the run's relay.",
177
+ "openai_embeddings_reranker_grep": "Meaning search with text-embedding-3-large, reranked by the upstream's gpt-5.2 and a grep tool. Every model call goes through the run's relay.",
178
+ "qwen_embeddings": "Meaning search with Qwen3-Embedding-8B through the run's OpenRouter relay.",
179
+ "qwen_embeddings_grep": "Meaning search with Qwen3-Embedding-8B and a grep tool through the run's OpenRouter relay.",
180
+ "qwen_embeddings_reranker": "Meaning search with Qwen3-Embedding-8B, reranked by the upstream's gpt-5.2 through the run's OpenRouter relay.",
181
+ "qwen_embeddings_reranker_grep": "Meaning search with Qwen3-Embedding-8B, reranked by the upstream's gpt-5.2 and a grep tool through the run's OpenRouter relay.",
182
+ "alltools-qwen": "Word search, Qwen3-Embedding-8B meaning search and a read-only shell over the documents. Needs OpenRouter and the sandbox tools on the machine."
183
+ },
184
+ "needs": {
185
+ "openai_embeddings": {
186
+ "embeddings": "text-embedding-3-large"
187
+ },
188
+ "openai_embeddings_grep": {
189
+ "embeddings": "text-embedding-3-large"
190
+ },
191
+ "alltools": {
192
+ "embeddings": "text-embedding-3-large",
193
+ "host": [
194
+ "sandbox"
195
+ ]
196
+ },
197
+ "terminal_use": {
198
+ "host": [
199
+ "sandbox"
200
+ ]
201
+ },
202
+ "terminal_use_write": {
203
+ "host": [
204
+ "sandbox"
205
+ ]
206
+ },
207
+ "bm25_reranker": {
208
+ "reranker": "gpt-5.2"
209
+ },
210
+ "bm25_reranker_grep": {
211
+ "reranker": "gpt-5.2"
212
+ },
213
+ "openai_embeddings_reranker": {
214
+ "reranker": "gpt-5.2",
215
+ "embeddings": "text-embedding-3-large"
216
+ },
217
+ "openai_embeddings_reranker_grep": {
218
+ "reranker": "gpt-5.2",
219
+ "embeddings": "text-embedding-3-large"
220
+ },
221
+ "qwen_embeddings": {
222
+ "embeddings": "qwen/qwen3-embedding-8b"
223
+ },
224
+ "qwen_embeddings_grep": {
225
+ "embeddings": "qwen/qwen3-embedding-8b"
226
+ },
227
+ "qwen_embeddings_reranker": {
228
+ "embeddings": "qwen/qwen3-embedding-8b",
229
+ "reranker": "gpt-5.2"
230
+ },
231
+ "qwen_embeddings_reranker_grep": {
232
+ "embeddings": "qwen/qwen3-embedding-8b",
233
+ "reranker": "gpt-5.2"
234
+ },
235
+ "alltools-qwen": {
236
+ "embeddings": "qwen/qwen3-embedding-8b",
237
+ "host": [
238
+ "sandbox"
239
+ ]
240
+ }
241
+ },
242
+ "setups": [
243
+ "original",
244
+ "yours"
245
+ ]
246
+ },
247
+ "task_split": {
248
+ "summary": "Which group of tasks to run within the selected domain.",
249
+ "datasets": [
250
+ "airline",
251
+ "retail",
252
+ "telecom",
253
+ "telecom-workflow"
254
+ ],
255
+ "setups": [
256
+ "original",
257
+ "yours"
258
+ ],
259
+ "values": [
260
+ "base",
261
+ "train",
262
+ "test",
263
+ "small",
264
+ "full"
265
+ ],
266
+ "default": "base",
267
+ "about": {
268
+ "base": "The upstream's default: airline 50, retail 114, telecom 114 tasks.",
269
+ "train": "airline 30, retail 74, telecom 74.",
270
+ "test": "airline 20, retail 40, telecom 40.",
271
+ "small": "telecom only: 20 tasks.",
272
+ "full": "telecom only: all 2285 tasks of its generated set."
273
+ }
274
+ },
275
+ "user_reasoning_effort": {
276
+ "summary": "How much reasoning the simulated user's model uses, with `default` keeping the upstream settings.",
277
+ "setups": [
278
+ "original",
279
+ "yours"
280
+ ],
281
+ "values": [
282
+ "default",
283
+ "none",
284
+ "minimal",
285
+ "low",
286
+ "medium",
287
+ "high",
288
+ "xhigh"
289
+ ],
290
+ "default": "default",
291
+ "about": {
292
+ "default": "The upstream's own arguments: temperature 0, no effort sent."
293
+ }
294
+ },
295
+ "agent_reasoning_effort": {
296
+ "summary": "How much reasoning the benchmark's own agent model uses, with `default` keeping the upstream settings.",
297
+ "setups": [
298
+ "original"
299
+ ],
300
+ "values": [
301
+ "default",
302
+ "none",
303
+ "minimal",
304
+ "low",
305
+ "medium",
306
+ "high",
307
+ "xhigh"
308
+ ],
309
+ "default": "default",
310
+ "about": {
311
+ "default": "The upstream's own arguments: temperature 0, no effort sent."
312
+ }
313
+ },
314
+ "upstream_agent": {
315
+ "summary": "Whether the benchmark's own agent runs normally, receives the reference actions, or works alone with the user's tools.",
316
+ "setups": [
317
+ "original"
318
+ ],
319
+ "values": [
320
+ "llm_agent",
321
+ "llm_agent_gt",
322
+ "llm_agent_solo"
323
+ ],
324
+ "default": "llm_agent",
325
+ "about": {
326
+ "llm_agent": "The upstream's LLMAgent: its instructions, the domain's policy and tools.",
327
+ "llm_agent_gt": "LLMGTAgent: the task's expected actions in its prompt; tasks with at least one action.",
328
+ "llm_agent_solo": "LLMSoloAgent: no conversation, the ticket and both sides' tools; tasks with a ticket (telecom)."
329
+ }
330
+ }
331
+ },
332
+ "host": {
333
+ "sandbox": {
334
+ "summary": "Anthropic's sandbox-runtime, which the upstream runs the agent's read-only shell in, with the programs it needs on Linux (ripgrep, bubblewrap, socat): installed on the machine the benchmark's driver runs on, whatever the backend.",
335
+ "check": [
336
+ "srt",
337
+ "rg",
338
+ "bwrap",
339
+ "socat"
340
+ ],
341
+ "install": {
342
+ "apt": [
343
+ "ripgrep",
344
+ "bubblewrap",
345
+ "socat"
346
+ ],
347
+ "npm": [
348
+ "@anthropic-ai/sandbox-runtime@0.0.23"
349
+ ]
350
+ }
351
+ }
352
+ },
353
+ "proven": {
354
+ "airline": {
355
+ "units": {
356
+ "units": 50,
357
+ "passed": "50 ids, the same twice, unique",
358
+ "on": "2026-10-13"
359
+ },
360
+ "leak": {
361
+ "units": 50,
362
+ "looked_for": 46,
363
+ "short": 4,
364
+ "passed": "50/50",
365
+ "note": "4 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
366
+ "on": "2026-10-13"
367
+ },
368
+ "nop": {
369
+ "passed": "0/50",
370
+ "value": 0.0,
371
+ "expect": {
372
+ "at_most": 0.0
373
+ },
374
+ "units": 50,
375
+ "job": "2026-10-13__14-48-52__tau3-bench__airline__prove-nop",
376
+ "source": "nullius",
377
+ "judge": "gpt-4.1-2025-04-14",
378
+ "on": "2026-10-13"
379
+ },
380
+ "oracle": {
381
+ "passed": "50/50",
382
+ "value": 1.0,
383
+ "expect": {
384
+ "at_least": 1.0
385
+ },
386
+ "units": 50,
387
+ "job": "2026-10-13__14-48-52__tau3-bench__airline__prove-oracle",
388
+ "source": "nullius",
389
+ "judge": "gpt-4.1-2025-04-14",
390
+ "on": "2026-10-13"
391
+ }
392
+ },
393
+ "retail": {
394
+ "units": {
395
+ "units": 114,
396
+ "passed": "114 ids, the same twice, unique",
397
+ "on": "2026-10-13"
398
+ },
399
+ "leak": {
400
+ "units": 114,
401
+ "looked_for": 97,
402
+ "short": 17,
403
+ "passed": "114/114",
404
+ "note": "17 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
405
+ "on": "2026-10-13"
406
+ },
407
+ "nop": {
408
+ "passed": "0/114",
409
+ "value": 0.0,
410
+ "expect": {
411
+ "at_most": 0.0
412
+ },
413
+ "units": 114,
414
+ "job": "2026-10-13__14-48-52__tau3-bench__retail__prove-nop",
415
+ "source": "nullius",
416
+ "judge": "gpt-4.1-2025-04-14",
417
+ "on": "2026-10-13"
418
+ },
419
+ "oracle": {
420
+ "passed": "114/114",
421
+ "value": 1.0,
422
+ "expect": {
423
+ "at_least": 1.0
424
+ },
425
+ "units": 114,
426
+ "job": "2026-10-13__14-48-52__tau3-bench__retail__prove-oracle",
427
+ "source": "nullius",
428
+ "judge": "gpt-4.1-2025-04-14",
429
+ "on": "2026-10-13"
430
+ }
431
+ },
432
+ "telecom": {
433
+ "units": {
434
+ "units": 114,
435
+ "passed": "114 ids, the same twice, unique",
436
+ "on": "2026-10-14"
437
+ },
438
+ "leak": {
439
+ "units": 114,
440
+ "looked_for": 114,
441
+ "short": 0,
442
+ "passed": "114/114",
443
+ "on": "2026-10-14"
444
+ },
445
+ "nop": {
446
+ "passed": "0/114",
447
+ "value": 0.0,
448
+ "expect": {
449
+ "at_most": 0.0
450
+ },
451
+ "units": 114,
452
+ "job": "2026-10-14__09-00-42__tau3-bench__telecom__prove-nop",
453
+ "source": "nullius",
454
+ "judge": "gpt-4.1-2025-04-14",
455
+ "on": "2026-10-14"
456
+ },
457
+ "oracle": {
458
+ "passed": "114/114",
459
+ "value": 1.0,
460
+ "expect": {
461
+ "at_least": 1.0
462
+ },
463
+ "units": 114,
464
+ "job": "2026-10-14__09-00-42__tau3-bench__telecom__prove-oracle",
465
+ "source": "nullius",
466
+ "judge": "gpt-4.1-2025-04-14",
467
+ "on": "2026-10-14"
468
+ }
469
+ },
470
+ "banking_knowledge": {
471
+ "units": {
472
+ "units": 97,
473
+ "passed": "97 ids, the same twice, unique",
474
+ "on": "2026-10-14"
475
+ },
476
+ "leak": {
477
+ "units": 97,
478
+ "looked_for": 97,
479
+ "short": 0,
480
+ "passed": "97/97",
481
+ "on": "2026-10-14"
482
+ },
483
+ "nop": {
484
+ "passed": "0/97",
485
+ "value": 0.0,
486
+ "expect": {
487
+ "at_most": 0.0
488
+ },
489
+ "units": 97,
490
+ "job": "2026-10-14__09-10-48__tau3-bench__banking_knowledge__prove-nop",
491
+ "source": "nullius",
492
+ "judge": "gpt-4.1-2025-04-14",
493
+ "on": "2026-10-14"
494
+ },
495
+ "oracle": {
496
+ "passed": "97/97",
497
+ "value": 1.0,
498
+ "expect": {
499
+ "at_least": 1.0
500
+ },
501
+ "units": 97,
502
+ "job": "2026-10-14__09-10-48__tau3-bench__banking_knowledge__prove-oracle",
503
+ "source": "nullius",
504
+ "judge": "gpt-4.1-2025-04-14",
505
+ "on": "2026-10-14"
506
+ }
507
+ },
508
+ "telecom-workflow": {
509
+ "units": {
510
+ "units": 114,
511
+ "passed": "114 ids, the same twice, unique",
512
+ "on": "2026-10-16"
513
+ },
514
+ "leak": {
515
+ "units": 114,
516
+ "looked_for": 114,
517
+ "short": 0,
518
+ "passed": "114/114",
519
+ "on": "2026-10-16"
520
+ },
521
+ "nop": {
522
+ "passed": "0/114",
523
+ "value": 0.0,
524
+ "expect": {
525
+ "at_most": 0.0
526
+ },
527
+ "units": 114,
528
+ "job": "2026-10-16__15-32-40__tau3-bench__telecom-workflow__prove-nop",
529
+ "source": "nullius",
530
+ "judge": "gpt-4.1-2025-04-14",
531
+ "on": "2026-10-16"
532
+ },
533
+ "oracle": {
534
+ "passed": "114/114",
535
+ "value": 1.0,
536
+ "expect": {
537
+ "at_least": 1.0
538
+ },
539
+ "units": 114,
540
+ "job": "2026-10-16__15-32-40__tau3-bench__telecom-workflow__prove-oracle",
541
+ "source": "nullius",
542
+ "judge": "gpt-4.1-2025-04-14",
543
+ "on": "2026-10-16"
544
+ }
545
+ }
546
+ }
547
+ }