nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,89 @@
1
+ {
2
+ "schema": "nullius.agent/v1",
3
+ "key": "reference",
4
+ "label": "Example agent",
5
+ "example": true,
6
+ "summary": "Nullius's example of the protocol, to read or copy.",
7
+ "upstream": {
8
+ "version": "1",
9
+ "license": "same as Nullius"
10
+ },
11
+ "start": {
12
+ "acp": [
13
+ "python3",
14
+ "{here}/agent.py"
15
+ ]
16
+ },
17
+ "model": {
18
+ "apis": [
19
+ "chat"
20
+ ],
21
+ "env": {
22
+ "OPENAI_BASE_URL": "{base_url}",
23
+ "OPENAI_API_KEY": "{key}",
24
+ "REFERENCE_MODEL": "{model}"
25
+ }
26
+ },
27
+ "offers": [
28
+ "agent.task/v1",
29
+ "agent.workspace/v1",
30
+ "agent.tools/v1",
31
+ "agent.conversation/v1"
32
+ ],
33
+ "proven": {
34
+ "agent.task/v1": {
35
+ "passed": "3/3",
36
+ "on": "2026-11-02",
37
+ "model": "openai/gpt-5-mini",
38
+ "provider": "openrouter",
39
+ "version": "1",
40
+ "kit_version": 1,
41
+ "backend": "docker",
42
+ "job": "2026-11-02__14-14-35__kit__agent-task-v1__reference"
43
+ },
44
+ "agent.workspace/v1": {
45
+ "passed": "2/2",
46
+ "on": "2026-11-02",
47
+ "model": "openai/gpt-5-mini",
48
+ "provider": "openrouter",
49
+ "version": "1",
50
+ "kit_version": 1,
51
+ "backend": "docker",
52
+ "job": "2026-11-02__14-15-19__kit__agent-workspace-v1__reference"
53
+ },
54
+ "agent.tools/v1": {
55
+ "passed": "2/2",
56
+ "on": "2026-11-02",
57
+ "model": "openai/gpt-5-mini",
58
+ "provider": "openrouter",
59
+ "version": "1",
60
+ "kit_version": 2,
61
+ "backend": "docker",
62
+ "job": "2026-11-02__14-15-59__kit__agent-tools-v1__reference",
63
+ "skills": {
64
+ "decoys": {
65
+ "passed": "1/1",
66
+ "ok": true
67
+ },
68
+ "chain": {
69
+ "passed": "1/1",
70
+ "ok": true
71
+ },
72
+ "retry": {
73
+ "passed": "1/1",
74
+ "ok": true
75
+ }
76
+ }
77
+ },
78
+ "agent.conversation/v1": {
79
+ "passed": "2/2",
80
+ "on": "2026-11-02",
81
+ "model": "openai/gpt-5-mini",
82
+ "provider": "openrouter",
83
+ "version": "1",
84
+ "kit_version": 1,
85
+ "backend": "docker",
86
+ "job": "2026-11-02__14-17-27__kit__agent-conversation-v1__reference"
87
+ }
88
+ }
89
+ }
@@ -0,0 +1,209 @@
1
+ """The reference agent: the smallest complete agent in the Nullius standard.
2
+
3
+ Python's standard library only, so it is also the one to copy. It speaks ACP
4
+ on stdin and stdout, keeps one conversation per session, and works in a loop:
5
+ ask the model (OpenAI Chat Completions at $OPENAI_BASE_URL), run the tools it
6
+ asks for, give it their results, until it answers without a tool. Its tools
7
+ are four of its own, working in the session's directory - read_file,
8
+ write_file, list_files, run - and every tool of the MCP servers the session
9
+ was given, named ``<server>__<tool>``.
10
+
11
+ Every step is told to the client as it happens: the model's text as
12
+ ``agent_message_chunk``, each tool call as ``tool_call`` then
13
+ ``tool_call_update`` with its result.
14
+ """
15
+ import json
16
+ import os
17
+ import subprocess
18
+ import sys
19
+ import threading
20
+ import urllib.error
21
+ import urllib.request
22
+
23
+ MAX_STEPS = int(os.environ.get("REFERENCE_MAX_STEPS") or 30)
24
+ OUT_LIMIT = 8000
25
+ SYSTEM = ("You are a careful software agent working in the directory {cwd}. Use the tools to inspect and change files "
26
+ "and to run commands there. When the task is done, reply with a short final message and no tool call. "
27
+ "If the task asks for a specific reply, give exactly that reply as your final message.")
28
+ OWN_TOOLS = [
29
+ {"type": "function", "function": {"name": "read_file", "description": "Read a text file in the working directory.",
30
+ "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}},
31
+ {"type": "function", "function": {"name": "write_file", "description": "Write a whole text file in the working directory.",
32
+ "parameters": {"type": "object", "properties": {"path": {"type": "string"}, "content": {"type": "string"}},
33
+ "required": ["path", "content"]}}},
34
+ {"type": "function", "function": {"name": "list_files", "description": "List the files under a folder of the working directory.",
35
+ "parameters": {"type": "object", "properties": {"path": {"type": "string"}}}}},
36
+ {"type": "function", "function": {"name": "run", "description": "Run a shell command in the working directory (60 s at most).",
37
+ "parameters": {"type": "object", "properties": {"command": {"type": "string"}}, "required": ["command"]}}},
38
+ ]
39
+
40
+ lock = threading.Lock()
41
+ sessions = {}
42
+
43
+
44
+ def send(msg):
45
+ with lock:
46
+ sys.stdout.write(json.dumps(msg) + "\n")
47
+ sys.stdout.flush()
48
+
49
+
50
+ def update(sid, u):
51
+ send({"jsonrpc": "2.0", "method": "session/update", "params": {"sessionId": sid, "update": u}})
52
+
53
+
54
+ class Mcp:
55
+ """A stdio MCP server the session was given: started once, asked for its
56
+ tools, called by name."""
57
+
58
+ def __init__(self, spec, cwd):
59
+ env = dict(os.environ, **{e["name"]: e["value"] for e in spec.get("env") or []})
60
+ self.name = spec["name"]
61
+ self.p = subprocess.Popen([spec["command"]] + list(spec.get("args") or []), cwd=cwd, env=env, stdin=subprocess.PIPE,
62
+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)
63
+ self.n = 0
64
+ self.ask("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "reference", "version": "1"}})
65
+ self.p.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
66
+ self.p.stdin.flush()
67
+ self.tools = self.ask("tools/list", {}).get("tools") or []
68
+
69
+ def ask(self, method, params):
70
+ self.n += 1
71
+ self.p.stdin.write(json.dumps({"jsonrpc": "2.0", "id": self.n, "method": method, "params": params}) + "\n")
72
+ self.p.stdin.flush()
73
+ for line in self.p.stdout:
74
+ m = json.loads(line)
75
+ if m.get("id") == self.n:
76
+ if "error" in m:
77
+ raise RuntimeError(m["error"].get("message"))
78
+ return m.get("result") or {}
79
+ raise RuntimeError(f"MCP server {self.name} ended")
80
+
81
+ def call(self, tool, args):
82
+ r = self.ask("tools/call", {"name": tool, "arguments": args})
83
+ text = "\n".join(c.get("text") or json.dumps(c) for c in r.get("content") or [])
84
+ return ("Error: " if r.get("isError") else "") + text
85
+
86
+
87
+ def inside(cwd, path):
88
+ p = os.path.realpath(os.path.join(cwd, path or "."))
89
+ if p != os.path.realpath(cwd) and not p.startswith(os.path.realpath(cwd) + os.sep):
90
+ raise ValueError(f"{path} is outside the working directory")
91
+ return p
92
+
93
+
94
+ def run_tool(s, name, args):
95
+ cwd = s["cwd"]
96
+ if name == "read_file":
97
+ with open(inside(cwd, args["path"]), errors="replace") as f:
98
+ return f.read()[:OUT_LIMIT]
99
+ if name == "write_file":
100
+ p = inside(cwd, args["path"])
101
+ os.makedirs(os.path.dirname(p), exist_ok=True)
102
+ with open(p, "w") as f:
103
+ f.write(args.get("content") or "")
104
+ return f"wrote {len(args.get('content') or '')} characters to {args['path']}"
105
+ if name == "list_files":
106
+ base = inside(cwd, args.get("path") or ".")
107
+ out = []
108
+ for root, dirs, files in os.walk(base):
109
+ dirs[:] = [d for d in dirs if not d.startswith(".")]
110
+ out += [os.path.relpath(os.path.join(root, f), cwd) for f in files]
111
+ return "\n".join(sorted(out)[:500]) or "(empty)"
112
+ if name == "run":
113
+ try:
114
+ r = subprocess.run(args["command"], shell=True, cwd=cwd, capture_output=True, text=True, timeout=60)
115
+ out = (r.stdout + r.stderr)[-OUT_LIMIT:]
116
+ return f"exit {r.returncode}\n{out}"
117
+ except subprocess.TimeoutExpired:
118
+ return "Error: the command did not end in 60 s"
119
+ server, _, tool = name.partition("__")
120
+ for m in s["mcp"]:
121
+ if m.name == server:
122
+ return m.call(tool, args)
123
+ raise ValueError(f"no tool {name}")
124
+
125
+
126
+ def model(messages, tools):
127
+ body = {"model": os.environ.get("REFERENCE_MODEL") or "default", "messages": messages, "tools": tools}
128
+ req = urllib.request.Request(os.environ["OPENAI_BASE_URL"].rstrip("/") + "/chat/completions", data=json.dumps(body).encode(),
129
+ headers={"Content-Type": "application/json", "Authorization": "Bearer " + os.environ.get("OPENAI_API_KEY", "")})
130
+ with urllib.request.urlopen(req, timeout=600) as r:
131
+ return json.loads(r.read())["choices"][0]["message"]
132
+
133
+
134
+ def turn(rid, sid, text):
135
+ s = sessions[sid]
136
+ s["messages"].append({"role": "user", "content": text})
137
+ tools = OWN_TOOLS + [{"type": "function", "function": {"name": f"{m.name}__{t['name']}", "description": t.get("description") or "",
138
+ "parameters": t.get("inputSchema") or {"type": "object", "properties": {}}}}
139
+ for m in s["mcp"] for t in m.tools]
140
+ try:
141
+ for step in range(MAX_STEPS):
142
+ if s["cancelled"]:
143
+ send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "cancelled"}})
144
+ return
145
+ msg = model(s["messages"], tools)
146
+ calls = msg.get("tool_calls") or []
147
+ s["messages"].append({k: v for k, v in msg.items() if k in ("role", "content", "tool_calls")})
148
+ if msg.get("content"):
149
+ update(sid, {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": msg["content"]}})
150
+ if not calls:
151
+ send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "end_turn"}})
152
+ return
153
+ for c in calls:
154
+ name = c["function"]["name"]
155
+ try:
156
+ args = json.loads(c["function"].get("arguments") or "{}")
157
+ except ValueError:
158
+ args = {}
159
+ kind = {"read_file": "read", "write_file": "edit", "list_files": "search", "run": "execute"}.get(name, "other")
160
+ update(sid, {"sessionUpdate": "tool_call", "toolCallId": c["id"], "title": name, "kind": kind,
161
+ "status": "in_progress", "rawInput": args})
162
+ try:
163
+ out, status = run_tool(s, name, args), "completed"
164
+ except Exception as e:
165
+ out, status = f"Error: {e}", "failed"
166
+ update(sid, {"sessionUpdate": "tool_call_update", "toolCallId": c["id"], "status": status,
167
+ "content": [{"type": "content", "content": {"type": "text", "text": out}}]})
168
+ s["messages"].append({"role": "tool", "tool_call_id": c["id"], "content": out})
169
+ send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "max_turn_requests"}})
170
+ except urllib.error.HTTPError as e:
171
+ send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": f"model: HTTP {e.code}: {e.read()[:300].decode('utf-8', 'replace')}"}})
172
+ except Exception as e:
173
+ send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": f"{type(e).__name__}: {e}"}})
174
+
175
+
176
+ def main():
177
+ for line in sys.stdin:
178
+ try:
179
+ m = json.loads(line)
180
+ except ValueError:
181
+ continue
182
+ method, params, rid = m.get("method"), m.get("params") or {}, m.get("id")
183
+ if method == "initialize":
184
+ send({"jsonrpc": "2.0", "id": rid, "result": {"protocolVersion": 1, "agentCapabilities": {
185
+ "loadSession": False, "promptCapabilities": {"image": False, "audio": False, "embeddedContext": False}},
186
+ "agentInfo": {"name": "reference", "version": "1"}}})
187
+ elif method == "session/new":
188
+ sid = f"ref-{len(sessions) + 1}"
189
+ try:
190
+ servers = [Mcp(x, params["cwd"]) for x in params.get("mcpServers") or []]
191
+ except Exception as e:
192
+ send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": f"MCP server: {e}"}})
193
+ continue
194
+ sessions[sid] = {"cwd": params["cwd"], "mcp": servers, "cancelled": False,
195
+ "messages": [{"role": "system", "content": SYSTEM.format(cwd=params["cwd"])}]}
196
+ send({"jsonrpc": "2.0", "id": rid, "result": {"sessionId": sid}})
197
+ elif method == "session/prompt":
198
+ text = "\n".join(c.get("text") or "" for c in params.get("prompt") or [] if c.get("type") == "text")
199
+ sessions[params["sessionId"]]["cancelled"] = False
200
+ threading.Thread(target=turn, args=(rid, params["sessionId"], text), daemon=True).start()
201
+ elif method == "session/cancel":
202
+ if params.get("sessionId") in sessions:
203
+ sessions[params["sessionId"]]["cancelled"] = True
204
+ elif rid is not None and method:
205
+ send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": f"{method} is not supported"}})
206
+
207
+
208
+ if __name__ == "__main__":
209
+ main()
@@ -0,0 +1,64 @@
1
+ {
2
+ "schema": "nullius.agent/v1",
3
+ "key": "terminus-2",
4
+ "label": "Terminus 2",
5
+ "summary": "Harbor's terminal agent, used by Terminal-Bench.",
6
+ "upstream": {
7
+ "package": "harbor",
8
+ "version": "0.24.0",
9
+ "license": "Apache-2.0"
10
+ },
11
+ "install": {
12
+ "python": "3.12",
13
+ "pip": [
14
+ "harbor==0.24.0"
15
+ ]
16
+ },
17
+ "start": {
18
+ "command": [
19
+ "{install}/.venv/bin/python",
20
+ "{here}/run.py",
21
+ "{prompt}"
22
+ ]
23
+ },
24
+ "model": {
25
+ "apis": [
26
+ "chat"
27
+ ],
28
+ "env": {
29
+ "OPENAI_API_BASE": "{base_url}",
30
+ "OPENAI_API_KEY": "{key}",
31
+ "NULLIUS_MODEL": "{model}",
32
+ "NULLIUS_REASONING_EFFORT": "{opt_reasoning_effort}"
33
+ }
34
+ },
35
+ "options": {
36
+ "reasoning_effort": {
37
+ "summary": "How much reasoning the model uses before answering, with `default` leaving the choice to the provider.",
38
+ "values": [
39
+ "default",
40
+ "minimal",
41
+ "low",
42
+ "medium",
43
+ "high"
44
+ ],
45
+ "default": "default"
46
+ }
47
+ },
48
+ "only_for": ["terminal-bench"],
49
+ "offers": [
50
+ "agent.workspace/v1"
51
+ ],
52
+ "proven": {
53
+ "agent.workspace/v1": {
54
+ "passed": "2/2",
55
+ "on": "2026-11-02",
56
+ "model": "openai/gpt-5-mini",
57
+ "provider": "openrouter",
58
+ "version": "0.24.0",
59
+ "kit_version": 1,
60
+ "backend": "docker",
61
+ "job": "2026-11-02__14-57-08__kit__agent-workspace-v1__terminus-2"
62
+ }
63
+ }
64
+ }
@@ -0,0 +1,139 @@
1
+ """Terminus 2, Harbor's own terminal agent, as a command agent of the Nullius standard.
2
+
3
+ Harbor runs Terminus 2 against a BaseEnvironment: a container it reaches by
4
+ ``exec``, ``upload_file`` and ``download_file``. Here the environment is the
5
+ machine this runs on - the unit's workspace on the local backend, or the
6
+ inside of the task's own container when the standard runs the unit in one -
7
+ so ``Here`` is that interface over the local shell, and nothing else. The
8
+ agent itself is Harbor's, untouched: its prompt, its parser, its tmux
9
+ session, its summarisation, its LiteLLM client.
10
+
11
+ run.py "<the task>"
12
+
13
+ Terminus works in the terminal and says nothing back: the answer is what it
14
+ left in the workspace, so nothing is printed. Its own trajectory.json and
15
+ pane log land in ``agent/terminus-2/`` beside the unit's workspace, the
16
+ rest of what it says in the agent's log (stderr).
17
+
18
+ The model, its endpoint and key come from the environment the definition
19
+ sets (NULLIUS_MODEL, OPENAI_API_BASE, OPENAI_API_KEY): the relay's; the
20
+ reasoning effort from NULLIUS_REASONING_EFFORT (the option reasoning_effort).
21
+ """
22
+ import asyncio
23
+ import logging
24
+ import os
25
+ import shutil
26
+ import subprocess
27
+ import sys
28
+ import tempfile
29
+ from pathlib import Path
30
+
31
+ from harbor.agents.terminus_2.terminus_2 import Terminus2
32
+ from harbor.environments.base import BaseEnvironment, ExecResult
33
+ from harbor.models.agent.context import AgentContext
34
+ from harbor.models.task.config import EnvironmentConfig
35
+ from harbor.models.trial.paths import EnvironmentPaths, TrialPaths
36
+
37
+ SESSION = "terminus-2"
38
+
39
+
40
+ class Here(BaseEnvironment):
41
+ """This machine, as the environment Terminus works in.
42
+
43
+ Harbor mounts the trial's agent folder at ``/logs/agent`` in its
44
+ containers; Terminus pipes its pane there. Here that one path is spelled
45
+ as ``logs``, the agent's own folder. tmux gets a server of its own
46
+ (``TMUX_TMPDIR``), so two units on one machine never share a session.
47
+ ``user`` is ignored: Terminus asks root only to install tmux, and runs as
48
+ whoever runs this otherwise - the same here."""
49
+
50
+ def __init__(self, workspace: str, logs: str, tmux_dir: str):
51
+ self.workspace = workspace
52
+ self.logs = logs
53
+ self.tmux_dir = tmux_dir
54
+ super().__init__(environment_dir=Path(workspace), environment_name="here", session_id=SESSION,
55
+ trial_paths=TrialPaths(trial_dir=Path(logs)), task_env_config=EnvironmentConfig())
56
+
57
+ @staticmethod
58
+ def type() -> str:
59
+ return "here"
60
+
61
+ def _validate_definition(self):
62
+ pass
63
+
64
+ async def start(self, force_build: bool) -> None:
65
+ pass
66
+
67
+ async def stop(self, delete: bool) -> None:
68
+ pass
69
+
70
+ def _spell(self, path: str) -> str:
71
+ agent_dir = str(EnvironmentPaths.agent_dir)
72
+ return self.logs + path[len(agent_dir):] if path == agent_dir or path.startswith(agent_dir + "/") else path
73
+
74
+ async def exec(self, command: str, cwd=None, env=None, timeout_sec=None, user=None) -> ExecResult:
75
+ command = command.replace(str(EnvironmentPaths.agent_dir) + "/", self.logs + "/")
76
+ full = dict(os.environ, TMUX_TMPDIR=self.tmux_dir, **(env or {}))
77
+ proc = await asyncio.create_subprocess_exec("bash", "-c", command, cwd=cwd or self.workspace, env=full,
78
+ stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
79
+ try:
80
+ out, err = await asyncio.wait_for(proc.communicate(), timeout_sec)
81
+ except asyncio.TimeoutError:
82
+ proc.kill()
83
+ out, err = await proc.communicate()
84
+ return ExecResult(stdout=out.decode("utf-8", "replace"), stderr="timed out", return_code=124)
85
+ return ExecResult(stdout=out.decode("utf-8", "replace"), stderr=err.decode("utf-8", "replace"), return_code=proc.returncode)
86
+
87
+ async def upload_file(self, source_path, target_path: str):
88
+ shutil.copy(str(source_path), self._spell(target_path))
89
+
90
+ async def upload_dir(self, source_dir, target_dir: str):
91
+ shutil.copytree(str(source_dir), self._spell(target_dir), dirs_exist_ok=True)
92
+
93
+ async def download_file(self, source_path: str, target_path):
94
+ shutil.copy(self._spell(source_path), str(target_path))
95
+
96
+ async def download_dir(self, source_dir: str, target_dir):
97
+ shutil.copytree(self._spell(source_dir), str(target_dir), dirs_exist_ok=True)
98
+
99
+
100
+ def logs_dir(workspace: str) -> str:
101
+ """Where Terminus writes its own files: ``agent/terminus-2/`` beside the
102
+ workspace when the unit has an agent folder there, else a folder of its own."""
103
+ beside = os.path.join(os.path.dirname(workspace), "agent")
104
+ where = os.path.join(beside, SESSION) if os.path.isdir(beside) else tempfile.mkdtemp(prefix="terminus-2-")
105
+ os.makedirs(where, exist_ok=True)
106
+ return where
107
+
108
+
109
+ async def main(prompt: str) -> int:
110
+ workspace = os.getcwd()
111
+ logs = logs_dir(workspace)
112
+ tmux_dir = tempfile.mkdtemp(prefix="tmux-")
113
+ effort = os.environ.get("NULLIUS_REASONING_EFFORT", "default")
114
+ agent = Terminus2(logs_dir=Path(logs), model_name="openai/" + os.environ["NULLIUS_MODEL"],
115
+ api_base=os.environ["OPENAI_API_BASE"], record_terminal_session=False,
116
+ reasoning_effort=None if effort in ("", "default") else effort)
117
+ env = Here(workspace, logs, tmux_dir)
118
+ rc = 0
119
+ try:
120
+ await agent.setup(env)
121
+ await agent.run(prompt, env, AgentContext())
122
+ except Exception as e:
123
+ logging.getLogger(SESSION).error("terminus-2: %s: %s", type(e).__name__, e)
124
+ rc = 1
125
+ finally:
126
+ await env.exec("tmux kill-server", timeout_sec=30)
127
+ shutil.rmtree(tmux_dir, ignore_errors=True)
128
+ return rc
129
+
130
+
131
+ if __name__ == "__main__":
132
+ # stdout is the answer and Terminus has none: everything it and litellm print goes to the log
133
+ answer = os.dup(1)
134
+ os.dup2(2, 1)
135
+ sys.stdout = sys.stderr
136
+ logging.basicConfig(stream=sys.stderr, level=logging.INFO)
137
+ code = asyncio.run(main(sys.argv[1]))
138
+ os.dup2(answer, 1)
139
+ sys.exit(code)
@@ -0,0 +1,72 @@
1
+ {
2
+ "key": "gaia",
3
+ "title": "GAIA",
4
+ "category": "General assistance",
5
+ "domain": "Real-world questions needing reasoning, web research and file understanding.",
6
+ "scoring": "Answer accuracy",
7
+ "harness": [
8
+ "standard"
9
+ ],
10
+ "origin": "official",
11
+ "good_to_know": [
12
+ "Agents need public internet access.",
13
+ "Accept GAIA's access conditions on Hugging Face, then add a token in Settings → Data providers. Do not redistribute its questions or answers in a crawlable form.",
14
+ "Levels 1, 2 and 3 (53, 86 and 26 questions) split the 165 validation questions.",
15
+ "The authors' scorer gets the text after the last FINAL ANSWER: marker, or the whole reply without one.",
16
+ "Original is the paper's no-tools API baseline; choose an agent for files and web research. The paper ran it 3 times: 3 attempts per question."
17
+ ],
18
+ "upstream_limitations": [
19
+ "The 301 test questions have private answers and are excluded from local scoring.",
20
+ "GAIA uses the live web; some evidence may have changed or become unavailable."
21
+ ],
22
+ "limitations": [],
23
+ "datasets": [
24
+ {
25
+ "key": "gaia-validation",
26
+ "title": "Validation · All levels",
27
+ "tasks": 165,
28
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
29
+ "source": "Hugging Face",
30
+ "registry": "",
31
+ "hub": "",
32
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
33
+ "notes": "Official 2023 validation split."
34
+ },
35
+ {
36
+ "key": "gaia-validation-level1",
37
+ "title": "Validation · Level 1",
38
+ "tasks": 53,
39
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
40
+ "source": "Hugging Face",
41
+ "registry": "",
42
+ "hub": "",
43
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
44
+ "notes": "Official 2023 validation split."
45
+ },
46
+ {
47
+ "key": "gaia-validation-level2",
48
+ "title": "Validation · Level 2",
49
+ "tasks": 86,
50
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
51
+ "source": "Hugging Face",
52
+ "registry": "",
53
+ "hub": "",
54
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
55
+ "notes": "Official 2023 validation split."
56
+ },
57
+ {
58
+ "key": "gaia-validation-level3",
59
+ "title": "Validation · Level 3",
60
+ "tasks": 26,
61
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
62
+ "source": "Hugging Face",
63
+ "registry": "",
64
+ "hub": "",
65
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
66
+ "notes": "Official 2023 validation split."
67
+ }
68
+ ],
69
+ "citation": "@misc{mialon2023gaia,\n title={GAIA: a benchmark for General AI Assistants},\n author={Grégoire Mialon and Clémentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom},\n year={2023},\n eprint={2311.12983},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}",
70
+ "citation_source": "https://huggingface.co/spaces/gaia-benchmark/leaderboard/blob/9f133d71362e77b3539f1514f31b9c101a545fec/content.py",
71
+ "task_count": 165
72
+ }
@@ -0,0 +1,29 @@
1
+ {
2
+ "repository": "gaia-benchmark/GAIA",
3
+ "revision": "682dd723ee1e1697e00360edccf2366dc8418dd9",
4
+ "datasets": {
5
+ "gaia-validation": {
6
+ "file": "2023/validation/metadata.parquet",
7
+ "units": 165,
8
+ "sha256": "d9141bad0081af438d1ee36d188c5197035dc22a01f64e3f2ee17e45469518e9"
9
+ },
10
+ "gaia-validation-level1": {
11
+ "file": "2023/validation/metadata.level1.parquet",
12
+ "units": 53,
13
+ "level": 1,
14
+ "sha256": "5e574b0faeb4603b816e426cf7c7aefb1fe398d32f9c4861e1a4e3304f2b1281"
15
+ },
16
+ "gaia-validation-level2": {
17
+ "file": "2023/validation/metadata.level2.parquet",
18
+ "units": 86,
19
+ "level": 2,
20
+ "sha256": "36fe47d728ef4e4a4477ab21bbddb88e6739ac81a885ededf99f8b395b196f2c"
21
+ },
22
+ "gaia-validation-level3": {
23
+ "file": "2023/validation/metadata.level3.parquet",
24
+ "units": 26,
25
+ "level": 3,
26
+ "sha256": "5d004e101d8fa8b071cc50d323f8437051975aea7e894892e7d5584e9920790d"
27
+ }
28
+ }
29
+ }