nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "terminal-bench",
|
|
3
|
+
"title": "Terminal-Bench 2.0",
|
|
4
|
+
"category": "Terminal",
|
|
5
|
+
"domain": "Real terminal tasks in a container: build, configure, debug.",
|
|
6
|
+
"scoring": "Test script",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard",
|
|
9
|
+
"harbor"
|
|
10
|
+
],
|
|
11
|
+
"origin": "official",
|
|
12
|
+
"published": {
|
|
13
|
+
"agent": "terminus-2",
|
|
14
|
+
"says": "Terminus 2, the benchmark's own reference agent, which its leaderboard runs through Harbor on every model it ranks."
|
|
15
|
+
},
|
|
16
|
+
"good_to_know": [
|
|
17
|
+
"Each task's agent time limit comes from the task: 600 to 12,000 seconds."
|
|
18
|
+
],
|
|
19
|
+
"upstream_limitations": [
|
|
20
|
+
"The agent must work in the image's working directory, /app.",
|
|
21
|
+
"A local run is not a leaderboard entry: that needs Harbor's submission flow.",
|
|
22
|
+
"In the offline set, the solutions of bn-fit-modify and mteb-retrieve download from CRAN and Hugging Face, so even the reference solution fails them there.",
|
|
23
|
+
"qemu-alpine-ssh and qemu-startup fail even with the reference solution: their tests install Debian packages that are no longer published.",
|
|
24
|
+
"The tests of torch-pipeline-parallelism and torch-tensor-parallelism download about 3 GB of PyTorch, which can outlast their time limit on a slow connection.",
|
|
25
|
+
"mailman's tests fail now and then even with the reference solution.",
|
|
26
|
+
"fix-git's image no longer builds as published, so it is pulled even when the image option builds.",
|
|
27
|
+
"The build-pmars reference solution pins dpkg-dev 1.22.21, which Debian's repositories did not serve during validation; it exited 100 before verification."
|
|
28
|
+
],
|
|
29
|
+
"limitations": [
|
|
30
|
+
"The benchmark grants public internet access to 36 tasks during the agent or reference solution. The other 53 keep only their model route. Proxy access is removed before the tests start, and its traffic is recorded.",
|
|
31
|
+
"On Nullius, the tests get internet access after the agent ends. What the agent left running is stopped first, except in the 8 tasks whose tests need it."
|
|
32
|
+
],
|
|
33
|
+
"datasets": [
|
|
34
|
+
{
|
|
35
|
+
"key": "terminal-bench",
|
|
36
|
+
"title": "Terminal-Bench 2.0",
|
|
37
|
+
"tasks": 89,
|
|
38
|
+
"official": "https://github.com/laude-institute/terminal-bench",
|
|
39
|
+
"registry": "terminal-bench@2.0",
|
|
40
|
+
"hub": "",
|
|
41
|
+
"verifier": "Each task's own test script, run after the agent stops.",
|
|
42
|
+
"source": "Harbor registry"
|
|
43
|
+
}
|
|
44
|
+
],
|
|
45
|
+
"report": {
|
|
46
|
+
"sources": [
|
|
47
|
+
"terminal-bench",
|
|
48
|
+
"terminalbench",
|
|
49
|
+
"terminal-bench-2",
|
|
50
|
+
"tb2-small"
|
|
51
|
+
],
|
|
52
|
+
"file": "verifier/ctrf.json",
|
|
53
|
+
"succeeded": "reward",
|
|
54
|
+
"gained": {
|
|
55
|
+
"tests": "results.tests",
|
|
56
|
+
"name": "name",
|
|
57
|
+
"status": "status",
|
|
58
|
+
"pass": "passed",
|
|
59
|
+
"fail": "failed",
|
|
60
|
+
"count_pass": "results.summary.passed",
|
|
61
|
+
"count_fail": "results.summary.failed"
|
|
62
|
+
},
|
|
63
|
+
"words": {
|
|
64
|
+
"gained": "Tests"
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"citation": "@inproceedings{merrill2026terminalbench,\n title={Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces},\n author={Mike A Merrill and Alexander Glenn Shaw and Nicholas Carlini and Boxuan Li and Harsh Raj and Ivan Bercovich and Lin Shi and Jeong Yeon Shin and Thomas Walshe and E. Kelly Buchanan and Junhong Shen and Guanghao Ye and Haowei Lin and Jason Poulos and Maoyu Wang and Jenia Jitsev and Marianna Nezhurina and Di Lu and Orfeas Menis Mastromichalakis and Zhiwei Xu and Zizhao Chen and Yue Liu and Robert Zhang and Leon Liangyu Chen and Anurag Kashyap and Jan-Lucas Uslu and Jeffrey Li and Jianbo Wu and Minghao Yan and Song Bian and Vedang Sharma and Ke Sun and Steven Dillmann and Akshay Anand and Andrew Lanpouthakoun and Bardia Koopah and Changran Hu and Etash Kumar Guha and Gabriel H. S. Dreiman and Jiacheng Zhu and Karl Krauth and Li Zhong and Niklas Muennighoff and Robert Kwesi Amanfu and Shangyin Tan and Shreyas Pimpalgaonkar and Tushar Aggarwal and Xiangning Lin and Xin Lan and Xuandong Zhao and Yiqing Liang and Yuanli Wang and Zilong Wang and Changzhi Zhou and David Heineman and Hange Liu and Harsh Trivedi and John Yang and Junhong Lin and Manish Shetty and Michael Yang and Nabil Omi and Negin Raoof and Shanda Li and Terry Yue Zhuo and Wuwei Lin and Yiwei Dai and Yuxin Wang and Wenhao Chai and Shang Zhou and Dariush Wahdany and Ziyu She and Jiaming Hu and Zhikang Dong and Yuxuan Zhu and Sasha Cui and Ahson Saiyed and Arinbj{\\\"o}rn Kolbeinsson and Christopher Michael Rytting and Ryan Marten and Yixin Wang and Alex Dimakis and Andy Konwinski and Ludwig Schmidt},\n booktitle={The Fourteenth International Conference on Learning Representations},\n year={2026},\n url={https://openreview.net/forum?id=a7Qa4CcHak}\n}",
|
|
68
|
+
"citation_source": "https://github.com/laude-institute/terminal-bench"
|
|
69
|
+
}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Terminal-Bench 2.0's driver for the Nullius standard (JSON lines on stdin and stdout).
|
|
2
|
+
|
|
3
|
+
Each task is a folder of the upstream at its pinned commit: ``instruction.md``
|
|
4
|
+
(the prompt), ``task.toml`` (its limits), ``environment/`` (its own
|
|
5
|
+
Dockerfile, its own FROM: the task's container), ``tests/`` (``test.sh``,
|
|
6
|
+
which writes ``/logs/verifier/reward.txt``) and ``solution/solve.sh`` (the
|
|
7
|
+
authors' reference solution).
|
|
8
|
+
|
|
9
|
+
units - the dataset's tasks: a few that build fast (``small``), or every
|
|
10
|
+
task whose reference solution downloads nothing (``offline``), or all 89 (``full``)
|
|
11
|
+
setup - the task's instruction as the prompt; its own image (``image``:
|
|
12
|
+
by the run's image option, the prebuilt image its task.toml names,
|
|
13
|
+
pulled by digest, or its environment folder built as published - a
|
|
14
|
+
task whose build no longer works is pulled either way - with the
|
|
15
|
+
memory and CPUs it asks for), its own agent timeout
|
|
16
|
+
(``agent_seconds``, task.toml's agent.timeout_sec), its preparation
|
|
17
|
+
(``prepare``: tmux installed when the image has none, as Terminal-Bench's
|
|
18
|
+
own harness gives every task) and its check (``verify``: tests/ copied to
|
|
19
|
+
/tests and test.sh run in the task's container once the agent has
|
|
20
|
+
ended and what it left running is stopped - kept, for the few tasks
|
|
21
|
+
whose tests talk to a service it was asked to leave running - with
|
|
22
|
+
the way out it downloads its test runner over, its results
|
|
23
|
+
collected from /logs/verifier). The tests reach the container only
|
|
24
|
+
then; nothing is written into the workspace.
|
|
25
|
+
score - the reward test.sh wrote, read from what the check collected
|
|
26
|
+
oracle - the reference control: solution/ copied to /solution and solve.sh
|
|
27
|
+
run in the task's container, as the agent would have worked
|
|
28
|
+
|
|
29
|
+
The driver never runs docker: the run does each step in the unit's container.
|
|
30
|
+
"""
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import sys
|
|
34
|
+
import tomllib
|
|
35
|
+
|
|
36
|
+
# Fast to build, their reference solutions fetching nothing: the first ones build in seconds.
|
|
37
|
+
SMALL = ["cancel-async-tasks", "regex-log", "sqlite-db-truncate", "fix-git", "polyglot-c-py",
|
|
38
|
+
"git-leak-recovery", "sanitize-git-repo", "extract-elf"]
|
|
39
|
+
|
|
40
|
+
# Every task whose solve.sh fetches nothing (no package manager, no download): the agent works with
|
|
41
|
+
# no way out but its model on these tasks.
|
|
42
|
+
OFFLINE = [
|
|
43
|
+
"bn-fit-modify", "break-filter-js-from-html", "cancel-async-tasks", "circuit-fibsqrt", "code-from-image",
|
|
44
|
+
"constraints-scheduling", "custom-memory-heap-crash", "db-wal-recovery", "distribution-search", "extract-elf",
|
|
45
|
+
"extract-moves-from-video", "feal-differential-cryptanalysis", "feal-linear-cryptanalysis", "filter-js-from-html",
|
|
46
|
+
"fix-code-vulnerability", "fix-git", "fix-ocaml-gc", "git-leak-recovery", "git-multibranch", "gpt2-codegolf",
|
|
47
|
+
"headless-terminal", "large-scale-text-editing", "llm-inference-batching-scheduler", "mailman",
|
|
48
|
+
"make-mips-interpreter", "model-extraction-relu-logits", "modernize-scientific-stack", "mteb-retrieve",
|
|
49
|
+
"multi-source-data-merger", "openssl-selfsigned-cert", "password-recovery", "path-tracing", "path-tracing-reverse",
|
|
50
|
+
"polyglot-c-py", "polyglot-rust-c", "portfolio-optimization", "prove-plus-comm", "pytorch-model-recovery",
|
|
51
|
+
"qemu-alpine-ssh", "qemu-startup", "query-optimize", "regex-chess", "regex-log", "reshard-c4-data",
|
|
52
|
+
"sanitize-git-repo", "schemelike-metacircular-eval", "sqlite-db-truncate", "torch-pipeline-parallelism",
|
|
53
|
+
"torch-tensor-parallelism", "tune-mjcf", "video-processing", "vulnerable-secret", "write-compressor",
|
|
54
|
+
]
|
|
55
|
+
# Tasks whose solutions need the internet at the pinned upstream. The benchmark grants it,
|
|
56
|
+
# through the public-address proxy, to the agent and reference solution alike.
|
|
57
|
+
INTERNET = [
|
|
58
|
+
"adaptive-rejection-sampler", "build-cython-ext", "build-pmars", "build-pov-ray",
|
|
59
|
+
"caffe-cifar-10", "chess-best-move", "cobol-modernization", "compile-compcert",
|
|
60
|
+
"configure-git-webserver", "count-dataset-tokens", "crack-7z-hash", "dna-assembly",
|
|
61
|
+
"dna-insert", "financial-document-processor", "gcode-to-text", "hf-model-inference",
|
|
62
|
+
"install-windows-3.11", "kv-store-grpc", "largest-eigenval", "log-summary-date-ranges",
|
|
63
|
+
"make-doom-for-mips", "mcmc-sampling-stan", "merge-diff-arc-agi-task", "mteb-leaderboard",
|
|
64
|
+
"nginx-request-logging", "overfull-hbox", "protein-assembly", "pypi-server",
|
|
65
|
+
"pytorch-model-cli", "raman-fitting", "rstan-to-pystan", "sam-cell-seg",
|
|
66
|
+
"sparql-university", "sqlite-with-gcov", "train-fasttext", "winning-avg-corewars",
|
|
67
|
+
]
|
|
68
|
+
ALL = sorted(OFFLINE + INTERNET)
|
|
69
|
+
DATASETS = {"small": SMALL, "offline": OFFLINE, "full": ALL}
|
|
70
|
+
|
|
71
|
+
# The prebuilt image each task's task.toml names (environment.docker_image, the one Harbor runs unless told
|
|
72
|
+
# to build), by the digest its tag had when pinned here (Docker Hub, 2026-10-14).
|
|
73
|
+
IMAGES = {
|
|
74
|
+
"bn-fit-modify": "docker.io/alexgshaw/bn-fit-modify@sha256:5aeb11ca6e802b83816c9a31f055ac9d028309a853509400c2e3139ac2d8380d",
|
|
75
|
+
"break-filter-js-from-html": "docker.io/alexgshaw/break-filter-js-from-html@sha256:d7bfb45e47cf10d2e21dc520239215498f4eba9c1ab2d0cd7d647985dddb4fa8",
|
|
76
|
+
"cancel-async-tasks": "docker.io/alexgshaw/cancel-async-tasks@sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d",
|
|
77
|
+
"circuit-fibsqrt": "docker.io/alexgshaw/circuit-fibsqrt@sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a",
|
|
78
|
+
"code-from-image": "docker.io/alexgshaw/code-from-image@sha256:2be9cdb746ed18c1b2b1404032efae0b2235a1b8fc479ef17730c95409d89f76",
|
|
79
|
+
"constraints-scheduling": "docker.io/alexgshaw/constraints-scheduling@sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0",
|
|
80
|
+
"custom-memory-heap-crash": "docker.io/alexgshaw/custom-memory-heap-crash@sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557",
|
|
81
|
+
"db-wal-recovery": "docker.io/alexgshaw/db-wal-recovery@sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e",
|
|
82
|
+
"distribution-search": "docker.io/alexgshaw/distribution-search@sha256:1448f7aa1c6080488251d00350e8e90477858989b1bec0accc866d7b3220738a",
|
|
83
|
+
"extract-elf": "docker.io/alexgshaw/extract-elf@sha256:6932e4cb318464307eacd497ef8dc617eaf551b6a90231f815ec0b911895cfed",
|
|
84
|
+
"extract-moves-from-video": "docker.io/alexgshaw/extract-moves-from-video@sha256:14a9dcca7d08ce53c5a0ed9fbd663a996ef28f207c199ed69d9ccf6064f8b184",
|
|
85
|
+
"feal-differential-cryptanalysis": "docker.io/alexgshaw/feal-differential-cryptanalysis@sha256:bea93bafe9eab601ca58729f5735a13f8339f52055322d1cf840e3742b54a287",
|
|
86
|
+
"feal-linear-cryptanalysis": "docker.io/alexgshaw/feal-linear-cryptanalysis@sha256:ea78749a8422c6228d26c8753f45906a6a2df089906764b7e3c3e27e8f718ee7",
|
|
87
|
+
"filter-js-from-html": "docker.io/alexgshaw/filter-js-from-html@sha256:92acda0f124b988036a6f426ce0bc47fac19f5efe9fc5e6ea3ea52ccb075d0a4",
|
|
88
|
+
"fix-code-vulnerability": "docker.io/alexgshaw/fix-code-vulnerability@sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0",
|
|
89
|
+
"fix-git": "docker.io/alexgshaw/fix-git@sha256:61e431c00c58df652287aadce5457634d9f9330cfdd153ebdf2802df0d540119",
|
|
90
|
+
"fix-ocaml-gc": "docker.io/alexgshaw/fix-ocaml-gc@sha256:66a1e2293076c9ef27a118611b3089a6284fdf467fd4675592d0cce7d11e5b0b",
|
|
91
|
+
"git-leak-recovery": "docker.io/alexgshaw/git-leak-recovery@sha256:62160e522a4238b8bcd522411ea97b5b200f811e5899d34947e66809bef7e586",
|
|
92
|
+
"git-multibranch": "docker.io/alexgshaw/git-multibranch@sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5",
|
|
93
|
+
"gpt2-codegolf": "docker.io/alexgshaw/gpt2-codegolf@sha256:537e7bd26b02db5c761230515fa09dc05f89403c6737fbdae4b5793ebc2c7066",
|
|
94
|
+
"headless-terminal": "docker.io/alexgshaw/headless-terminal@sha256:eb7e209672bf6cef2785fafd9e13509b10626c327bcc2b37f5bf40ca83eaf3aa",
|
|
95
|
+
"large-scale-text-editing": "docker.io/alexgshaw/large-scale-text-editing@sha256:719adca3f1388220546ce6a155eee56eff3c4fe318183100320606a210f6b59c",
|
|
96
|
+
"llm-inference-batching-scheduler": "docker.io/alexgshaw/llm-inference-batching-scheduler@sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774",
|
|
97
|
+
"mailman": "docker.io/alexgshaw/mailman@sha256:04fac73db7e2b4f49d43f09fbc2266a94d70131f07efa1393d370abc059f4094",
|
|
98
|
+
"make-mips-interpreter": "docker.io/alexgshaw/make-mips-interpreter@sha256:082fc8821b317f30fdfbf8d08d528874ce331c03e8083aef0406d48cdd7132a2",
|
|
99
|
+
"model-extraction-relu-logits": "docker.io/alexgshaw/model-extraction-relu-logits@sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce",
|
|
100
|
+
"modernize-scientific-stack": "docker.io/alexgshaw/modernize-scientific-stack@sha256:64e69cee13bf6b0b9016e735b51891bce996a60f1e2d1a005bf12c949e221d71",
|
|
101
|
+
"mteb-retrieve": "docker.io/alexgshaw/mteb-retrieve@sha256:99dd676a21fd3124b7b8164cd93e8f0c51f1fdab2f84a831c61049eff78d063d",
|
|
102
|
+
"multi-source-data-merger": "docker.io/alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e",
|
|
103
|
+
"openssl-selfsigned-cert": "docker.io/alexgshaw/openssl-selfsigned-cert@sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12",
|
|
104
|
+
"password-recovery": "docker.io/alexgshaw/password-recovery@sha256:15fcd84bf3b06270ee148211cc6da6599b13ec2f41ac1d0894cf75c2225872ee",
|
|
105
|
+
"path-tracing": "docker.io/alexgshaw/path-tracing@sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482",
|
|
106
|
+
"path-tracing-reverse": "docker.io/alexgshaw/path-tracing-reverse@sha256:7911e42479d8f6d8af1248bc52442a0efbc5a33e3ecf535fdc20d252a2c64142",
|
|
107
|
+
"polyglot-c-py": "docker.io/alexgshaw/polyglot-c-py@sha256:0f1c3b7816d70cf5551573fd6aeef76893f2ae3000be2419997b6871b5d987ed",
|
|
108
|
+
"polyglot-rust-c": "docker.io/alexgshaw/polyglot-rust-c@sha256:3ba84f0e138a4b65e58c53f920a0fc852571e6402901c3bdbb876e9dab6e6c5e",
|
|
109
|
+
"portfolio-optimization": "docker.io/alexgshaw/portfolio-optimization@sha256:93461db3cec697c31585941a3b9387c1398495aa1ff0aee2ac752c7230b50ef8",
|
|
110
|
+
"prove-plus-comm": "docker.io/alexgshaw/prove-plus-comm@sha256:e26741d01681a9da1beeff8b7b7b65fe2b921b8b122a306a3707a81ebd051f73",
|
|
111
|
+
"pytorch-model-recovery": "docker.io/alexgshaw/pytorch-model-recovery@sha256:83efba8d1a163dfe62694789e271ed55010509b6deabbf2d7d6472b40ca8d69a",
|
|
112
|
+
"qemu-alpine-ssh": "docker.io/alexgshaw/qemu-alpine-ssh@sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964",
|
|
113
|
+
"qemu-startup": "docker.io/alexgshaw/qemu-startup@sha256:1a666e049db553c5e713830b1f5de1953210ab7c99212d006bb4e46cbc6b6d04",
|
|
114
|
+
"query-optimize": "docker.io/alexgshaw/query-optimize@sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14",
|
|
115
|
+
"regex-chess": "docker.io/alexgshaw/regex-chess@sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c",
|
|
116
|
+
"regex-log": "docker.io/alexgshaw/regex-log@sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9",
|
|
117
|
+
"reshard-c4-data": "docker.io/alexgshaw/reshard-c4-data@sha256:92d209e704785e17ad6b3f740e813dd72bc7ac33982ce435d197ab9d890eac2a",
|
|
118
|
+
"sanitize-git-repo": "docker.io/alexgshaw/sanitize-git-repo@sha256:4b5234da5bb0d67f3b0bf8db40a2883c07a5219f62b64c2bf9ff1ac84cd0f672",
|
|
119
|
+
"schemelike-metacircular-eval": "docker.io/alexgshaw/schemelike-metacircular-eval@sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db",
|
|
120
|
+
"sqlite-db-truncate": "docker.io/alexgshaw/sqlite-db-truncate@sha256:aabac93c93bd1f310e6a6fb893911d7735026ed18491c72133c9196a09092ca4",
|
|
121
|
+
"torch-pipeline-parallelism": "docker.io/alexgshaw/torch-pipeline-parallelism@sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b",
|
|
122
|
+
"torch-tensor-parallelism": "docker.io/alexgshaw/torch-tensor-parallelism@sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690",
|
|
123
|
+
"tune-mjcf": "docker.io/alexgshaw/tune-mjcf@sha256:56c197739db2db79562384581fcc9d3d71e6c171bf052104b1a7ee0282e7c3cf",
|
|
124
|
+
"video-processing": "docker.io/alexgshaw/video-processing@sha256:a2c0f39e3ab04e67ac6e49cad9167bb0d987babe32511768c01aa0c6905a875f",
|
|
125
|
+
"vulnerable-secret": "docker.io/alexgshaw/vulnerable-secret@sha256:61ebb40454dd103aa2f7e71ad6dafd91cf2b301e6bb07e69d5b472412d1ee15b",
|
|
126
|
+
"write-compressor": "docker.io/alexgshaw/write-compressor@sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1",
|
|
127
|
+
# The additional internet tasks: task.toml tags resolved on 2026-11-12.
|
|
128
|
+
"adaptive-rejection-sampler": "docker.io/alexgshaw/adaptive-rejection-sampler@sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f",
|
|
129
|
+
"build-cython-ext": "docker.io/alexgshaw/build-cython-ext@sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594",
|
|
130
|
+
"build-pmars": "docker.io/alexgshaw/build-pmars@sha256:57a82706f7491e0b2d1ff7e927e97ae5339892dd09c47ac43290b4ba1f5abc6f",
|
|
131
|
+
"build-pov-ray": "docker.io/alexgshaw/build-pov-ray@sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b",
|
|
132
|
+
"caffe-cifar-10": "docker.io/alexgshaw/caffe-cifar-10@sha256:5b54d6e8f1b392cdeb819e28215853f500b72b8e08b91270c7a3693bf44d1fe2",
|
|
133
|
+
"chess-best-move": "docker.io/alexgshaw/chess-best-move@sha256:bb447f94d9e2a8ed879f85c85a514b213b7418f9fe11fd7b2428a0b0e436e647",
|
|
134
|
+
"cobol-modernization": "docker.io/alexgshaw/cobol-modernization@sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0",
|
|
135
|
+
"compile-compcert": "docker.io/alexgshaw/compile-compcert@sha256:f316947b7be23ad0d187320126eb1cbb5685f48342c6543fdd3ac839aa9e1a7f",
|
|
136
|
+
"configure-git-webserver": "docker.io/alexgshaw/configure-git-webserver@sha256:9e48389b917fd4650dda1f64406bd7198bbf20dc1c6b0e5c4d81a3c16369f88c",
|
|
137
|
+
"count-dataset-tokens": "docker.io/alexgshaw/count-dataset-tokens@sha256:9c513c4ba342bd95be501f6f5eadd4806a2b18749c87c251615825100130232d",
|
|
138
|
+
"crack-7z-hash": "docker.io/alexgshaw/crack-7z-hash@sha256:0f4453abd774c5a3d3d7e66ba28fae88ec2e49ada3a993b324ebc16c348666d3",
|
|
139
|
+
"dna-assembly": "docker.io/alexgshaw/dna-assembly@sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569",
|
|
140
|
+
"dna-insert": "docker.io/alexgshaw/dna-insert@sha256:4dd8760694e355de85ecf03605cb487ccdb33322f6afad513d2920829baa33b2",
|
|
141
|
+
"financial-document-processor": "docker.io/alexgshaw/financial-document-processor@sha256:ef6c9cfaaf14cdd200163008a188d5baa9f626f3bfb8d3e7d6f30c08518e6251",
|
|
142
|
+
"gcode-to-text": "docker.io/alexgshaw/gcode-to-text@sha256:0979ef40c6a3e8c4e7ab5b6c2524c74625a84799e15cd45dd98dcf83b11efd4d",
|
|
143
|
+
"hf-model-inference": "docker.io/alexgshaw/hf-model-inference@sha256:f96dd7d3c85c7257b404fd63ba8d24ced2478cd90232c20e297c372f953fa607",
|
|
144
|
+
"install-windows-3.11": "docker.io/alexgshaw/install-windows-3.11@sha256:c6695097316ef8810abce0545e347e65354aef9d51d4f7f662c3242e742f6a3e",
|
|
145
|
+
"kv-store-grpc": "docker.io/alexgshaw/kv-store-grpc@sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791",
|
|
146
|
+
"largest-eigenval": "docker.io/alexgshaw/largest-eigenval@sha256:cb9e3e3b09d0783f3bbcfeec8d77bc32d3791204fb2be6650b19be4fbc082b2e",
|
|
147
|
+
"log-summary-date-ranges": "docker.io/alexgshaw/log-summary-date-ranges@sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51",
|
|
148
|
+
"make-doom-for-mips": "docker.io/alexgshaw/make-doom-for-mips@sha256:80a9459f1b94b692ea8346f68e6b864f6c4d984fbdf6ffd68ad332308300c2ec",
|
|
149
|
+
"mcmc-sampling-stan": "docker.io/alexgshaw/mcmc-sampling-stan@sha256:073fc36a3347b4f6b43bd6e98339542c41ed531c771af8bee40ba10f00cfabaf",
|
|
150
|
+
"merge-diff-arc-agi-task": "docker.io/alexgshaw/merge-diff-arc-agi-task@sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4",
|
|
151
|
+
"mteb-leaderboard": "docker.io/alexgshaw/mteb-leaderboard@sha256:968de9456bfc53728f7df0cb6c9ce33e17bd0e5563ddfe031e6ed2964965aecf",
|
|
152
|
+
"nginx-request-logging": "docker.io/alexgshaw/nginx-request-logging@sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d",
|
|
153
|
+
"overfull-hbox": "docker.io/alexgshaw/overfull-hbox@sha256:67fe0ccc0ff2305d6fcb3652977d377b901bb7bd96904b6765835935574992e1",
|
|
154
|
+
"protein-assembly": "docker.io/alexgshaw/protein-assembly@sha256:94de701b57e7ecfdbada6044f8534876e4b58874f8f313184dadc4d7533e0818",
|
|
155
|
+
"pypi-server": "docker.io/alexgshaw/pypi-server@sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276",
|
|
156
|
+
"pytorch-model-cli": "docker.io/alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67",
|
|
157
|
+
"raman-fitting": "docker.io/alexgshaw/raman-fitting@sha256:401fa95d5d491f44a473eda9d2c52930f9565cc889c796eabcb6e20fec26852e",
|
|
158
|
+
"rstan-to-pystan": "docker.io/alexgshaw/rstan-to-pystan@sha256:b23d48839408524326095c15ada4145c772b87646cf299022ff1f58c97baba91",
|
|
159
|
+
"sam-cell-seg": "docker.io/alexgshaw/sam-cell-seg@sha256:d76bdeef19d8113f4094013249c982eca93e62e46130484846269b2d378879d0",
|
|
160
|
+
"sparql-university": "docker.io/alexgshaw/sparql-university@sha256:92fa4304e51167f9fb4466144eba33ef9272a7df0ac1cab9fc3986bccd1ab708",
|
|
161
|
+
"sqlite-with-gcov": "docker.io/alexgshaw/sqlite-with-gcov@sha256:f986820b00c74ce75db7288719d364df313fde7c7917bea6eb6352a08431a89f",
|
|
162
|
+
"train-fasttext": "docker.io/alexgshaw/train-fasttext@sha256:3c77da9617e7fa577de04aef73fb1ac3114be32d7ffef72609c83929d842b157",
|
|
163
|
+
"winning-avg-corewars": "docker.io/alexgshaw/winning-avg-corewars@sha256:ad7a6b1c53aed59e10916c692aa7e2ae00042a4283187430aee1e3e79f72d5c6",
|
|
164
|
+
}
|
|
165
|
+
# Tasks whose Dockerfile no longer builds as published (it clones a repository that is gone): pulled whatever
|
|
166
|
+
# the run's image option says.
|
|
167
|
+
PULL = {"fix-git"}
|
|
168
|
+
# Where each task's container comes from: the run's ``image`` option (version 2) - pull, its prebuilt image,
|
|
169
|
+
# or build, from its environment/Dockerfile as published. A run without the option (version 1) builds.
|
|
170
|
+
IMAGE = os.environ.get("NULLIUS_OPTION_IMAGE", "build")
|
|
171
|
+
LOGS = "/logs/verifier"
|
|
172
|
+
|
|
173
|
+
# Tasks whose tests talk to a service the agent was asked to start and leave running (a mail server, an
|
|
174
|
+
# SSH or telnet login into a VM, a web server behind a push hook): what the agent left running is kept
|
|
175
|
+
# for their check. Every other task's check runs with nothing of the agent's still running.
|
|
176
|
+
KEEP_PROCESSES = {"git-multibranch", "mailman", "qemu-alpine-ssh", "qemu-startup",
|
|
177
|
+
"configure-git-webserver", "kv-store-grpc", "nginx-request-logging", "pypi-server"}
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# Terminal-Bench's own harness gives every task's container tmux (its reference agent works in a tmux
|
|
181
|
+
# session, and git-multibranch's solution starts one). Most images have none: the unit's container is
|
|
182
|
+
# prepared with it, as root, through the check's proxy, before anyone works in it; an image that has it
|
|
183
|
+
# is left alone.
|
|
184
|
+
TMUX = r"""
|
|
185
|
+
command -v tmux >/dev/null 2>&1 && { echo "tmux is already here: $(command -v tmux)"; exit 0; }
|
|
186
|
+
if command -v apt-get >/dev/null 2>&1; then
|
|
187
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
188
|
+
apt-get update -qq && apt-get install -y -qq --no-install-recommends tmux
|
|
189
|
+
elif command -v apk >/dev/null 2>&1; then apk add --no-cache tmux
|
|
190
|
+
elif command -v dnf >/dev/null 2>&1; then dnf install -y tmux
|
|
191
|
+
elif command -v microdnf >/dev/null 2>&1; then microdnf install -y tmux
|
|
192
|
+
elif command -v yum >/dev/null 2>&1; then yum install -y tmux
|
|
193
|
+
elif command -v zypper >/dev/null 2>&1; then zypper --non-interactive install tmux
|
|
194
|
+
elif command -v pacman >/dev/null 2>&1; then pacman -Sy --noconfirm tmux
|
|
195
|
+
else echo "no package manager to install tmux with"; exit 1
|
|
196
|
+
fi
|
|
197
|
+
command -v tmux || { echo "tmux was not installed"; exit 1; }
|
|
198
|
+
"""
|
|
199
|
+
PREPARE = {"run": ["sh", "-c", TMUX], "seconds": 600, "network": True}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def task(unit):
|
|
203
|
+
if unit not in ALL:
|
|
204
|
+
raise ValueError(f"{unit} is not a task of this benchmark's datasets")
|
|
205
|
+
here = os.path.join(os.environ["NULLIUS_UPSTREAM"], unit)
|
|
206
|
+
with open(os.path.join(here, "task.toml"), "rb") as f:
|
|
207
|
+
return here, tomllib.load(f)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def units(dataset):
|
|
211
|
+
return {"units": list(DATASETS[dataset])}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def setup(unit, dir):
|
|
215
|
+
here, t = task(unit)
|
|
216
|
+
with open(os.path.join(here, "instruction.md")) as f:
|
|
217
|
+
prompt = f.read().strip()
|
|
218
|
+
env = t.get("environment") or {}
|
|
219
|
+
image = {"pull": IMAGES[unit]} if IMAGE == "pull" or unit in PULL else {"build": f"{unit}/environment"}
|
|
220
|
+
if env.get("memory_mb"):
|
|
221
|
+
image["memory_mb"] = env["memory_mb"]
|
|
222
|
+
if env.get("cpus"):
|
|
223
|
+
image["cpus"] = env["cpus"]
|
|
224
|
+
# the task's own agent timeout from version 2 on; version 1 ran every task at the run's turn_seconds
|
|
225
|
+
agent = (t.get("agent") or {}).get("timeout_sec") if os.environ.get("NULLIUS_VERSION", "1") != "1" else None
|
|
226
|
+
return {"prompt": prompt, "environment": "task", "image": image,
|
|
227
|
+
**({"internet": True} if unit in INTERNET else {}),
|
|
228
|
+
**({"agent_seconds": agent} if agent else {}), # the task's own agent timeout, not the run's
|
|
229
|
+
"prepare": PREPARE,
|
|
230
|
+
"verify": {"copy": {f"{unit}/tests": "/tests"}, "run": ["bash", "/tests/test.sh"], "collect": LOGS,
|
|
231
|
+
"seconds": (t.get("verifier") or {}).get("timeout_sec") or 900, "network": True,
|
|
232
|
+
**({"keep_processes": True} if unit in KEEP_PROCESSES else {})}}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def score(unit, dir, transcript, answer, verified=None):
|
|
236
|
+
if not verified:
|
|
237
|
+
return {"reward": 0.0, "passed": False, "detail": {"why": "the task's check did not run"}}
|
|
238
|
+
detail = {"exit": verified.get("exit"), "timed_out": verified.get("timed_out"),
|
|
239
|
+
"output": (verified.get("output") or "")[-3000:]}
|
|
240
|
+
try:
|
|
241
|
+
with open(os.path.join(verified["dir"], "reward.txt")) as f:
|
|
242
|
+
reward = float(f.read().strip() or 0)
|
|
243
|
+
except (OSError, ValueError):
|
|
244
|
+
detail["why"] = "test.sh wrote no reward"
|
|
245
|
+
reward = 0.0
|
|
246
|
+
try:
|
|
247
|
+
with open(os.path.join(verified["dir"], "ctrf.json")) as f:
|
|
248
|
+
detail["tests"] = (json.load(f).get("results") or {}).get("summary")
|
|
249
|
+
except (OSError, ValueError):
|
|
250
|
+
pass
|
|
251
|
+
return {"reward": 1.0 if reward >= 1.0 else 0.0, "passed": reward >= 1.0, "detail": detail}
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def oracle(unit, dir):
|
|
255
|
+
here, t = task(unit)
|
|
256
|
+
return {"answer": "", "solve": {"copy": {f"{unit}/solution": "/solution"}, "run": ["bash", "/solution/solve.sh"],
|
|
257
|
+
"seconds": (t.get("agent") or {}).get("timeout_sec") or 900}}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
HANDLERS = {"units": units, "setup": setup, "score": score, "oracle": oracle}
|
|
261
|
+
for line in sys.stdin:
|
|
262
|
+
try:
|
|
263
|
+
req = json.loads(line)
|
|
264
|
+
out = HANDLERS[req.pop("op")](**req)
|
|
265
|
+
except Exception as e:
|
|
266
|
+
out = {"error": f"{type(e).__name__}: {e}"}
|
|
267
|
+
print(json.dumps(out), flush=True)
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "terminal-bench",
|
|
4
|
+
"label": "Terminal-Bench 2.0",
|
|
5
|
+
"summary": "Each task is its own container, built from the task's Dockerfile as published; the agent works in it from the image's working directory, and the task's own tests are copied in and run there once it has ended.",
|
|
6
|
+
"family": "agent",
|
|
7
|
+
"needs": [
|
|
8
|
+
"agent.workspace/v1"
|
|
9
|
+
],
|
|
10
|
+
"upstream": {
|
|
11
|
+
"repository": "https://github.com/laude-institute/terminal-bench-2",
|
|
12
|
+
"tested": "2fd12b88aafdd04a52c298e3940bcb189f9766d6",
|
|
13
|
+
"tested_date": "2026-04-29",
|
|
14
|
+
"license": "Apache-2.0"
|
|
15
|
+
},
|
|
16
|
+
"driver": {
|
|
17
|
+
"command": [
|
|
18
|
+
"python3",
|
|
19
|
+
"{here}/driver.py"
|
|
20
|
+
]
|
|
21
|
+
},
|
|
22
|
+
"datasets": [
|
|
23
|
+
{
|
|
24
|
+
"id": "small",
|
|
25
|
+
"label": "Eight tasks that build fast",
|
|
26
|
+
"units": 8,
|
|
27
|
+
"environment": "task"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "offline",
|
|
31
|
+
"label": "Every task whose reference solution downloads nothing",
|
|
32
|
+
"units": 53,
|
|
33
|
+
"environment": "task"
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"id": "full",
|
|
37
|
+
"label": "All 89 tasks",
|
|
38
|
+
"units": 89,
|
|
39
|
+
"environment": "task"
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
"units": {
|
|
43
|
+
"noun": "task",
|
|
44
|
+
"nouns": "tasks"
|
|
45
|
+
},
|
|
46
|
+
"environments": {
|
|
47
|
+
"task": {
|
|
48
|
+
"summary": "The task's own image, built from its environment folder at the pinned commit - its own FROM, as published - with the agent added at run time.",
|
|
49
|
+
"image": "own"
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
"controls": {
|
|
53
|
+
"nop": {
|
|
54
|
+
"source": "nullius",
|
|
55
|
+
"from": "Nullius: no agent runs; the task's tests run on its container as its image left it."
|
|
56
|
+
},
|
|
57
|
+
"oracle": {
|
|
58
|
+
"source": "official",
|
|
59
|
+
"from": "The task's solution/solve.sh, the authors' reference solution, run in the task's container."
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
"score": {
|
|
63
|
+
"kind": "pass",
|
|
64
|
+
"says": "the task's own tests/test.sh, run in its container after the agent ends, writes reward 1"
|
|
65
|
+
},
|
|
66
|
+
"limits": {
|
|
67
|
+
"turn_seconds": 900
|
|
68
|
+
},
|
|
69
|
+
"options": {
|
|
70
|
+
"image": {
|
|
71
|
+
"summary": "Whether each task uses its pinned prebuilt container image or builds one from its published Dockerfile.",
|
|
72
|
+
"values": [
|
|
73
|
+
"pull",
|
|
74
|
+
"build"
|
|
75
|
+
],
|
|
76
|
+
"default": "pull",
|
|
77
|
+
"about": {
|
|
78
|
+
"pull": "The authors' prebuilt image, as they froze it: what Harbor runs.",
|
|
79
|
+
"build": "Built now from the task's Dockerfile: its packages are whatever the mirrors serve today."
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"proven": {
|
|
84
|
+
"small": {
|
|
85
|
+
"units": {
|
|
86
|
+
"units": 8,
|
|
87
|
+
"passed": "8 ids, the same twice, unique",
|
|
88
|
+
"on": "2026-11-03"
|
|
89
|
+
},
|
|
90
|
+
"leak": {
|
|
91
|
+
"units": 8,
|
|
92
|
+
"looked_for": 0,
|
|
93
|
+
"short": 0,
|
|
94
|
+
"passed": "8/8",
|
|
95
|
+
"note": "8 unit(s) whose oracle gave nothing to look for",
|
|
96
|
+
"on": "2026-11-03"
|
|
97
|
+
},
|
|
98
|
+
"nop": {
|
|
99
|
+
"passed": "0/8",
|
|
100
|
+
"value": 0.0,
|
|
101
|
+
"expect": {
|
|
102
|
+
"at_most": 0.0
|
|
103
|
+
},
|
|
104
|
+
"units": 8,
|
|
105
|
+
"job": "2026-11-03__05-20-12__terminal-bench__small__prove-nop",
|
|
106
|
+
"source": "nullius",
|
|
107
|
+
"on": "2026-11-03"
|
|
108
|
+
},
|
|
109
|
+
"oracle": {
|
|
110
|
+
"passed": "8/8",
|
|
111
|
+
"value": 1.0,
|
|
112
|
+
"expect": {
|
|
113
|
+
"at_least": 1.0
|
|
114
|
+
},
|
|
115
|
+
"units": 8,
|
|
116
|
+
"job": "2026-11-03__05-20-12__terminal-bench__small__prove-oracle",
|
|
117
|
+
"source": "official",
|
|
118
|
+
"on": "2026-11-03"
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
"offline": {
|
|
122
|
+
"units": {
|
|
123
|
+
"units": 53,
|
|
124
|
+
"passed": "53 ids, the same twice, unique",
|
|
125
|
+
"on": "2026-11-03"
|
|
126
|
+
},
|
|
127
|
+
"leak": {
|
|
128
|
+
"units": 53,
|
|
129
|
+
"looked_for": 0,
|
|
130
|
+
"short": 0,
|
|
131
|
+
"passed": "53/53",
|
|
132
|
+
"note": "53 unit(s) whose oracle gave nothing to look for",
|
|
133
|
+
"on": "2026-11-03"
|
|
134
|
+
},
|
|
135
|
+
"failed": {
|
|
136
|
+
"nop": {
|
|
137
|
+
"passed": "0/53",
|
|
138
|
+
"value": 0.0,
|
|
139
|
+
"expect": {
|
|
140
|
+
"at_most": 0.0
|
|
141
|
+
},
|
|
142
|
+
"units": 53,
|
|
143
|
+
"job": "2026-11-03__05-20-12__terminal-bench__offline__prove-nop",
|
|
144
|
+
"on": "2026-11-03",
|
|
145
|
+
"problems": [
|
|
146
|
+
"2 unit(s) ended in an error before the score: read the job's trials"
|
|
147
|
+
]
|
|
148
|
+
},
|
|
149
|
+
"oracle": {
|
|
150
|
+
"passed": "45/53",
|
|
151
|
+
"value": 0.8491,
|
|
152
|
+
"expect": {
|
|
153
|
+
"at_least": 1.0
|
|
154
|
+
},
|
|
155
|
+
"units": 53,
|
|
156
|
+
"job": "2026-11-03__05-20-12__terminal-bench__offline__prove-oracle",
|
|
157
|
+
"on": "2026-11-03",
|
|
158
|
+
"problems": [
|
|
159
|
+
"3 unit(s) ended in an error before the score: read the job's trials",
|
|
160
|
+
"pass rate 0.8491, expected at least 1.0"
|
|
161
|
+
]
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"entry": "harbor run", "setting": "--dataset / -t / -i / -x / -l (which tasks)", "values": "terminal-bench@2.0: all 89 tasks", "default": "all 89",
|
|
3
|
+
"nullius": "dataset:offline"},
|
|
4
|
+
{"entry": "harbor run", "setting": "the 8 fast-building tasks", "values": "", "default": "",
|
|
5
|
+
"nullius": "dataset:small"},
|
|
6
|
+
{"entry": "harbor run", "setting": "the full 89-task set", "values": "", "default": "",
|
|
7
|
+
"nullius": "dataset:full"},
|
|
8
|
+
{"entry": "harbor run", "setting": "-a/--agent terminus-2 (the reference agent, README with anthropic/claude-opus-4-1)", "values": "", "default": "",
|
|
9
|
+
"nullius": "fixed:Terminus 2 from the agents catalogue - Harbor's own agent at its pinned release, never re-implemented, working in the task's container; tmux is added to each task's container before it starts, as Terminal-Bench's own harness does"},
|
|
10
|
+
{"entry": "harbor run", "setting": "-a nop / oracle", "values": "", "default": "",
|
|
11
|
+
"nullius": "fixed:the standard's nop and oracle controls - the same checks"},
|
|
12
|
+
{"entry": "harbor run", "setting": "-m/--model", "values": "the agent's model", "default": "",
|
|
13
|
+
"nullius": "fixed:the run's model - sent through the relay to the agent chosen"},
|
|
14
|
+
{"entry": "harbor run", "setting": "-k/--n-attempts", "values": "attempts per task", "default": "1",
|
|
15
|
+
"nullius": "fixed:the run's attempts (cli run --repeat k, the builder's Attempts per task) - each a trial of its own, the job reporting pass^k and pass@k; 1 by default, Harbor's"},
|
|
16
|
+
{"entry": "harbor run", "setting": "--timeout-multiplier and the agent/verifier/setup/build multipliers", "values": "", "default": "1.0",
|
|
17
|
+
"nullius": "fixed:1.0 - each task's own timeouts as task.toml declares them (version 2)"},
|
|
18
|
+
{"entry": "task.toml", "setting": "agent.timeout_sec", "values": "600 to 12000", "default": "per task",
|
|
19
|
+
"nullius": "fixed:each task's own - as Harbor runs it (version 2; version 1 cut every task at 900 s); said in Good to know"},
|
|
20
|
+
{"entry": "task.toml", "setting": "verifier.timeout_sec", "values": "360 to 12000", "default": "per task",
|
|
21
|
+
"nullius": "fixed:each task's own - as Harbor runs it"},
|
|
22
|
+
{"entry": "task.toml", "setting": "environment.docker_image vs its Dockerfile", "values": "prebuilt image, or build", "default": "prebuilt (Harbor's)",
|
|
23
|
+
"nullius": "option:image"},
|
|
24
|
+
{"entry": "task.toml", "setting": "environment.cpus, memory_mb", "values": "per task", "default": "per task",
|
|
25
|
+
"nullius": "fixed:each task's own - given to its container, as Harbor does"},
|
|
26
|
+
{"entry": "task.toml", "setting": "environment.storage_mb", "values": "10240 for every task", "default": "10240",
|
|
27
|
+
"nullius": "fixed:docker's own limit - not applied: a container's disk is not capped on the local docker storage drivers it runs on"},
|
|
28
|
+
{"entry": "task.toml", "setting": "environment.build_timeout_sec", "values": "600 for every task", "default": "600",
|
|
29
|
+
"nullius": "fixed:no limit on a build - an image is built once and kept, pulled by digest by default"},
|
|
30
|
+
{"entry": "task.toml", "setting": "environment.gpus", "values": "0 for every task", "default": "0",
|
|
31
|
+
"nullius": "fixed:0 - no task asks for one"},
|
|
32
|
+
{"entry": "task.toml", "setting": "environment.allow_internet", "values": "true for every task", "default": "true",
|
|
33
|
+
"nullius": "fixed:false for the agent - its only way out is its model, through the relay; the tests get a network after it ends; said in the limitations"},
|
|
34
|
+
{"entry": "harbor run", "setting": "--override-cpus/-memory/-storage/-gpus, -e (docker, daytona, modal ...)", "values": "", "default": "none, docker",
|
|
35
|
+
"nullius": "not offered - where and on what the container runs, not what is measured; the standard's backends are local and docker"}
|
|
36
|
+
]
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.capability/v1",
|
|
3
|
+
"key": "keyword-memory",
|
|
4
|
+
"label": "Example memory",
|
|
5
|
+
"example": true,
|
|
6
|
+
"summary": "Nullius's example memory, to read or copy: BM25 over whole sessions.",
|
|
7
|
+
"upstream": {
|
|
8
|
+
"version": "1",
|
|
9
|
+
"license": "same as Nullius"
|
|
10
|
+
},
|
|
11
|
+
"kind": "memory",
|
|
12
|
+
"start": {
|
|
13
|
+
"command": [
|
|
14
|
+
"python3",
|
|
15
|
+
"{here}/capability.py"
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"offers": [
|
|
19
|
+
"cap.memory/v1"
|
|
20
|
+
],
|
|
21
|
+
"proven": {
|
|
22
|
+
"cap.memory/v1": {
|
|
23
|
+
"passed": "2/2",
|
|
24
|
+
"on": "2026-11-02",
|
|
25
|
+
"model": "openai/gpt-5-mini",
|
|
26
|
+
"provider": "openrouter",
|
|
27
|
+
"version": "1",
|
|
28
|
+
"kit_version": 1,
|
|
29
|
+
"backend": "docker",
|
|
30
|
+
"job": "2026-11-02__14-19-42__kit__cap-memory-v1__keyword-memory"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""The reference memory: BM25 over whole sessions, the top few recalled.
|
|
2
|
+
|
|
3
|
+
cap.memory/v1 in JSON lines: ``add {session}`` keeps a session
|
|
4
|
+
(``{id, date, turns: [{role, text}], text}``), ``recall {question}`` answers
|
|
5
|
+
``{memory}`` - the best-matching sessions, each with its date, as text.
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
TOP = 3
|
|
13
|
+
WORD = re.compile(r"[a-z0-9]+")
|
|
14
|
+
kept = []
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def words(text):
|
|
18
|
+
return WORD.findall((text or "").lower())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def recall(question):
|
|
22
|
+
q = set(words(question))
|
|
23
|
+
if not kept or not q:
|
|
24
|
+
return ""
|
|
25
|
+
n = len(kept)
|
|
26
|
+
avg = sum(len(d["words"]) for d in kept) / n
|
|
27
|
+
df = {w: sum(1 for d in kept if w in d["set"]) for w in q}
|
|
28
|
+
def bm25(d):
|
|
29
|
+
s = 0.0
|
|
30
|
+
for w in q:
|
|
31
|
+
if w in d["set"]:
|
|
32
|
+
tf = d["words"].count(w)
|
|
33
|
+
idf = math.log(1 + (n - df[w] + 0.5) / (df[w] + 0.5))
|
|
34
|
+
s += idf * tf * 2.2 / (tf + 1.2 * (0.25 + 0.75 * len(d["words"]) / avg))
|
|
35
|
+
return s
|
|
36
|
+
best = sorted(kept, key=bm25, reverse=True)[:TOP]
|
|
37
|
+
return "\n\n".join(f"[{d['session'].get('date') or d['session'].get('id')}]\n{d['text']}" for d in best if bm25(d) > 0)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
for line in sys.stdin:
|
|
41
|
+
try:
|
|
42
|
+
req = json.loads(line)
|
|
43
|
+
if req["op"] == "add":
|
|
44
|
+
s = req["session"]
|
|
45
|
+
text = s.get("text") or "\n".join(f"{t.get('role')}: {t.get('text')}" for t in s.get("turns") or [])
|
|
46
|
+
ws = words(text)
|
|
47
|
+
kept.append({"session": s, "text": text, "words": ws, "set": set(ws)})
|
|
48
|
+
out = {}
|
|
49
|
+
elif req["op"] == "recall":
|
|
50
|
+
out = {"memory": recall(req["question"])}
|
|
51
|
+
else:
|
|
52
|
+
out = {"error": f"unknown op {req['op']!r}"}
|
|
53
|
+
except Exception as e:
|
|
54
|
+
out = {"error": f"{type(e).__name__}: {e}"}
|
|
55
|
+
print(json.dumps(out), flush=True)
|