nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,243 @@
1
+ """HumanEval's driver for the Nullius standard (JSON lines on stdin and stdout).
2
+
3
+ What HumanEval's authors decide - the problems' loader and the check of a
4
+ completion - is their code, human-eval at the tested commit
5
+ ($NULLIUS_UPSTREAM), imported, not copied. Both modules it calls import the
6
+ standard library only, so the driver runs on the machine's python3.
7
+
8
+ units - the problems' task ids, in the file's order (human_eval/data.py,
9
+ read_problems)
10
+ setup - writes solution.py (the problem's prompt: imports, signature,
11
+ docstring) into the workspace and asks for the body
12
+ score - the upstream's own check_correctness (human_eval/execution.py) with
13
+ the problem and a completion made of the agent's solution.py, at the
14
+ timeout evaluate_functional_correctness gives it by default (3 s):
15
+ the prompt, the completion and the problem's test run as one program,
16
+ in a process of its own under the upstream's reliability_guard. The
17
+ workspace files solution.py's import statements name are listed in
18
+ the detail (``not_tested``): the check is given solution.py alone
19
+ oracle - the reference control: writes the problem's prompt and its
20
+ canonical_solution, the authors' own body, as solution.py
21
+ answer - the original setup: the problem's prompt completed by a model in one
22
+ call through the unit's relay, as the paper had Codex complete it,
23
+ and the result written as solution.py for the same score. The
24
+ paper's models were never released and chat models take no raw
25
+ completion, so the prompt goes in a message asking for the code,
26
+ any model; the paper sampled at temperature 0.2 for pass@1 and 0.8
27
+ for pass@100, the temperature option (default: the provider's own)
28
+
29
+ The completion is what the upstream would be handed as the model's text after
30
+ the prompt (``completion`` below, the glue): when solution.py still begins with
31
+ the prompt exactly, the rest of the file; when the agent changed the prompt's
32
+ part (its docstring, its imports, a blank line), the whole file, after a line
33
+ break. The upstream then runs the prompt, whose function's body is its
34
+ docstring, and the agent's whole file after it, which defines the function
35
+ again: the agent's definition is the one the test calls.
36
+
37
+ The tests are never in the workspace while the agent works: the upstream
38
+ reads them from the problem when the score is taken.
39
+
40
+ """
41
+ import ast
42
+ import json
43
+ import os
44
+ import re
45
+ import shutil
46
+ import subprocess
47
+ import sys
48
+ import tempfile
49
+ import urllib.request
50
+
51
+ # The protocol keeps the driver's stdout; file descriptor 1 becomes stderr, so that the process the
52
+ # upstream runs a completion in - which inherits it - can never write into the protocol.
53
+ PROTOCOL = os.fdopen(os.dup(1), "w")
54
+ os.dup2(2, 1)
55
+ sys.stdout = sys.stderr
56
+
57
+ sys.path.insert(0, os.environ["NULLIUS_UPSTREAM"])
58
+ from human_eval.data import read_problems # noqa: E402 as published
59
+ from human_eval.execution import check_correctness # noqa: E402
60
+ from nullius.standard.container import CHECK_IMAGE, check_argv # noqa: E402
61
+
62
+ PROBLEMS = {}
63
+
64
+
65
+ def default_timeout():
66
+ """evaluate_functional_correctness's own default timeout. Its module
67
+ imports numpy and tqdm, which this driver's interpreter has not, so the
68
+ value is read from its source at the tested commit, not imported."""
69
+ path = os.path.join(os.environ["NULLIUS_UPSTREAM"], "human_eval", "evaluation.py")
70
+ with open(path) as f:
71
+ tree = ast.parse(f.read())
72
+ for node in ast.walk(tree):
73
+ if isinstance(node, ast.FunctionDef) and node.name == "evaluate_functional_correctness":
74
+ args = node.args.args[len(node.args.args) - len(node.args.defaults):]
75
+ for a, d in zip(args, node.args.defaults):
76
+ if a.arg == "timeout":
77
+ return float(ast.literal_eval(d))
78
+ raise RuntimeError("human-eval's evaluate_functional_correctness no longer has a timeout default where the driver reads it")
79
+
80
+
81
+ TIMEOUT = default_timeout()
82
+
83
+
84
+ def load():
85
+ if not PROBLEMS:
86
+ PROBLEMS.update(read_problems(os.environ["NULLIUS_DATASET"]))
87
+ return PROBLEMS
88
+
89
+
90
+ TEMPERATURE = os.environ.get("NULLIUS_OPTION_TEMPERATURE", "default")
91
+ FENCE = re.compile(r"```[A-Za-z0-9_+-]*\n(.*?)```", re.S)
92
+
93
+
94
+ def units(dataset, setup=None):
95
+ return {"units": list(load())}
96
+
97
+
98
+ def code_of(text):
99
+ """The code in a reply: its first fenced block, else the reply itself."""
100
+ m = FENCE.search(text)
101
+ return (m.group(1) if m else text).strip("\n") + "\n"
102
+
103
+
104
+ def answer(unit, dir, setup, model, base_url, key):
105
+ """The original: the prompt completed in one call, written as solution.py. A reply that defines the
106
+ function is the whole file; one that gives the body alone is appended to the prompt, as the paper did."""
107
+ if setup != "original":
108
+ raise ValueError(f"HumanEval has no {setup} setup")
109
+ p = load()[unit]
110
+ ask = ("Complete the following Python function so it does what its docstring says. Reply with the completed "
111
+ "code only, in one code block: the imports and the function with its body, using only the standard "
112
+ f"library.\n\n```python\n{p['prompt']}```")
113
+ body = {"model": model, "messages": [{"role": "user", "content": ask}]}
114
+ if TEMPERATURE != "default":
115
+ body["temperature"] = float(TEMPERATURE)
116
+ req = urllib.request.Request(base_url.rstrip("/") + "/chat/completions", data=json.dumps(body).encode(),
117
+ headers={"Content-Type": "application/json", "Authorization": "Bearer " + key})
118
+ with urllib.request.urlopen(req, timeout=600) as r:
119
+ reply = json.load(r)
120
+ text = ((reply.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
121
+ if not text.strip():
122
+ raise RuntimeError("the reply had no text")
123
+ code = code_of(text)
124
+ whole = re.search(rf"^\s*def\s+{re.escape(p['entry_point'])}\s*\(", code, re.M) is not None
125
+ with open(os.path.join(dir, "solution.py"), "w") as f:
126
+ f.write(code if whole else p["prompt"] + code)
127
+ return {"answer": text, "model": model}
128
+
129
+
130
+ def setup(unit, dir):
131
+ p = load()[unit]
132
+ with open(os.path.join(dir, "solution.py"), "w") as f:
133
+ f.write(p["prompt"])
134
+ return {"prompt": (f"The file solution.py in the current working directory holds the signature and docstring of the "
135
+ f"Python function `{p['entry_point']}`, with no body. Complete the function in solution.py so it does "
136
+ f"what its docstring says. Keep its name and signature; use only Python's standard library. Hidden tests "
137
+ f"will run solution.py and call `{p['entry_point']}`.")}
138
+
139
+
140
+ def completion(problem, text):
141
+ """The text the upstream appends to the prompt, and how it was taken."""
142
+ if text.startswith(problem["prompt"]):
143
+ return text[len(problem["prompt"]):], "after the prompt"
144
+ return "\n" + text, "the whole file: the prompt's part was changed"
145
+
146
+
147
+ LISTED_UP_TO = 1 << 20 # characters: a larger solution.py cannot run within the check's time, and is not read for names
148
+
149
+
150
+ def not_tested(dir, text):
151
+ """The workspace files solution.py's import statements name. The check is given solution.py alone, as the
152
+ upstream is given one completion, so a solution split over two files fails on its import: listed so that
153
+ the failure reads as what it is. Read from the file's syntax, nothing run; none when it does not parse."""
154
+ if len(text) > LISTED_UP_TO:
155
+ return []
156
+ try:
157
+ tree = ast.parse(text)
158
+ except (SyntaxError, ValueError):
159
+ return []
160
+ names = set()
161
+ for node in ast.walk(tree):
162
+ if isinstance(node, ast.Import):
163
+ names.update(a.name.split(".")[0] for a in node.names)
164
+ elif isinstance(node, ast.ImportFrom):
165
+ if node.module:
166
+ names.add(node.module.split(".")[0])
167
+ elif node.level: # from . import helpers
168
+ names.update(a.name for a in node.names)
169
+ found = []
170
+ for name in sorted(names - {"solution"}):
171
+ path = os.path.join(dir, name)
172
+ if os.path.lexists(path + ".py"):
173
+ found.append(name + ".py")
174
+ elif os.path.islink(path) or os.path.isdir(path): # a package; a link is named, never followed
175
+ found.append(name + "/")
176
+ return found
177
+
178
+
179
+ def score(unit, dir, transcript, answer, setup=None, **_):
180
+ p = load()[unit]
181
+ path = os.path.join(dir, "solution.py")
182
+ if os.path.islink(path) or not os.path.isfile(path): # a link is not followed out of the workspace
183
+ return {"reward": 0.0, "passed": False, "detail": {"why": "no solution.py"}}
184
+ with open(path, errors="replace") as f:
185
+ text = f.read()
186
+ done, how = completion(p, text)
187
+ r = check_in_container(dir, p, done) if os.environ.get(CHECK_IMAGE) else check_correctness(p, done, TIMEOUT)
188
+ ok = bool(r["passed"])
189
+ try:
190
+ left = not_tested(dir, text)
191
+ except Exception: # the names explain the verdict, never replace it
192
+ left = []
193
+ return {"reward": 1.0 if ok else 0.0, "passed": ok,
194
+ "detail": {"check": "container" if os.environ.get(CHECK_IMAGE) else "this machine",
195
+ "result": r["result"], "timeout": TIMEOUT, "completion": how, "solution.py": text[-4000:],
196
+ **({"not_tested": left} if left else {})}}
197
+
198
+
199
+ # Run in the unit's image: the upstream's check_correctness, imported from its own files copied in.
200
+ CHECK = ("import json, sys\nsys.path.insert(0, sys.argv[1])\nfrom human_eval.execution import check_correctness\n"
201
+ "a = json.load(open(sys.argv[1] + '/check.json'))\nprint(json.dumps(check_correctness(a['problem'], a['completion'], a['timeout'])))\n")
202
+
203
+
204
+ def check_in_container(dir, problem, done):
205
+ """check_correctness in a throwaway container of the unit's image: a fresh
206
+ folder in the workspace (the agent has ended) holds the upstream's
207
+ human_eval package and the problem, and goes after."""
208
+ here = tempfile.mkdtemp(prefix=".nullius-check-", dir=dir)
209
+ try:
210
+ os.makedirs(os.path.join(here, "human_eval"))
211
+ for name in ("__init__.py", "execution.py"):
212
+ src = os.path.join(os.environ["NULLIUS_UPSTREAM"], "human_eval", name)
213
+ if os.path.isfile(src):
214
+ shutil.copyfile(src, os.path.join(here, "human_eval", name))
215
+ with open(os.path.join(here, "check.json"), "w") as f:
216
+ json.dump({"problem": problem, "completion": done, "timeout": TIMEOUT}, f)
217
+ argv = check_argv(dir, [], ["python3", "-I", "-c", CHECK, here])
218
+ r = subprocess.run(argv, capture_output=True, text=True, timeout=TIMEOUT + 120)
219
+ lines = r.stdout.strip().splitlines()
220
+ if r.returncode != 0 or not lines:
221
+ raise RuntimeError(f"the check's container: exit {r.returncode}: {(r.stderr or r.stdout).strip()[-300:]}")
222
+ return json.loads(lines[-1])
223
+ finally:
224
+ shutil.rmtree(here, ignore_errors=True)
225
+
226
+
227
+ def oracle(unit, dir):
228
+ p = load()[unit]
229
+ with open(os.path.join(dir, "solution.py"), "w") as f:
230
+ f.write(p["prompt"] + p["canonical_solution"])
231
+ return {"answer": ""}
232
+
233
+
234
+ HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score, "oracle": oracle}
235
+
236
+ if __name__ == "__main__": # check_correctness starts a process: where it re-imports this module, nothing is served
237
+ for line in sys.stdin:
238
+ try:
239
+ req = json.loads(line)
240
+ out = HANDLERS[req.pop("op")](**req)
241
+ except Exception as e:
242
+ out = {"error": f"{type(e).__name__}: {e}"}
243
+ print(json.dumps(out), file=PROTOCOL, flush=True)
@@ -0,0 +1,115 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "humaneval",
4
+ "label": "HumanEval",
5
+ "summary": "Each problem is a workspace holding solution.py - the function's signature and docstring; the agent completes it, and the problem's own tests run against it.",
6
+ "family": "agent",
7
+ "needs": [
8
+ "agent.workspace/v1"
9
+ ],
10
+ "upstream": {
11
+ "repository": "https://github.com/openai/human-eval",
12
+ "tested": "6d43fb980f9fee3c892a914eda09951f772ad10d",
13
+ "tested_date": "2025-01-17",
14
+ "license": "MIT"
15
+ },
16
+ "driver": {
17
+ "command": [
18
+ "python3",
19
+ "{here}/driver.py"
20
+ ]
21
+ },
22
+ "datasets": [
23
+ {
24
+ "id": "humaneval",
25
+ "label": "HumanEval",
26
+ "path": "data/HumanEval.jsonl.gz",
27
+ "sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
28
+ "units": 164
29
+ }
30
+ ],
31
+ "units": {
32
+ "noun": "problem",
33
+ "nouns": "problems"
34
+ },
35
+ "controls": {
36
+ "nop": {
37
+ "source": "nullius",
38
+ "from": "Nullius: no agent runs; the answer is empty and the workspace is left as setup made it."
39
+ },
40
+ "oracle": {
41
+ "source": "official",
42
+ "from": "The problem's canonical_solution field, the authors' own body, written after its prompt as solution.py."
43
+ }
44
+ },
45
+ "score": {
46
+ "kind": "pass",
47
+ "runs_code": true,
48
+ "says": "the upstream's own check_correctness (human_eval/execution.py) passes: the problem's prompt, the completion (the agent's solution.py after the prompt, or the whole file when the agent changed the prompt's part) and the problem's test run as one program, in a process of its own under the upstream's reliability_guard, 3 s at most (its evaluate_functional_correctness default)"
49
+ },
50
+ "setups": {
51
+ "original": {
52
+ "summary": "The problem's prompt - its imports, signature and docstring - completed by the model in one call, as the paper had Codex complete it, and checked by the upstream's own check_correctness, as published. The paper's models were never released and a chat model takes no raw completion: the prompt goes in a message asking for the code, any model in their place.",
53
+ "providers": {
54
+ "openai": "",
55
+ "openrouter": "",
56
+ "custom": ""
57
+ },
58
+ "turn_seconds": 600,
59
+ "paper": "the paper's pass@k (table 1), which no served model reproduces",
60
+ "comparable": false
61
+ }
62
+ },
63
+ "options": {
64
+ "temperature": {
65
+ "summary": "Sampling randomness. Provider default sends no temperature; enter a non-negative decimal to override it. Accepted values depend on the model.",
66
+ "default": "default",
67
+ "setups": [
68
+ "original"
69
+ ],
70
+ "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?"
71
+ }
72
+ },
73
+ "limits": {
74
+ "turn_seconds": 600
75
+ },
76
+ "proven": {
77
+ "humaneval": {
78
+ "units": {
79
+ "units": 164,
80
+ "passed": "164 ids, the same twice, unique",
81
+ "on": "2026-10-13"
82
+ },
83
+ "leak": {
84
+ "units": 164,
85
+ "looked_for": 163,
86
+ "short": 1,
87
+ "passed": "164/164",
88
+ "note": "1 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
89
+ "on": "2026-10-13"
90
+ },
91
+ "nop": {
92
+ "passed": "0/164",
93
+ "value": 0.0,
94
+ "expect": {
95
+ "at_most": 0.0
96
+ },
97
+ "units": 164,
98
+ "job": "2026-10-13__14-46-10__humaneval-2__humaneval__prove-nop",
99
+ "source": "nullius",
100
+ "on": "2026-10-13"
101
+ },
102
+ "oracle": {
103
+ "passed": "164/164",
104
+ "value": 1.0,
105
+ "expect": {
106
+ "at_least": 1.0
107
+ },
108
+ "units": 164,
109
+ "job": "2026-10-13__14-46-10__humaneval-2__humaneval__prove-oracle",
110
+ "source": "official",
111
+ "on": "2026-10-13"
112
+ }
113
+ }
114
+ }
115
+ }
@@ -0,0 +1,72 @@
1
+ [
2
+ {
3
+ "entry": "evaluate_functional_correctness",
4
+ "setting": "--problem_file",
5
+ "values": "data/HumanEval.jsonl.gz; data/example_problem.jsonl (a sanity fixture)",
6
+ "default": "HumanEval.jsonl.gz",
7
+ "nullius": "dataset:humaneval"
8
+ },
9
+ {
10
+ "entry": "evaluate_functional_correctness",
11
+ "setting": "--k",
12
+ "values": "1,10,100",
13
+ "default": "1,10,100",
14
+ "nullius": "fixed:pass@1 to pass@n with n the run's attempts (cli run --repeat n, the builder's Attempts per problem) - the job's result.json reports it, 1 - C(n-c, k)/C(n, k) as estimate_pass_at_k reads it; one attempt by default, pass@1"
15
+ },
16
+ {
17
+ "entry": "README",
18
+ "setting": "num_samples_per_task",
19
+ "values": "200",
20
+ "default": "200",
21
+ "nullius": "fixed:the run's attempts (--repeat), 1 by default - 200 agent runs per problem are offered, never assumed"
22
+ },
23
+ {
24
+ "entry": "evaluate_functional_correctness",
25
+ "setting": "--timeout",
26
+ "values": "seconds",
27
+ "default": "3.0",
28
+ "nullius": "fixed:3.0 - the upstream's default, read from its source"
29
+ },
30
+ {
31
+ "entry": "evaluate_functional_correctness",
32
+ "setting": "--n_workers",
33
+ "values": "threads",
34
+ "default": "4",
35
+ "nullius": "fixed:the run's concurrency - how many problems at once, not what is measured"
36
+ },
37
+ {
38
+ "entry": "human_eval/execution.py",
39
+ "setting": "reliability_guard (maximum_memory_bytes)",
40
+ "values": "None",
41
+ "default": "None",
42
+ "nullius": "fixed:None - the upstream's check_correctness, called as published"
43
+ },
44
+ {
45
+ "entry": "README",
46
+ "setting": "generate_one_completion (the answerer)",
47
+ "values": "a stub the user writes",
48
+ "default": "",
49
+ "nullius": "setup:original"
50
+ },
51
+ {
52
+ "entry": "paper",
53
+ "setting": "the paper's models (Codex 12M to 12B)",
54
+ "values": "",
55
+ "default": "",
56
+ "nullius": "not offered - never released; the Codex API they were served by is retired"
57
+ },
58
+ {
59
+ "entry": "paper",
60
+ "setting": "sampling (temperature 0.2 for pass@1, 0.8 for pass@100, top_p 0.95, stop sequences)",
61
+ "values": "",
62
+ "default": "",
63
+ "nullius": "option:temperature"
64
+ },
65
+ {
66
+ "entry": "driver (ours)",
67
+ "setting": "completion glue",
68
+ "values": "the agent's solution.py after the prompt, else the whole file",
69
+ "default": "",
70
+ "nullius": "fixed:Nullius's - the upstream reads a completion string; said in the limitations"
71
+ }
72
+ ]
@@ -0,0 +1,43 @@
1
+ {
2
+ "key": "locomo",
3
+ "title": "LoCoMo",
4
+ "category": "Memory",
5
+ "domain": "Very long-term conversational memory over conversations of 19 to 32 sessions.",
6
+ "scoring": "Token F1",
7
+ "harness": [
8
+ "standard"
9
+ ],
10
+ "origin": "official",
11
+ "good_to_know": [
12
+ "Each conversation is one task, with its questions inside. The smallest run is one conversation.",
13
+ "The score is the mean token F1 over the questions, as in the paper. There is no pass or fail.",
14
+ "What goes into the memory is an option: the conversation itself by default, or the authors' released facts or session summaries."
15
+ ],
16
+ "upstream_limitations": [
17
+ "The 446 adversarial questions are left out, because the published file has no answer for them. 1,540 of the 1,986 questions remain.",
18
+ "The reader's model name must contain gpt-4 or gpt-3.5, as LoCoMo's own code requires. Any other model fails every question.",
19
+ "The paper's readers, gpt-4-turbo and gpt-3.5-turbo, now point to newer models. Scores can differ from the paper's for that reason.",
20
+ "The reader gets 32 tokens per answer. A reasoning model can spend them all reasoning, and that question fails.",
21
+ "One unreadable reply stops the whole conversation, because LoCoMo's script writes its answers only at the end.",
22
+ "Original memory runs the paper's retrieval setup and needs a CUDA GPU. It writes facts and summaries anew each run, so they differ from the authors' released ones.",
23
+ "Contriever, LoCoMo's default retriever, and the OpenAI retriever fail as published, so they are not offered."
24
+ ],
25
+ "limitations": [
26
+ "The paper's other readers are not offered. Claude 3 Sonnet and Gemini Pro 1.0 are retired, and the Llama 3 and Mistral readers run only as local weights."
27
+ ],
28
+ "datasets": [
29
+ {
30
+ "key": "locomo10",
31
+ "title": "LoCoMo-10",
32
+ "tasks": 10,
33
+ "official": "https://github.com/snap-research/locomo",
34
+ "source": "GitHub",
35
+ "registry": "",
36
+ "hub": "",
37
+ "verifier": "LoCoMo's own task_eval/evaluate_qa.py: the token F1 of each answer against the reference; no judge model.",
38
+ "notes": "Ten conversations. CC BY-NC 4.0."
39
+ }
40
+ ],
41
+ "citation": "@article{maharana2024evaluating,\n title={Evaluating very long-term conversational memory of llm agents},\n author={Maharana, Adyasha and Lee, Dong-Ho and Tulyakov, Sergey and Bansal, Mohit and Barbieri, Francesco and Fang, Yuwei},\n journal={arXiv preprint arXiv:2402.17753},\n year={2024}\n}",
42
+ "citation_source": "https://github.com/snap-research/locomo"
43
+ }