nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,241 @@
1
+ """The verifier's own report, read as the benchmark's entry describes it.
2
+
3
+ Everything else this app reads is Harbor's and the same for every benchmark:
4
+ ``result.json``, ``config.json``, ``exception_info``, ``trace.json``, the ATIF
5
+ trajectory and ``verifier_result.rewards.reward``. The verifier's report is the
6
+ exception - one benchmark writes a ``report.json``, another a CTRF
7
+ ``ctrf.json`` - and its keys are the benchmark's. So no code here names them:
8
+ a benchmark's ``benchmark.json`` carries a ``report`` block, and this module
9
+ reads any report by it::
10
+
11
+ "report": {
12
+ "sources": [...], the dataset names a Harbor job records for it
13
+ "file": "verifier/report.json",
14
+ "required": true, no file: the verifier never ran (else the
15
+ reward file stands in for it)
16
+ "under_task": true, the document is keyed by the task's name
17
+ "succeeded": "resolved", a field, or "reward": Harbor's 1.0
18
+ "produced": "patch_exists", "produced_unless": "patch_is_None",
19
+ "usable": "patch_successfully_applied",
20
+ "gained": {"pass": [paths], "fail": [paths]}, lists of check names
21
+ "held": {...}, the same, or
22
+ "gained": {"tests": path, "name": "name", "status": "status",
23
+ "pass": "passed", "fail": "failed",
24
+ "count_pass": path, "count_fail": path}, a list of tests
25
+ "words": {"gained": "Fail → Pass", "gained_abbr", "gained_term",
26
+ "held", ..., "has_patch": true}
27
+ }
28
+
29
+ A path is dotted (``tests_status.FAIL_TO_PASS.success``). *gained* are the
30
+ checks that had to start passing, *held* the ones that had to keep passing: a
31
+ benchmark with one pass or fail leaves ``held`` out. A benchmark with no block
32
+ still parses: the reward is all that is read, and its pages show fewer
33
+ columns. A field the report cannot answer is ``None``, never a plausible zero.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import json
39
+ import os
40
+ from dataclasses import dataclass
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class Verdict:
45
+ """What a verifier said, in terms no single benchmark owns.
46
+
47
+ `None` means *this benchmark does not report that*, and is distinct from
48
+ `False` / `0`, which mean it reported the absence.
49
+ """
50
+
51
+ succeeded: bool | None = None
52
+ produced: bool | None = None # did the agent emit a product at all?
53
+ usable: bool | None = None # did that product apply / run?
54
+ gained_pass: int = 0
55
+ gained_fail: int = 0
56
+ held_pass: int = 0
57
+ held_fail: int = 0
58
+ gained_failures: tuple[str, ...] = ()
59
+ held_failures: tuple[str, ...] = ()
60
+
61
+
62
+ UNREAD = Verdict()
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class Vocabulary:
67
+ """The benchmark's own names for its checks on screen (``report.words``).
68
+
69
+ `None` means the benchmark has no such category, and the pages leave the
70
+ row or column out rather than dash it under a foreign label. `has_patch`
71
+ gates the "Patch applied" card the same way.
72
+ """
73
+
74
+ gained: str | None = None # card label, e.g. "Fail → Pass" / "Tests"
75
+ gained_abbr: str = "" # table-column form; empty falls back to `gained`
76
+ gained_term: str = "" # the dataset's jargon name, when it has one
77
+ held: str | None = None
78
+ held_abbr: str = ""
79
+ held_term: str = ""
80
+ has_patch: bool = False
81
+
82
+
83
+ # What a benchmark without a report block gets: no named categories, so the
84
+ # check columns and patch cards are simply not drawn.
85
+ GENERIC_VOCABULARY = Vocabulary()
86
+
87
+
88
+ def reward_is_success(reward: float | None) -> bool | None:
89
+ """Harbor's convention: 1.0 is a solve. `None` in, `None` out."""
90
+ return None if reward is None else reward >= 1.0
91
+
92
+
93
+ def _read_json(path: str):
94
+ try:
95
+ with open(path, encoding="utf-8", errors="replace") as fh:
96
+ return json.load(fh)
97
+ except (OSError, json.JSONDecodeError, UnicodeDecodeError):
98
+ return None
99
+
100
+
101
+ def reward_marker(trial_dir: str):
102
+ """`{}` when the verifier left its reward file, `None` when it left nothing.
103
+
104
+ `verifier/reward.txt`, or a `verifier/reward.json` object with a numeric
105
+ `reward` (harbor-rewardkit): either is evidence the verifier ran. The empty
106
+ dict is deliberately not `None`: the verifier spoke and said no more than
107
+ the reward.
108
+ """
109
+ verifier = os.path.join(trial_dir, "verifier")
110
+ if os.path.isfile(os.path.join(verifier, "reward.txt")):
111
+ return {}
112
+ doc = _read_json(os.path.join(verifier, "reward.json"))
113
+ if isinstance(doc, dict) and isinstance(doc.get("reward"), (int, float)) \
114
+ and not isinstance(doc.get("reward"), bool):
115
+ return {}
116
+ return None
117
+
118
+
119
+
120
+ _CACHE: dict = {}
121
+
122
+
123
+ def _blocks() -> dict:
124
+ """Every ``report`` block in the catalogue, by each source name it lists."""
125
+ from nullius.catalogue import catalog
126
+ sig = catalog.signature()
127
+ if _CACHE.get("sig") != sig:
128
+ table = {}
129
+ for b in catalog.load().benchmarks:
130
+ if b.report:
131
+ for name in b.report.get("sources") or [b.key]:
132
+ table[name.strip().lower()] = (b.key, b.report)
133
+ _CACHE.update(sig=sig, table=table)
134
+ return _CACHE["table"]
135
+
136
+
137
+ def block_for(source: str | None):
138
+ """``(benchmark key, report block)`` for *source*, or ``None``."""
139
+ return _blocks().get((source or "").strip().lower())
140
+
141
+
142
+ def _dig(doc, path: str):
143
+ for part in path.split("."):
144
+ if not isinstance(doc, dict):
145
+ return None
146
+ doc = doc.get(part)
147
+ return doc
148
+
149
+
150
+ def _names(doc, paths) -> list:
151
+ out = []
152
+ for p in paths or []:
153
+ got = _dig(doc, p)
154
+ if isinstance(got, list):
155
+ out += [str(x) for x in got]
156
+ return out
157
+
158
+
159
+ def _checks(doc, spec) -> tuple[int, int, tuple]:
160
+ """``(passed, failed, failing names)`` of one category."""
161
+ if not spec:
162
+ return 0, 0, ()
163
+ if "tests" in spec: # a list of test objects with a status
164
+ tests = [t for t in (_dig(doc, spec["tests"]) or []) if isinstance(t, dict)]
165
+ name, status = spec.get("name", "name"), spec.get("status", "status")
166
+ failing = tuple(str(t.get(name, "")) for t in tests if t.get(status) == spec.get("fail", "failed"))
167
+ # the producer's own counts when it gives them, as trace totals are taken
168
+ passed = _dig(doc, spec["count_pass"]) if spec.get("count_pass") else None
169
+ failed = _dig(doc, spec["count_fail"]) if spec.get("count_fail") else None
170
+ if passed is None and failed is None and tests:
171
+ passed = sum(1 for t in tests if t.get(status) == spec.get("pass", "passed"))
172
+ failed = len(failing)
173
+ return int(passed or 0), int(failed or 0), failing
174
+ ok, bad = _names(doc, spec.get("pass")), _names(doc, spec.get("fail"))
175
+ return len(ok), len(bad), tuple(bad)
176
+
177
+
178
+ def _read_block(spec: dict, report, reward) -> Verdict:
179
+ if not isinstance(report, dict):
180
+ return Verdict()
181
+ how = spec.get("succeeded", "reward")
182
+ succeeded = reward_is_success(reward) if how == "reward" else bool(_dig(report, how))
183
+ produced = usable = None
184
+ if spec.get("produced"):
185
+ produced = bool(_dig(report, spec["produced"])) and not (
186
+ spec.get("produced_unless") and _dig(report, spec["produced_unless"]))
187
+ # only meaningful once something was produced
188
+ usable = bool(_dig(report, spec["usable"])) if spec.get("usable") and produced else None
189
+ gp, gf, gfail = _checks(report, spec.get("gained"))
190
+ hp, hf, hfail = _checks(report, spec.get("held"))
191
+ return Verdict(succeeded=succeeded, produced=produced, usable=usable,
192
+ gained_pass=gp, gained_fail=gf, held_pass=hp, held_fail=hf,
193
+ gained_failures=gfail, held_failures=hfail)
194
+
195
+
196
+ # --- what the rest of the app calls ------------------------------------------
197
+
198
+ def vocabulary_for(source: str | None) -> Vocabulary:
199
+ """How *source* names its checks, or the empty vocabulary. Never raises."""
200
+ found = block_for(source)
201
+ if not found:
202
+ return GENERIC_VOCABULARY
203
+ words = found[1].get("words") or {}
204
+ return Vocabulary(**{k: v for k, v in words.items() if k in Vocabulary.__dataclass_fields__})
205
+
206
+
207
+ def load(source: str | None, trial_dir: str, task_base: str):
208
+ """Find *source*'s verifier report inside *trial_dir*.
209
+
210
+ Without a block: a ``verifier/report.json`` if there is one, else the
211
+ reward marker. Returns `None` when the verifier left nothing at all, which
212
+ `read()` refuses to score.
213
+ """
214
+ found = block_for(source)
215
+ if not found:
216
+ report = _read_json(os.path.join(trial_dir, "verifier", "report.json"))
217
+ return report if isinstance(report, dict) else reward_marker(trial_dir)
218
+ spec = found[1]
219
+ report = _read_json(os.path.join(trial_dir, spec.get("file", "verifier/report.json")))
220
+ if not isinstance(report, dict):
221
+ return None if spec.get("required") else reward_marker(trial_dir)
222
+ if spec.get("under_task"):
223
+ inner = report.get(task_base)
224
+ if inner is None and len(report) == 1:
225
+ inner = next(iter(report.values()))
226
+ return inner
227
+ return report
228
+
229
+
230
+ def read(source: str | None, report, reward: float | None) -> Verdict:
231
+ """Turn a verifier report into a `Verdict`, however little is understood.
232
+
233
+ No report means the verifier never spoke, which is not the same as it
234
+ saying no — so `succeeded` stays `None` and the reward is *not* consulted.
235
+ """
236
+ if report is None:
237
+ return UNREAD
238
+ found = block_for(source)
239
+ if not found:
240
+ return Verdict(succeeded=reward_is_success(reward))
241
+ return _read_block(found[1], report, reward)
nullius/sandbox.py ADDED
@@ -0,0 +1,109 @@
1
+ """The Settings page's Sandbox: the container backend's engine on this machine,
2
+ docker or podman (``container.DOCKER``), and its Docker Hub login under Data
3
+ providers.
4
+
5
+ Two things, each something a user acts on:
6
+
7
+ - **Engine**: whether it answers, and its name and version (``container.unusable``).
8
+ - **Registry login**: Docker Hub limits anonymous pulls, which stops a
9
+ benchmark with an image per unit part-way. The username and an access token
10
+ go to ``docker login --password-stdin`` (``podman login ... docker.io``)
11
+ once; the engine keeps them in its own store (``~/.docker/config.json`` or
12
+ the credential helper it names; podman's ``auth.json``), the same as a
13
+ login typed in a terminal. Nullius stores neither, and reads back only the
14
+ username.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import base64
19
+ import json
20
+ import os
21
+ import re
22
+ import subprocess
23
+ from typing import List, Optional
24
+
25
+ from nullius.standard import container
26
+
27
+ REGISTRY = "https://index.docker.io/v1/" # Docker Hub, as docker names it in its config
28
+ HUB = "docker.io" # as podman names it: without it, podman logs in to its first search registry
29
+ _USER = re.compile(r"^[a-z0-9][a-z0-9._-]{1,63}$")
30
+ _TOKEN_MAX = 4096
31
+
32
+
33
+ def docker_state() -> dict:
34
+ """{ok, version} when the engine answers - its name and version, ``podman
35
+ 6.1.4`` - else {ok: False, error}."""
36
+ why = container.unusable()
37
+ if why:
38
+ return {"ok": False, "error": why}
39
+ r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
40
+ return {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"}
41
+
42
+
43
+ def _config_dir() -> str:
44
+ return os.environ.get("DOCKER_CONFIG") or os.path.join(os.path.expanduser("~"), ".docker")
45
+
46
+
47
+ def login_user(registry: str = REGISTRY) -> Optional[str]:
48
+ """The username the engine holds a Docker Hub login for, or None. Read from
49
+ docker's config, or asked of the credential helper it names; the secret
50
+ that comes with it is dropped here, never kept or returned. Podman says the
51
+ username itself."""
52
+ if container.podman():
53
+ try:
54
+ r = container._docker(["login", "--get-login", HUB], timeout=30, check=False)
55
+ except container.Unavailable:
56
+ return None
57
+ return (r.stdout.strip() or None) if r.returncode == 0 else None
58
+ try:
59
+ with open(os.path.join(_config_dir(), "config.json")) as f:
60
+ config = json.load(f)
61
+ except (OSError, ValueError):
62
+ return None
63
+ auth = (config.get("auths") or {}).get(registry) or {}
64
+ if auth.get("auth"):
65
+ try:
66
+ return base64.b64decode(auth["auth"]).decode().split(":", 1)[0] or None
67
+ except (ValueError, UnicodeDecodeError):
68
+ return None
69
+ helper = (config.get("credHelpers") or {}).get(registry) or config.get("credsStore")
70
+ if not helper or not re.match(r"^[A-Za-z0-9_.-]+$", helper):
71
+ return None
72
+ try:
73
+ r = subprocess.run(["docker-credential-" + helper, "get"], input=registry, text=True,
74
+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=15)
75
+ return (json.loads(r.stdout).get("Username") or None) if r.returncode == 0 else None
76
+ except (OSError, ValueError, subprocess.TimeoutExpired):
77
+ return None
78
+
79
+
80
+ def login(username: str, token: str) -> dict:
81
+ """``docker login`` (``podman login ... docker.io``) to Docker Hub, the token over stdin: {ok, user} or
82
+ {ok: False, error}. Neither value is stored or logged here."""
83
+ username = (username or "").strip().lower()
84
+ if not _USER.match(username):
85
+ return {"ok": False, "error": "Enter your Docker Hub username."}
86
+ if not token or len(token) > _TOKEN_MAX or "\n" in token.strip():
87
+ return {"ok": False, "error": "Enter an access token."}
88
+ try:
89
+ r = container._docker(["login", "--username", username, "--password-stdin", *_hub()], input=token.strip() + "\n",
90
+ timeout=60, check=False)
91
+ except container.Unavailable as e:
92
+ return {"ok": False, "error": str(e)}
93
+ if r.returncode != 0:
94
+ said = [ln for ln in (r.stderr or r.stdout).strip().splitlines() if ln.strip()]
95
+ return {"ok": False, "error": (said[-1] if said else f"{container.DOCKER} login: exit {r.returncode}")[:300]}
96
+ return {"ok": True, "user": login_user() or username}
97
+
98
+
99
+ def _hub() -> List[str]:
100
+ return [HUB] if container.podman() else []
101
+
102
+
103
+ def logout() -> dict:
104
+ try:
105
+ r = container._docker(["logout", *_hub()], timeout=30, check=False)
106
+ except container.Unavailable as e:
107
+ return {"ok": False, "error": str(e)}
108
+ return {"ok": r.returncode == 0} if r.returncode == 0 else {"ok": False, "error": (r.stderr or r.stdout).strip()[-300:]}
109
+
nullius/secrets.py ADDED
@@ -0,0 +1,218 @@
1
+ """A benchmark's licences: the secrets its environments need, kept per
2
+ benchmark outside the repository.
3
+
4
+ A licence belongs to the software of one benchmark's dataset, so it is kept
5
+ under that benchmark: ``secrets/<benchmark>/<NAME>`` in the user configuration directory
6
+ (``SECRETS_DIR`` moves the root). Two benchmarks may each ask for a
7
+ ``LICENSE_KEY``; they are two files in two folders and never mix. Each
8
+ folder is 0700 and each entry a 0600 file holding the bytes it was given -
9
+ a licence file keeps its bytes, a key or a server address its text - so
10
+ nothing of it is ever in the settings file or in this process's environment.
11
+ A data provider's token (Settings → Data providers) has a folder of its own
12
+ (``PROVIDERS``): a credential declared with that ``provider`` is read from
13
+ there alone, never from the benchmark's (:func:`store_of`).
14
+
15
+ ``index.json`` beside them says, per name, when it was set, when it expires
16
+ (when said), its kind and the file it came from - never the value. A page
17
+ reads :func:`status` or :func:`state`; only a run reads :func:`value` or
18
+ :func:`path`, when a unit starts (``standard/run.py``), or to send it to a
19
+ host over stdin (``execution``). Standard library only.
20
+ """
21
+ from __future__ import annotations
22
+
23
+ import base64
24
+ import json
25
+ import os
26
+ import re
27
+ import tempfile
28
+ import time
29
+ from typing import Dict, Optional, Tuple, Union
30
+ from nullius import paths
31
+
32
+ KINDS = ("value", "file", "server")
33
+ INDEX = "index.json"
34
+ # Shared data-provider credentials remain in the owner-only secret store.
35
+ HUGGINGFACE = "data-provider-huggingface"
36
+ # The data providers a benchmark's credential may name (``provider`` in its ``needs_secrets``):
37
+ # the folder each one's token is kept in, and its name there.
38
+ PROVIDERS = {"huggingface": (HUGGINGFACE, "HF_TOKEN")}
39
+ _BENCH = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,79}$")
40
+ _NAME = re.compile(r"^[A-Z_][A-Z0-9_]{0,63}$")
41
+ _DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
42
+ # A licence server: a host name or an IPv4 address, and its port.
43
+ _SERVER = re.compile(r"^(?=.{1,253}:)[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?:([0-9]{1,5})$")
44
+ MAX_BYTES = 1_000_000 # a licence file is small; anything larger is not one
45
+
46
+
47
+ def root(env=os.environ) -> str:
48
+ return paths.secrets_dir(env)
49
+
50
+
51
+ def _folder(bench: str, make: bool = False) -> str:
52
+ if not isinstance(bench, str) or not _BENCH.match(bench):
53
+ raise ValueError(f"{bench!r} is not a benchmark's key")
54
+ d = os.path.join(root(), bench)
55
+ if make:
56
+ os.makedirs(root(), mode=0o700, exist_ok=True)
57
+ os.makedirs(d, mode=0o700, exist_ok=True)
58
+ os.chmod(root(), 0o700)
59
+ os.chmod(d, 0o700)
60
+ return d
61
+
62
+
63
+ def _name(name: str) -> str:
64
+ if not isinstance(name, str) or not _NAME.match(name):
65
+ raise ValueError(f"{name!r} is not a licence's name (capitals, digits, underscores)")
66
+ return name
67
+
68
+
69
+ def _write(path: str, data: bytes) -> None:
70
+ """Written whole under a temporary name beside it, 0600 from its first
71
+ byte, then renamed over the old one."""
72
+ fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=os.path.dirname(path))
73
+ try:
74
+ os.fchmod(fd, 0o600)
75
+ with os.fdopen(fd, "wb") as f:
76
+ f.write(data)
77
+ os.replace(tmp, path)
78
+ except BaseException:
79
+ try:
80
+ os.unlink(tmp)
81
+ except OSError:
82
+ pass
83
+ raise
84
+
85
+
86
+ def _index(bench: str) -> Dict[str, dict]:
87
+ try:
88
+ with open(os.path.join(_folder(bench), INDEX)) as f:
89
+ got = json.load(f)
90
+ return got if isinstance(got, dict) else {}
91
+ except (OSError, ValueError):
92
+ return {}
93
+
94
+
95
+ def problem(kind: str, value: Union[str, bytes]) -> Optional[str]:
96
+ """Why ``value`` cannot be a licence of ``kind``, or None. A key is one
97
+ line; a server is ``host:port``; a file is at most a megabyte."""
98
+ if kind not in KINDS:
99
+ return f"a licence is one of {KINDS}"
100
+ data = value.encode() if isinstance(value, str) else bytes(value)
101
+ if not data:
102
+ return "it is empty"
103
+ if len(data) > MAX_BYTES:
104
+ return f"it is larger than {MAX_BYTES // 1000} kB"
105
+ if kind == "file":
106
+ return None
107
+ try:
108
+ text = data.decode()
109
+ except UnicodeDecodeError:
110
+ return "it is not text"
111
+ if "\n" in text or "\r" in text or "\0" in text:
112
+ return "it is one line"
113
+ if kind == "server":
114
+ m = _SERVER.match(text)
115
+ if not m or not 0 < int(m.group(1)) < 65536:
116
+ return "a licence server is host:port (a name or an IPv4 address, and its port)"
117
+ return None
118
+
119
+
120
+ def set(bench: str, name: str, value: Union[str, bytes], *, kind: str = "value", filename: str = "",
121
+ expires: str = "") -> None:
122
+ """Keep ``value`` as ``bench``'s licence ``name``, replacing any before.
123
+ ``expires``: YYYY-MM-DD, or empty. Refused (ValueError) when it is not a
124
+ licence of its kind."""
125
+ _name(name)
126
+ if isinstance(value, str):
127
+ value = value.strip()
128
+ why = problem(kind, value)
129
+ if why:
130
+ raise ValueError(f"{name}: {why}")
131
+ if expires and not _DATE.match(expires):
132
+ raise ValueError(f"{name}: an expiry date is YYYY-MM-DD")
133
+ d = _folder(bench, make=True)
134
+ _write(os.path.join(d, name), value.encode() if isinstance(value, str) else bytes(value))
135
+ idx = _index(bench)
136
+ idx[name] = {"on": time.strftime("%Y-%m-%d"), "kind": kind,
137
+ **({"expires": expires} if expires else {}),
138
+ **({"filename": os.path.basename(filename)[:120]} if filename else {})}
139
+ _write(os.path.join(d, INDEX), json.dumps(idx, indent=1, sort_keys=True).encode())
140
+
141
+
142
+ def delete(bench: str, name: str) -> bool:
143
+ """Remove ``bench``'s licence ``name``; whether there was one."""
144
+ _name(name)
145
+ d = _folder(bench)
146
+ try:
147
+ os.unlink(os.path.join(d, name))
148
+ there = True
149
+ except FileNotFoundError:
150
+ there = False
151
+ idx = _index(bench)
152
+ if idx.pop(name, None) is not None:
153
+ _write(os.path.join(d, INDEX), json.dumps(idx, indent=1, sort_keys=True).encode())
154
+ return there
155
+
156
+
157
+ def status(bench: str) -> Dict[str, dict]:
158
+ """Every licence ``bench`` holds: {name: {set, on, expires, kind,
159
+ filename, expired}} - dates and names, never a value."""
160
+ out = {}
161
+ d = _folder(bench)
162
+ today = time.strftime("%Y-%m-%d")
163
+ for name, rec in _index(bench).items():
164
+ if not _NAME.match(name) or not os.path.isfile(os.path.join(d, name)):
165
+ continue
166
+ exp = rec.get("expires") or ""
167
+ out[name] = {"set": True, "on": rec.get("on") or "", "expires": exp, "kind": rec.get("kind") or "value",
168
+ "filename": rec.get("filename") or "", "expired": bool(exp) and exp < today}
169
+ return out
170
+
171
+
172
+ def path(bench: str, name: str) -> Optional[str]:
173
+ """Where ``bench``'s licence ``name`` is on this machine, or None: what a
174
+ run mounts, or names to a process, for a licence file."""
175
+ p = os.path.join(_folder(bench), _name(name))
176
+ return p if os.path.isfile(p) else None
177
+
178
+
179
+ def value(bench: str, name: str) -> Optional[bytes]:
180
+ """The bytes of ``bench``'s licence ``name``, or None. A run's alone."""
181
+ p = path(bench, name)
182
+ if p is None:
183
+ return None
184
+ with open(p, "rb") as f:
185
+ return f.read()
186
+
187
+
188
+ def store_of(bench: str, name: str, provider: str = "") -> Tuple[str, str]:
189
+ """Where ``bench``'s credential ``name`` is kept, (folder, name): its
190
+ data provider's alone when ``provider`` is said, else the benchmark's.
191
+ ValueError for a provider that is not one."""
192
+ if not provider:
193
+ return bench, name
194
+ if provider not in PROVIDERS:
195
+ raise ValueError(f"{provider!r} is not a data provider ({', '.join(PROVIDERS)})")
196
+ return PROVIDERS[provider]
197
+
198
+
199
+ def state(bench: str, name: str, provider: str = "") -> Optional[dict]:
200
+ """:func:`status` of one credential, read where it is kept
201
+ (:func:`store_of`); None when it is not set."""
202
+ folder, kept = store_of(bench, name, provider)
203
+ return status(folder).get(kept)
204
+
205
+
206
+ # What a run on a host is handed of the licences, one line of its launcher's stdin (``execution``).
207
+ HANDED = "NULLIUS_LICENCES"
208
+
209
+
210
+ def unpack(line: str) -> Dict[str, bytes]:
211
+ """Decode a host's licence line: base64 of {name: base64 bytes}.
212
+ Return {name: bytes}, or nothing for a line that is not one."""
213
+ try:
214
+ got = json.loads(base64.b64decode(line.encode(), validate=True))
215
+ return {n: base64.b64decode(v.encode(), validate=True) for n, v in got.items()
216
+ if isinstance(n, str) and _NAME.match(n) and isinstance(v, str)}
217
+ except (ValueError, AttributeError, TypeError):
218
+ return {}