nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,426 @@
1
+ """The benchmark catalogue: which benchmarks exist, in what state, and where
2
+ each dataset comes from.
3
+
4
+ Bundled JSON files beside this module, overlaid by the user's catalogue,
5
+ one folder per benchmark. The app renders them as ``/benchmarks`` - a card
6
+ per benchmark - and
7
+ ``/benchmarks/<key>``, one page per folder: how it runs, its datasets, its
8
+ limitations (the upstream's, then Nullius's), its citation. ``validate`` gates it.
9
+ The knowledge itself — what runs a benchmark, how its dataset is fetched —
10
+ is written once, here::
11
+
12
+ benchmarks/<key>/ one folder per benchmark, named by its key
13
+ benchmark.json the entry, its datasets nested in it
14
+ standard.json its definition on the Nullius standard, when
15
+ it runs there (nullius.standard)
16
+
17
+ Everything about a benchmark is in its folder. A dataset's ``benchmark`` is
18
+ the folder it sits in, never written; its key is unique across the whole
19
+ catalogue. ``load`` reads the folders in name order, so the catalogue's
20
+ order is the folders', and nothing depends on it: the page and the builder's
21
+ benchmark list sort by title, a lookup goes by key, and a benchmark's
22
+ datasets keep the order its entry writes them in. ``layout`` is the check on
23
+ the tree itself - a folder named after its key - and ``validate`` runs it.
24
+
25
+ A dataset's ``source`` is a short phrase (Harbor registry, Harbor hub,
26
+ GitHub, …). Each benchmark carries a ``category`` (Coding, Terminal, Customer
27
+ service, …) used as a tag and a filter.
28
+
29
+ An optional ``task_count`` gives the number of distinct tasks when datasets
30
+ overlap, such as a full split beside difficulty subsets. It controls the
31
+ benchmark headline only; every dataset keeps its own count.
32
+
33
+ Every entry says where it comes from (``origin``): ``official``, the
34
+ publisher's own benchmark, or
35
+ ``community``, adapted by the community. A card and a page show it as a tag
36
+ beside the category, and the Benchmarks page shelves by it. Whether anyone
37
+ has run it is not part of an entry.
38
+
39
+ Routes: a benchmark reaches a run through a harness, and ``Benchmark.routes``
40
+ says, for each harness in ``HARNESSES``, whether that route is usable here.
41
+ A harness the entry names is usable: the Nullius standard, or Harbor. The page
42
+ lists a benchmark with a usable route, and says why for one without. A
43
+ dataset is shown inside its
44
+ benchmark's entry, so it is listed exactly when its benchmark is.
45
+ """
46
+ from __future__ import annotations
47
+
48
+ import json
49
+ import os
50
+ from dataclasses import dataclass, field
51
+
52
+ from nullius import paths
53
+ from . import storage
54
+
55
+ # Where an entry comes from: the publisher's benchmark (official),
56
+ # or adapted by the community (community).
57
+ ORIGINS = ("official", "community")
58
+ ROOT = os.path.dirname(os.path.abspath(__file__))
59
+ BENCHMARKS = "benchmarks" # the folder of benchmark folders, under ROOT
60
+ ENTRY = "benchmark.json" # a benchmark's entry, in its folder
61
+
62
+ # The harnesses a benchmark can be run through, in the order the Benchmarks
63
+ # page lists them: the Nullius standard first (benchmarks/<key>/standard.json,
64
+ # nullius.standard), then Harbor, each a route of the entries that name it. A
65
+ # benchmark's own scripts run as the standard's Original setup.
66
+ HARNESSES = (("standard", "Nullius"), ("harbor", "Harbor"))
67
+
68
+
69
+
70
+ @dataclass(frozen=True)
71
+ class Route:
72
+ """One harness route of a benchmark: usable, or unavailable with a reason.
73
+
74
+ ``usable`` is the one thing a selector may act on. ``reason`` is filled
75
+ only when it is not — the text a page shows greyed beside the route, so a
76
+ reader learns *why* rather than only that it cannot be picked."""
77
+
78
+ harness: str # a key of HARNESSES: harbor or standard
79
+ label: str # its label: Harbor
80
+ usable: bool
81
+ reason: str = ""
82
+
83
+ @property
84
+ def summary(self) -> str:
85
+ """The route as one phrase: `Harbor · runnable` or `Harbor · unavailable — …`."""
86
+ return f"{self.label} · runnable" if self.usable else f"{self.label} · unavailable — {self.reason}"
87
+
88
+
89
+ @dataclass
90
+ class Benchmark:
91
+ key: str
92
+ title: str
93
+ category: str = "" # one tag: Coding, Terminal, Customer service, …
94
+ domain: str = ""
95
+ #: how a task is scored, in two or three words for its card; each dataset's verifier says it in full
96
+ scoring: str = ""
97
+ harness: list = field(default_factory=list)
98
+ good_to_know: list = field(default_factory=list)
99
+ #: what the upstream itself gets wrong or leaves out, as published: not Nullius's to fix
100
+ upstream_limitations: list = field(default_factory=list)
101
+ #: Nullius's own: how it runs the benchmark differently, what it does not offer yet
102
+ limitations: list = field(default_factory=list)
103
+ datasets: list = field(default_factory=list)
104
+ #: declared per-phase timeouts in seconds (env_setup, agent_setup,
105
+ #: agent_run, verify) as recorded from a real run; empty when no run has
106
+ #: recorded them
107
+ timeouts: dict = field(default_factory=dict)
108
+ #: the publisher's benchmark or a community adaptation (ORIGINS)
109
+ origin: str = "official"
110
+ #: how it runs as published when Harbor runs it: the upstream's own agent, by
111
+ #: the name Harbor registers it (``agent``), and why it is the published one
112
+ #: (``says``); a benchmark on the standard says it in its ``setups`` instead
113
+ published: dict = field(default_factory=dict)
114
+ #: how its verifier's own report is read (``nullius.runs.verifier``):
115
+ #: the file, the fields that say pass or fail, the words the pages use; empty
116
+ #: when the reward is all there is
117
+ report: dict = field(default_factory=dict)
118
+ #: how to cite it: the BibTeX its official repository gives, else its paper's
119
+ citation: str = ""
120
+ #: where that citation was taken from
121
+ citation_source: str = ""
122
+ #: Distinct tasks across dataset views, when known; otherwise show their count summary.
123
+ task_count: int | None = None
124
+
125
+ @property
126
+ def label(self) -> str:
127
+ return self.origin.capitalize()
128
+
129
+ @property
130
+ def can_run(self) -> bool:
131
+ """Listed in the catalogue: a harness it names is here."""
132
+ return bool(self.usable_routes)
133
+
134
+ @property
135
+ def routes(self) -> list:
136
+ """Every harness, as a `Route`: usable when the entry names it, since
137
+ a harness in `HARNESSES` is here; Harbor, named or not, says so when
138
+ it is not this entry's."""
139
+ out = []
140
+ for harness, label in HARNESSES:
141
+ if harness in self.harness:
142
+ out.append(Route(harness, label, True))
143
+ elif harness == "harbor":
144
+ out.append(Route(harness, label, False, "no Harbor dataset"))
145
+ return out
146
+
147
+ @property
148
+ def usable_routes(self) -> list:
149
+ return [r for r in self.routes if r.usable]
150
+
151
+ @property
152
+ def blocker(self) -> str:
153
+ """Why nothing is usable, in one phrase — empty when a route is: the
154
+ reason a benchmark's page shows in place of Build a run."""
155
+ if self.usable_routes:
156
+ return ""
157
+ missing = [h for h in self.harness if h not in dict(HARNESSES)]
158
+ if missing:
159
+ return "no harness here for " + ", ".join(missing)
160
+ return "it names no harness"
161
+
162
+ @property
163
+ def route_case(self) -> str:
164
+ """Which of the four discovery cases this entry is — what a test and a
165
+ walk-through name: `harbor-only`, `multi-route`, `non-harbor` (usable
166
+ routes, none of them Harbor) or `none`. A benchmark served only by the
167
+ Nullius standard is `non-harbor`; one with both a Harbor adapter and a
168
+ definition on the standard is `multi-route`."""
169
+ usable = [r.harness for r in self.usable_routes]
170
+ if not usable:
171
+ return "none"
172
+ if usable == ["harbor"]:
173
+ return "harbor-only"
174
+ return "multi-route" if "harbor" in usable else "non-harbor"
175
+
176
+
177
+ @dataclass
178
+ class Dataset:
179
+ key: str
180
+ title: str
181
+ benchmark: str = ""
182
+ tasks: int = 0
183
+ official: str = ""
184
+ source: str = "" # where it comes from, in a phrase: Harbor hub, GitHub …
185
+ registry: str = ""
186
+ hub: str = ""
187
+ #: how its tasks are scored
188
+ verifier: str = ""
189
+ notes: str = ""
190
+
191
+
192
+ def sort_key(title: str) -> str:
193
+ """Alphabetical as a reader expects: a title starting with τ files under T."""
194
+ return title.replace("τ", "t").replace("³", "3").lower()
195
+
196
+
197
+ @dataclass
198
+ class Catalog:
199
+ benchmarks: list
200
+ datasets: list
201
+ path: str = ROOT # explicit root, or the writable user catalogue
202
+ _folders: dict = field(default_factory=dict, repr=False)
203
+
204
+ def entry_dir(self, key: str) -> str:
205
+ """The selected folder: all files come from the same entry copy."""
206
+ return self._folders.get(key, os.path.join(self.path, BENCHMARKS, key))
207
+
208
+ def benchmark(self, key: str):
209
+ return next((b for b in self.benchmarks if b.key == key), None)
210
+
211
+ def dataset(self, key: str):
212
+ return next((d for d in self.datasets if d.key == key), None)
213
+
214
+ def datasets_of(self, b: "Benchmark") -> list:
215
+ """A benchmark's datasets, as `Dataset`, in the order its entry writes them."""
216
+ return [d for d in (self.dataset(k) for k in b.datasets) if d]
217
+
218
+ def tasks_of(self, b: "Benchmark") -> tuple:
219
+ """Its tasks as a number and a label, for a card: ``("500", "tasks")``,
220
+ ``("500", "tasks each")`` when its datasets agree, ``("10–89", "tasks")``
221
+ when they do not, ``("—", "tasks")`` when none is recorded.
222
+ An explicit distinct task count takes precedence over those summaries."""
223
+ if type(b.task_count) is int and b.task_count > 0:
224
+ return f"{b.task_count:,}", "tasks"
225
+ counts = [d.tasks for d in self.datasets_of(b) if d.tasks]
226
+ if not counts:
227
+ return "—", "tasks"
228
+ if len(set(counts)) == 1:
229
+ return f"{counts[0]:,}", "tasks each" if len(counts) > 1 else "tasks"
230
+ return f"{min(counts):,}–{max(counts):,}", "tasks"
231
+
232
+ def files(self, key: str) -> list:
233
+ """The files in a benchmark's folder, by name: what its page shows it is made of."""
234
+ base = self.entry_dir(key)
235
+ try:
236
+ return sorted(n for n in os.listdir(base)
237
+ if os.path.isfile(os.path.join(base, n)) and not n.startswith("."))
238
+ except OSError:
239
+ return []
240
+
241
+ @property
242
+ def listed_benchmarks(self) -> list:
243
+ """What the catalogue shows (the Benchmarks tab, the rail): the
244
+ benchmarks with a usable route, alphabetically."""
245
+ return sorted((b for b in self.benchmarks if b.can_run),
246
+ key=lambda b: sort_key(b.title))
247
+
248
+ @property
249
+ def categories(self) -> list:
250
+ """Every category the listed benchmarks use, alphabetically."""
251
+ return sorted({b.category for b in self.listed_benchmarks if b.category}, key=sort_key)
252
+
253
+
254
+ def _pick(cls, data: dict):
255
+ known = {f for f in cls.__dataclass_fields__}
256
+ return cls(**{k: v for k, v in data.items() if k in known})
257
+
258
+
259
+ def _read(path: str):
260
+ with open(path, encoding="utf-8") as fh:
261
+ return json.load(fh)
262
+
263
+
264
+ def folders(root: str | None = None) -> list:
265
+ """The benchmark folders under ``root``, by name: the catalogue's order.
266
+ A folder holding nothing but Python's ``__pycache__`` is left out: git
267
+ does not track the cache, so a checkout that once had a package there
268
+ keeps the folder after the package moves."""
269
+ return list(storage.folders(BENCHMARKS, ROOT, root))
270
+
271
+
272
+ def entry_dir(key: str, root: str | None = None) -> str | None:
273
+ return storage.entry_dir(BENCHMARKS, key, ROOT, root)
274
+
275
+
276
+ def writable_dir(key: str, root: str | None = None) -> str:
277
+ return storage.writable_dir(BENCHMARKS, key, ROOT, root)
278
+
279
+
280
+ def signature(root: str | None = None) -> tuple:
281
+ """What changes when any file ``load`` reads changes, or a benchmark
282
+ folder comes or goes: the app's cache reloads on it."""
283
+ entries = [os.path.join(folder, ENTRY) for folder in storage.folders(BENCHMARKS, ROOT, root).values()]
284
+ out = [(base, None) for base in storage.roots(ROOT, root)]
285
+ for p in entries:
286
+ try:
287
+ info = os.stat(p)
288
+ out.append((p, (info.st_mtime_ns, info.st_size)))
289
+ except OSError:
290
+ out.append((p, None))
291
+ return tuple(out)
292
+
293
+
294
+ def load(root: str | None = None) -> Catalog:
295
+ """Every ``benchmarks/<key>/benchmark.json`` under
296
+ ``root``, the folders in name order. A benchmark's datasets are nested in
297
+ its entry; each comes out as a `Dataset` naming the folder it sits in,
298
+ and the entry's ``datasets`` as their keys, in the order written.
299
+
300
+ With no root, user folders replace bundled folders of the same name.
301
+ An explicit root reads that tree alone, without the user overlay.
302
+ """
303
+ benchmarks, datasets = [], []
304
+ selected = storage.folders(BENCHMARKS, ROOT, root)
305
+ for name, folder in selected.items():
306
+ path = os.path.join(folder, ENTRY)
307
+ if not os.path.isfile(path):
308
+ continue # layout() names it; nothing to list
309
+ try:
310
+ benchmark, nested = _entry(path, name)
311
+ except (OSError, ValueError, TypeError, AttributeError):
312
+ continue # a hand-edited entry that does not read: layout() names it, every page still opens
313
+ benchmarks.append(benchmark)
314
+ datasets.extend(nested)
315
+ return Catalog(
316
+ benchmarks=benchmarks,
317
+ datasets=datasets,
318
+ path=os.fspath(root) if root is not None else paths.catalogue_dir(),
319
+ _folders=selected,
320
+ )
321
+
322
+
323
+ def _entry(path: str, name: str) -> tuple:
324
+ """(Benchmark, [Dataset]) of one ``benchmark.json``; ValueError or
325
+ TypeError when it is not JSON, not an object or lacks a field."""
326
+ entry = _read(path)
327
+ if not isinstance(entry, dict):
328
+ raise ValueError("not a JSON object")
329
+ if entry.get("experiment") or entry.get("origin") == "experiment":
330
+ entry = {**entry, "origin": "official"} # experiment is now official
331
+ nested = [d for d in entry.get("datasets", []) if isinstance(d, dict)]
332
+ return (_pick(Benchmark, {**entry, "datasets": [d.get("key") for d in nested]}),
333
+ [_pick(Dataset, {**d, "benchmark": name}) for d in nested])
334
+
335
+
336
+ def layout(root: str | None = None) -> list:
337
+ """What is wrong with the tree itself, which no loaded object can show:
338
+ a folder without its entry, an entry whose key is not its folder's name,
339
+ a dataset that names another benchmark than the folder it sits in.
340
+ Empty means the folders say what they hold."""
341
+ return _layout(storage.folders(BENCHMARKS, ROOT, root))
342
+
343
+
344
+ def _layout(selected: dict) -> list:
345
+ bad = []
346
+ for name, folder in selected.items():
347
+ where = f"{BENCHMARKS}/{name}/"
348
+ entry = {}
349
+ if not os.path.isfile(os.path.join(folder, ENTRY)):
350
+ bad.append(f"{where} has no {ENTRY}")
351
+ else:
352
+ try:
353
+ entry = _read(os.path.join(folder, ENTRY))
354
+ _entry(os.path.join(folder, ENTRY), name)
355
+ except (OSError, ValueError, TypeError, AttributeError) as e:
356
+ bad.append(f"{where}{ENTRY}: {e}")
357
+ entry = entry if isinstance(entry, dict) else {}
358
+ if entry and entry.get("key") != name:
359
+ bad.append(f"{where}{ENTRY}: key {entry.get('key')!r} is not its folder's name")
360
+ if "standard" in entry.get("harness", []) and not os.path.isfile(os.path.join(folder, "standard.json")):
361
+ bad.append(f"{where} names the Nullius standard but has no standard.json")
362
+ for d in entry.get("datasets", []):
363
+ if not isinstance(d, dict):
364
+ bad.append(f"{where}{ENTRY}: a dataset is a JSON object")
365
+ elif d.get("benchmark", name) != name:
366
+ bad.append(f"{where}{ENTRY}: dataset {d.get('key')!r} names benchmark "
367
+ f"{d['benchmark']!r}, and sits in {name!r}")
368
+ return bad
369
+
370
+
371
+ def validate(cat: Catalog) -> list:
372
+ """Everything that would make a tab lie. Empty means consistent.
373
+
374
+ The selected folders are checked too (`layout`): a folder named after
375
+ its key, whether from the bundle, the user catalogue or an explicit root."""
376
+ bad = []
377
+ if cat._folders:
378
+ bad.extend(_layout(cat._folders))
379
+ elif os.path.isdir(os.path.join(cat.path, BENCHMARKS)):
380
+ bad.extend(layout(cat.path))
381
+ keys = [b.key for b in cat.benchmarks]
382
+ if len(set(keys)) != len(keys):
383
+ bad.append("duplicate benchmark keys")
384
+ every = [d.key for d in cat.datasets]
385
+ for k in sorted({k for k in every if every.count(k) > 1}):
386
+ bad.append(f"dataset key {k!r} used twice: a dataset key is unique across the catalogue")
387
+ dkeys = {d.key for d in cat.datasets}
388
+ # a report block's sources name the job's dataset: one name, one way to read it
389
+ claimed = {}
390
+ for b in cat.benchmarks:
391
+ for s in (b.report or {}).get("sources") or []:
392
+ if s.strip().lower() in claimed:
393
+ bad.append(f"{b.key}: report source {s!r} is {claimed[s.strip().lower()]}'s too")
394
+ claimed.setdefault(s.strip().lower(), b.key)
395
+ for b in cat.benchmarks:
396
+ if b.origin not in ORIGINS:
397
+ bad.append(f"{b.key}: origin is official or community")
398
+ if b.task_count is not None:
399
+ if type(b.task_count) is not int or b.task_count < 1:
400
+ bad.append(f"{b.key}: task_count must be a positive integer")
401
+ elif any(d.tasks > b.task_count for d in cat.datasets_of(b)):
402
+ bad.append(f"{b.key}: task_count cannot be smaller than a dataset")
403
+ if b.published and not (isinstance(b.published, dict) and isinstance(b.published.get("agent"), str) and b.published["agent"]
404
+ and isinstance(b.published.get("says"), str) and b.published["says"]):
405
+ bad.append(f"{b.key}: published is {{agent: a Harbor agent's name, says: why it is the published one}}")
406
+ if not b.category:
407
+ bad.append(f"{b.key}: no category")
408
+ for h in b.harness:
409
+ if h not in dict(HARNESSES):
410
+ bad.append(f"{b.key}: harness {h!r} is not one of {[k for k, _ in HARNESSES]}")
411
+ for d in b.datasets:
412
+ if d not in dkeys:
413
+ bad.append(f"{b.key}: dataset {d!r} is not in the catalogue")
414
+ for d in cat.datasets:
415
+ # A dataset is shown on its benchmark's page and
416
+ # nowhere else, so one its benchmark does not list would never be seen.
417
+ owner = cat.benchmark(d.benchmark) if d.benchmark else None
418
+ if not d.benchmark:
419
+ bad.append(f"dataset {d.key}: names no benchmark")
420
+ elif owner is None:
421
+ bad.append(f"dataset {d.key}: benchmark {d.benchmark!r} is not in the catalogue")
422
+ elif d.key not in owner.datasets:
423
+ bad.append(f"dataset {d.key}: benchmark {d.benchmark!r} does not list it")
424
+ if not d.source:
425
+ bad.append(f"dataset {d.key}: no source")
426
+ return bad
@@ -0,0 +1,116 @@
1
+ """A benchmark's citation, read: its ``citation`` field is BibTeX, as the
2
+ benchmark's authors publish it; this turns each entry into the parts a page
3
+ shows - the title, the people, the venue and year, the link - beside the
4
+ entry itself, which is what a reader copies. Nothing is fetched and nothing
5
+ is rewritten: a field the entry lacks is left empty.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import re
10
+
11
+ _ACCENTS = {'"': {"o": "ö", "a": "ä", "u": "ü", "e": "ë", "i": "ï", "O": "Ö", "A": "Ä", "U": "Ü"},
12
+ "'": {"e": "é", "a": "á", "o": "ó", "i": "í", "u": "ú", "E": "É"},
13
+ "`": {"e": "è", "a": "à"}, "^": {"o": "ô", "e": "ê"}, "~": {"n": "ñ"}, "c": {"c": "ç"}}
14
+ _SHORT = [(re.compile(r"International Conference on Learning Representations"), "ICLR"),
15
+ (re.compile(r"Empirical Methods in Natural Language Processing"), "EMNLP"),
16
+ (re.compile(r"Neural Information Processing Systems"), "NeurIPS"),
17
+ (re.compile(r"International Conference on Machine Learning"), "ICML")]
18
+ KINDS = {"conference": "Conference paper", "preprint": "Preprint", "workshop": "Workshop paper"}
19
+
20
+
21
+ def delatex(s: str) -> str:
22
+ """BibTeX's markup as text: the τ of τ-bench, dashes, accents; braces gone."""
23
+ s = s.replace("$\\tau^2$", "τ²").replace("$\\tau^3$", "τ³").replace("$\\tau$", "τ")
24
+ s = re.sub(r"\{?\\textemdash\}?", "—", s)
25
+ s = re.sub(r"\{?\\textendash\}?", "–", s)
26
+ s = re.sub(r"\{?\\([\"'`^~c])\{?([A-Za-z])\}?\}?", lambda m: _ACCENTS.get(m.group(1), {}).get(m.group(2), m.group(2)), s)
27
+ return re.sub(r"\s+", " ", s.replace("{", "").replace("}", "")).strip()
28
+
29
+
30
+ def _value(s: str, j: int) -> tuple:
31
+ if j < len(s) and s[j] == "{":
32
+ d, k = 1, j + 1
33
+ while k < len(s) and d:
34
+ d += 1 if s[k] == "{" else -1 if s[k] == "}" else 0
35
+ k += 1
36
+ return s[j + 1:k - 1], k
37
+ if j < len(s) and s[j] == '"':
38
+ d, k = 0, j + 1
39
+ while k < len(s) and not (s[k] == '"' and d == 0):
40
+ d += 1 if s[k] == "{" else -1 if s[k] == "}" else 0
41
+ k += 1
42
+ return s[j + 1:k], k + 1
43
+ k = j
44
+ while k < len(s) and s[k] != ",":
45
+ k += 1
46
+ return s[j:k].strip(), k
47
+
48
+
49
+ def _name(n: str) -> str:
50
+ n = delatex(n)
51
+ if "," in n:
52
+ last, first = (x.strip() for x in n.split(",", 1))
53
+ return f"{first} {last}".strip()
54
+ return n
55
+
56
+
57
+ def _paper(kind: str, key: str, f: dict, raw: str) -> dict:
58
+ authors = [_name(a) for a in re.split(r"\s+and\s+", f.get("author", "")) if a.strip()]
59
+ journal = delatex(f.get("journal", ""))
60
+ m = re.search(r"arXiv:(\d{4}\.\d{4,5})", journal)
61
+ arxiv = f.get("eprint", "") or (m.group(1) if m else "")
62
+ venue = delatex(f["booktitle"]) if f.get("booktitle") else journal
63
+ if not venue and arxiv:
64
+ venue = "arXiv preprint arXiv:" + arxiv
65
+ short = venue
66
+ for pattern, name in _SHORT:
67
+ if pattern.search(venue) and "Workshop" not in venue:
68
+ short = name
69
+ if venue.startswith("arXiv"):
70
+ short = "arXiv"
71
+ if "Workshop" in venue:
72
+ short = re.sub(r" on .*", "", venue)
73
+ year = delatex(f.get("year", ""))
74
+ kind_of = "workshop" if "Workshop" in venue else "preprint" if short == "arXiv" else "conference"
75
+ shown = len(authors) <= 4
76
+ return {"key": key, "type": kind, "bibtex": raw, "title": delatex(f.get("title", "")), "authors": authors,
77
+ "authors_short": (", ".join(authors[:-1]) + (" and " if len(authors) > 1 else "") + (authors[-1] if authors else "")) if shown
78
+ else f"{', '.join(authors[:3])} and {len(authors) - 3} others",
79
+ "year": year, "venue": venue, "venue_short": "arXiv" if short == "arXiv" else re.sub(r" \d{4}.*", "", short),
80
+ "kind": kind_of, "kind_label": KINDS[kind_of], "arxiv": arxiv,
81
+ # a link is an http(s) one or none: an entry is text anyone may have written, and a page links it
82
+ "url": (f.get("url", "").strip() if re.match(r"^https?://", f.get("url", "").strip(), re.I) else "")
83
+ or (f"https://arxiv.org/abs/{arxiv}" if re.fullmatch(r"\d{4}\.\d{4,5}(v\d+)?", arxiv) else "")}
84
+
85
+
86
+ def parse(text: str) -> list:
87
+ """Every entry of a BibTeX text, in order: a dict of its parts (see
88
+ :func:`_paper`) and ``bibtex``, the entry as written."""
89
+ text = (text or "").replace("\r\n", "\n")
90
+ out, i = [], 0
91
+ while True:
92
+ i = text.find("@", i)
93
+ if i < 0:
94
+ break
95
+ m = re.match(r"@(\w+)\s*\{", text[i:])
96
+ if not m:
97
+ i += 1
98
+ continue
99
+ j = start = i + m.end()
100
+ d = 1
101
+ while j < len(text) and d:
102
+ d += 1 if text[j] == "{" else -1 if text[j] == "}" else 0
103
+ j += 1
104
+ body, raw = text[start:j - 1], text[i:j].strip()
105
+ c = body.find(",")
106
+ key, fields, k = body[:c].strip(), {}, c + 1
107
+ while k < len(body):
108
+ nm = re.match(r"[\s,]*([A-Za-z][\w-]*)\s*=\s*", body[k:])
109
+ if not nm:
110
+ break
111
+ k += nm.end()
112
+ value, k = _value(body, k)
113
+ fields[nm.group(1).lower()] = value
114
+ out.append(_paper(m.group(1).lower(), key, fields, raw))
115
+ i = j
116
+ return out