nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,367 @@
1
+ """A Harbor benchmark added by hand: read from Harbor, then written as a catalogue folder.
2
+
3
+ The Benchmarks page's *Add a Harbor benchmark* form uses two steps:
4
+
5
+ - :func:`read` asks Harbor about a dataset, on the hub (``org/name``) or in
6
+ the registry (``name@version``): how many tasks, its description and
7
+ version, and the limits one of its tasks declares in its ``task.toml`` -
8
+ the environment build, the agent and the verifier. Harbor applies those
9
+ limits to every trial, so they are what the builder's timeouts start
10
+ from. Agent setup is the agent's own default, Harbor's 360 s.
11
+ - :func:`build` turns the form into a ``benchmark.json`` and :func:`save`
12
+ writes it to ``benchmarks/<key>/``, after the catalogue's own checks
13
+ (:func:`catalog.validate`) pass on the catalogue with it added. An
14
+ existing folder is never written over.
15
+ - An entry the form can show (:func:`not_editable`) is edited the same way:
16
+ :func:`values_of` fills the form, :func:`merged` keeps whatever the form
17
+ does not show, and :func:`update` writes it back over itself.
18
+
19
+ Only Harbor routes: a Nullius route is brought in by ``/import-benchmark``,
20
+ then ``/import-standard``. Nothing here imports Flask; Harbor is reached through a ``fetch``
21
+ seam.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import os
27
+ import re
28
+ import tempfile
29
+ import tomllib
30
+ from dataclasses import replace
31
+ from urllib import error, parse, request
32
+
33
+ from . import catalog, storage
34
+
35
+ HUB = "https://hub.harborframework.com"
36
+ REGISTRY = "https://raw.githubusercontent.com/laude-institute/harbor/main/registry.json"
37
+ AGENT_SETUP_SEC = 360 # Harbor's default for an agent's setup; the task does not declare it
38
+ KEY = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
39
+ HUB_REF = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
40
+ REGISTRY_REF = re.compile(r"^[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+$")
41
+ SOURCES = {"hub": "Harbor hub", "registry": "Harbor registry"}
42
+
43
+
44
+ class ReadError(Exception):
45
+ """Harbor could not be read for that dataset; the message says why."""
46
+
47
+
48
+ def _fetch(url: str, timeout: float = 20.0) -> bytes:
49
+ req = request.Request(url, headers={"User-Agent": "nullius", "Accept": "application/json, text/plain, */*"})
50
+ with request.urlopen(req, timeout=timeout) as r:
51
+ return r.read()
52
+
53
+
54
+ def _get(fetch, url: str, what: str) -> bytes:
55
+ try:
56
+ return fetch(url)
57
+ except error.HTTPError as e:
58
+ raise ReadError(f"{what}: Harbor answered {e.code}") from e
59
+ except (error.URLError, OSError, TimeoutError) as e:
60
+ raise ReadError(f"{what}: Harbor could not be reached ({getattr(e, 'reason', e)})") from e
61
+
62
+
63
+ def timeouts_of(toml_text: str) -> dict:
64
+ """The four phase limits, in seconds, from a task's ``task.toml``; a
65
+ limit the task leaves out is Harbor's default."""
66
+ try:
67
+ t = tomllib.loads(toml_text)
68
+ except tomllib.TOMLDecodeError as e:
69
+ raise ReadError(f"the task's task.toml does not parse: {e}") from e
70
+ sec = lambda v, d: int(round(float(v))) if isinstance(v, (int, float)) and v > 0 else d
71
+ return {"env_setup": sec((t.get("environment") or {}).get("build_timeout_sec"), 600),
72
+ "agent_setup": AGENT_SETUP_SEC,
73
+ "agent_run": sec((t.get("agent") or {}).get("timeout_sec"), 3600),
74
+ "verify": sec((t.get("verifier") or {}).get("timeout_sec"), 600)}
75
+
76
+
77
+ def read(source: str, ref: str, fetch=_fetch) -> dict:
78
+ """What Harbor says of a dataset: ``{source, ref, tasks, version,
79
+ description, task, timeouts, key, title}``. ``key`` and ``title`` are
80
+ suggestions from the name."""
81
+ ref = (ref or "").strip()
82
+ if source == "hub":
83
+ if not HUB_REF.match(ref):
84
+ raise ReadError("a hub dataset is org/name, as on hub.harborframework.com")
85
+ org, name = ref.split("/", 1)
86
+ listing = json.loads(_get(fetch, f"{HUB}/api/datasets/{parse.quote(org)}/{parse.quote(name)}/latest/tasks", f"the hub dataset {ref}"))
87
+ items = listing.get("items") or []
88
+ if not items:
89
+ raise ReadError(f"the hub dataset {ref} has no tasks")
90
+ first = items[0]
91
+ toml = _get(fetch, f"{HUB}/api/tasks/{parse.quote(first['org_name'])}/{parse.quote(first['task_name'])}/latest/files/task.toml",
92
+ f"the task {first['task_name']}").decode("utf-8", "replace")
93
+ version, description = "", ""
94
+ try: # the page's own summary: "Tasks: 731 · Version: 2.0.0 · …"
95
+ page = _get(fetch, f"{HUB}/datasets/{parse.quote(org)}/{parse.quote(name)}", "the dataset page").decode("utf-8", "replace")
96
+ meta = re.search(r'<meta name="description" content="([^"]*)"', page)
97
+ if meta:
98
+ parts = [p.strip() for p in meta.group(1).split("·")]
99
+ version = next((p.split(":", 1)[1].strip() for p in parts if p.startswith("Version:")), "")
100
+ description = next((p for p in parts if not p.startswith(("Tasks:", "Version:"))), "")
101
+ if description == "A collection of evaluation tasks on Harbor.": # the hub's words for a dataset that gave none
102
+ description = ""
103
+ except ReadError:
104
+ pass
105
+ n, task, base = int(listing.get("total") or len(items)), first["task_name"], name
106
+ elif source == "registry":
107
+ if not REGISTRY_REF.match(ref):
108
+ raise ReadError("a registry dataset is name@version, as Harbor's registry lists it")
109
+ name, version = ref.split("@", 1)
110
+ entries = json.loads(_get(fetch, REGISTRY, "Harbor's registry"))
111
+ found = next((e for e in entries if e.get("name") == name and str(e.get("version")) == version), None)
112
+ if found is None:
113
+ versions = sorted(str(e.get("version")) for e in entries if e.get("name") == name)
114
+ raise ReadError(f"{ref} is not in Harbor's registry" + (f"; its versions: {', '.join(versions)}" if versions else ""))
115
+ tasks = found.get("tasks") or []
116
+ if not tasks:
117
+ raise ReadError(f"{ref} has no tasks in Harbor's registry")
118
+ first = tasks[0]
119
+ m = re.match(r"^https://github\.com/([^/]+/[^/.]+)(?:\.git)?$", first.get("git_url") or "")
120
+ if not m:
121
+ raise ReadError(f"{ref}'s tasks live at {first.get('git_url')!r}, which is not a GitHub repository this can read")
122
+ toml = _get(fetch, f"https://raw.githubusercontent.com/{m.group(1)}/{first.get('git_commit_id') or 'main'}/{first['path']}/task.toml",
123
+ f"the task {first.get('name')}").decode("utf-8", "replace")
124
+ n, task, base, description = len(tasks), first.get("name") or first["path"], name, found.get("description") or ""
125
+ else:
126
+ raise ReadError("the source is the hub or the registry")
127
+ key = re.sub(r"[^a-z0-9]+", "-", base.lower()).strip("-")[:63] or "benchmark"
128
+ return {"source": source, "ref": ref, "tasks": n, "version": version, "description": description, "task": task,
129
+ "timeouts": timeouts_of(toml), "key": key, "title": base.replace("-", " ").replace("_", " ").strip().title()}
130
+
131
+
132
+ def _lines(text) -> list:
133
+ return [x.strip() for x in str(text or "").splitlines() if x.strip()]
134
+
135
+
136
+ def build(form: dict) -> dict:
137
+ """The ``benchmark.json`` the form describes, in the catalogue's field
138
+ order. Fields are trimmed; empty optional ones are left out."""
139
+ g = lambda k: str(form.get(k) or "").strip()
140
+ source, ref = g("source"), g("ref")
141
+ timeouts = {}
142
+ for phase in ("env_setup", "agent_setup", "agent_run", "verify"):
143
+ try:
144
+ timeouts[phase] = int(float(g(f"timeout_{phase}")))
145
+ except ValueError:
146
+ timeouts = {}
147
+ break
148
+ dataset = {"key": g("dataset_key") or g("key"), "title": g("dataset_title") or g("title"),
149
+ "tasks": int(g("tasks")) if g("tasks").isdigit() else 0, "official": g("official"),
150
+ "source": SOURCES.get(source, source),
151
+ # the builder runs `harbor run -d <registry>`: a hub dataset names itself there too
152
+ **({"hub": ref, "registry": ref} if source == "hub" else {"registry": ref}),
153
+ "verifier": g("verifier")}
154
+ entry = {"key": g("key"), "title": g("title"), "category": g("category_new") or g("category"),
155
+ "domain": g("domain"), "scoring": g("scoring"), "harness": ["harbor"],
156
+ "origin": g("origin") or "official"}
157
+ if _lines(form.get("upstream_limitations")):
158
+ entry["upstream_limitations"] = _lines(form.get("upstream_limitations"))
159
+ entry["limitations"] = _lines(form.get("limitations"))
160
+ entry["datasets"] = [dataset]
161
+ if timeouts:
162
+ entry["timeouts"] = timeouts
163
+ if g("citation"):
164
+ entry["citation"] = g("citation")
165
+ if g("citation_source"):
166
+ entry["citation_source"] = g("citation_source")
167
+ return entry
168
+
169
+
170
+ def problems(entry: dict, root: str | None = None, replacing: str = "", dataset: int = 0) -> list:
171
+ """What stops the entry from being saved: the form's own rules, then the
172
+ catalogue's checks run on the catalogue with this entry added - or, when
173
+ ``replacing`` names the entry being edited, with it in that one's place."""
174
+ bad = []
175
+ key = entry.get("key", "")
176
+ if not KEY.fullmatch(key):
177
+ bad.append("key: lowercase letters, digits and dashes, starting with a letter or digit")
178
+ elif key != replacing and any(os.path.lexists(os.path.join(base, catalog.BENCHMARKS, key))
179
+ for base in storage.roots(catalog.ROOT, root)):
180
+ bad.append(f"key: benchmarks/{key}/ exists already; an entry is never written over")
181
+ if replacing and not KEY.fullmatch(replacing):
182
+ bad.append("key: the entry being edited must have a valid key")
183
+ elif replacing and replacing != key and root is None and os.path.isdir(os.path.join(catalog.ROOT, catalog.BENCHMARKS, replacing)):
184
+ bad.append("key: a bundled entry keeps its key; edit its user copy under the same key")
185
+ for field, label in (("title", "title"), ("category", "category"), ("domain", "domain"), ("scoring", "scoring")):
186
+ if not entry.get(field):
187
+ bad.append(f"{label}: needed")
188
+ d = (entry.get("datasets") or [{}])[dataset] # the Harbor dataset the form wrote: the first, or a route's, the last
189
+ ref = d.get("hub") or d.get("registry") or ""
190
+ if not (HUB_REF.match(ref) if d.get("hub") else REGISTRY_REF.match(ref)):
191
+ bad.append("dataset: org/name on the hub, name@version in the registry")
192
+ if not d.get("tasks"):
193
+ bad.append("tasks: how many the dataset holds")
194
+ if not d.get("verifier"):
195
+ bad.append("verifier: how a task is scored, in a sentence")
196
+ if bad:
197
+ return bad
198
+ cat = catalog.load(root)
199
+ before = catalog.validate(cat)
200
+ if replacing:
201
+ cat = replace(cat, benchmarks=[b for b in cat.benchmarks if b.key != replacing],
202
+ datasets=[x for x in cat.datasets if x.benchmark != replacing])
203
+ new_b = catalog._pick(catalog.Benchmark, {**entry, "datasets": [x.get("key") for x in entry["datasets"]]})
204
+ new_d = [catalog._pick(catalog.Dataset, {**x, "benchmark": key}) for x in entry["datasets"]]
205
+ added = replace(cat, benchmarks=cat.benchmarks + [new_b], datasets=cat.datasets + new_d)
206
+ collisions = {d.key for d in new_d} & {d.key for d in cat.datasets}
207
+ # A pre-existing collision must not permit a third entry to claim the
208
+ # same dataset merely because validate() already reported that key.
209
+ relevant = {f"dataset key {k!r} used twice: a dataset key is unique across the catalogue" for k in collisions}
210
+ return [p for p in catalog.validate(added) if p not in before or p in relevant]
211
+
212
+
213
+ def _write(path: str, entry: dict) -> None:
214
+ temporary = None
215
+ try:
216
+ with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=os.path.dirname(path),
217
+ prefix=".entry-", delete=False) as fh:
218
+ temporary = fh.name
219
+ json.dump(entry, fh, indent=2, ensure_ascii=False)
220
+ fh.write("\n")
221
+ os.replace(temporary, path)
222
+ finally:
223
+ if temporary is not None and os.path.exists(temporary):
224
+ os.unlink(temporary)
225
+
226
+
227
+ def save(entry: dict, root: str | None = None) -> str:
228
+ """Write ``benchmarks/<key>/benchmark.json``; the path written. Refuses,
229
+ with what is wrong, anything :func:`problems` names."""
230
+ bad = problems(entry, root)
231
+ if bad:
232
+ raise ValueError("; ".join(bad))
233
+ folder = catalog.writable_dir(entry["key"], root)
234
+ os.makedirs(folder)
235
+ path = os.path.join(folder, catalog.ENTRY)
236
+ _write(path, entry)
237
+ return path
238
+
239
+
240
+
241
+ # what the form shows of an entry, and so what a save of it replaces; anything
242
+ # else an entry holds (its report block, a branch, notes)
243
+ # is kept as it is. status, status_note and experiment are an older entry's
244
+ # words for its origin, and todo its open work, no longer kept: a save drops them
245
+ FORM_FIELDS = ("title", "category", "domain", "scoring", "harness", "status", "status_note", "origin",
246
+ "experiment", "upstream_limitations", "limitations", "todo", "timeouts", "citation", "citation_source")
247
+ DATASET_FIELDS = ("key", "title", "tasks", "official", "source", "hub", "registry", "verifier")
248
+
249
+
250
+ def entry_of(key: str, root: str | None = None) -> dict | None:
251
+ """The entry as written in ``benchmarks/<key>/benchmark.json``, or None."""
252
+ folder = catalog.entry_dir(key, root)
253
+ if folder is None:
254
+ return None
255
+ path = os.path.join(folder, catalog.ENTRY)
256
+ try:
257
+ with open(path, encoding="utf-8") as fh:
258
+ return json.load(fh)
259
+ except (OSError, ValueError):
260
+ return None
261
+
262
+
263
+ def not_editable(entry: dict) -> str:
264
+ """Why the form cannot edit this entry, or ''. It edits a Harbor entry
265
+ with one dataset, on the hub or in the registry; an entry that also runs
266
+ on the Nullius standard is edited in its files, with the /import-standard skill."""
267
+ if entry.get("harness") != ["harbor"]:
268
+ return "an entry on the Nullius standard is edited in its files (the /import-standard skill)"
269
+ ds = entry.get("datasets") or []
270
+ if len(ds) != 1:
271
+ return "an entry with several datasets is edited in its benchmark.json"
272
+ if not (ds[0].get("hub") or ds[0].get("registry")):
273
+ return "its dataset is neither on the hub nor in the registry"
274
+ return ""
275
+
276
+
277
+ def values_of(entry: dict) -> dict:
278
+ """The form's fields, filled from an entry."""
279
+ d = (entry.get("datasets") or [{}])[0]
280
+ hub = d.get("hub") or ""
281
+ t = entry.get("timeouts") or {}
282
+ values = {"key": entry.get("key", ""), "title": entry.get("title", ""), "category": entry.get("category", ""),
283
+ "domain": entry.get("domain", ""), "scoring": entry.get("scoring", ""),
284
+ "origin": "official" if entry.get("experiment") or entry.get("origin") == "experiment"
285
+ else entry.get("origin", "official"), "upstream_limitations": "\n".join(entry.get("upstream_limitations") or []),
286
+ "limitations": "\n".join(entry.get("limitations") or []),
287
+ "citation": entry.get("citation", ""),
288
+ "citation_source": entry.get("citation_source", ""),
289
+ "source": "hub" if hub else "registry", "ref": hub or d.get("registry", ""),
290
+ "tasks": str(d.get("tasks") or ""), "verifier": d.get("verifier", ""), "official": d.get("official", ""),
291
+ "dataset_key": d.get("key", ""), "dataset_title": d.get("title", "")}
292
+ values.update({f"timeout_{k}": str(t[k]) for k in ("env_setup", "agent_setup", "agent_run", "verify") if k in t})
293
+ return values
294
+
295
+
296
+ def merged(old: dict, new: dict) -> dict:
297
+ """``old`` with what the form owns taken from ``new``: a field the form
298
+ left empty is removed, everything the form does not show is kept."""
299
+ out = {k: v for k, v in old.items() if k not in FORM_FIELDS}
300
+ for k in FORM_FIELDS:
301
+ if k in new:
302
+ out[k] = new[k]
303
+ old_d, new_d = (old.get("datasets") or [{}])[0], new["datasets"][0]
304
+ d = {k: v for k, v in old_d.items() if k not in DATASET_FIELDS}
305
+ d.update({k: v for k, v in new_d.items() if k in DATASET_FIELDS})
306
+ out["datasets"] = [d] + list((old.get("datasets") or [])[1:])
307
+ return out
308
+
309
+
310
+ def update(entry: dict, root: str | None = None, was: str = "", dataset: int = 0) -> str:
311
+ """Write an edited entry over its own ``benchmark.json``; the path
312
+ written. A bundled entry is first copied in full to the user catalogue.
313
+ ``was`` is its key before the edit: for a user-only entry a new key moves
314
+ the folder, with everything in it, to the new name. Refuses, with what
315
+ is wrong, anything :func:`problems` names."""
316
+ was = was or entry.get("key", "")
317
+ bad = problems(entry, root, replacing=was, dataset=dataset)
318
+ if bad:
319
+ raise ValueError("; ".join(bad))
320
+ selected = catalog.entry_dir(was, root)
321
+ if selected is None or not os.path.isfile(os.path.join(selected, catalog.ENTRY)):
322
+ raise ValueError(f"benchmarks/{was}/ has no entry to edit")
323
+ old = catalog.writable_dir(was, root)
324
+ folder = os.path.join(os.path.dirname(old), entry["key"])
325
+ storage.ensure_writable(folder, catalog.ROOT)
326
+ if folder != old:
327
+ os.rename(old, folder) # problems() said the new name is free
328
+ path = os.path.join(folder, catalog.ENTRY)
329
+ _write(path, entry)
330
+ return path
331
+
332
+
333
+ def delete(key: str, root: str | None = None) -> str:
334
+ """Remove a user entry folder; removing an override restores the bundle.
335
+ Bundled-only entries cannot be deleted. Only an entry the form edits
336
+ (:func:`not_editable`) is removed here."""
337
+ import shutil
338
+ entry = entry_of(key, root)
339
+ if entry is None or not KEY.fullmatch(key):
340
+ raise ValueError(f"benchmarks/{key}/ has no entry")
341
+ why = not_editable(entry)
342
+ if why:
343
+ raise ValueError(f"not removed here: {why}")
344
+ folder = os.path.join(storage.writable_root(catalog.ROOT, root), catalog.BENCHMARKS, key)
345
+ storage.ensure_writable(folder, catalog.ROOT)
346
+ if not os.path.isdir(folder):
347
+ raise ValueError("the bundled catalogue is read-only; only user entries can be removed")
348
+ shutil.rmtree(folder)
349
+ return folder
350
+
351
+
352
+ def harbor_route(old: dict, values: dict) -> dict:
353
+ """``old`` - a benchmark run on the Nullius standard - with a Harbor route
354
+ added: the form's dataset appended to its datasets, ``harbor`` to its
355
+ harnesses, the limits its tasks declare when it had none. Everything
356
+ else the entry says stays: it is one benchmark, run either way, and one
357
+ card. The dataset's key is the form's, else ``<key>-harbor``."""
358
+ built = build(dict(values, key=old["key"], title=values.get("dataset_title") or old.get("title", "")))
359
+ d = built["datasets"][0]
360
+ d["key"] = (values.get("dataset_key") or "").strip() or f"{old['key']}-harbor"
361
+ d["title"] = (values.get("dataset_title") or "").strip() or f"{old.get('title', old['key'])} on Harbor"
362
+ out = dict(old)
363
+ out["harness"] = list(old.get("harness") or []) + ([] if "harbor" in (old.get("harness") or []) else ["harbor"])
364
+ out["datasets"] = list(old.get("datasets") or []) + [d]
365
+ if built.get("timeouts") and not old.get("timeouts"):
366
+ out["timeouts"] = built["timeouts"]
367
+ return out
@@ -0,0 +1,107 @@
1
+ """Where a dataset's data comes from, one row for each way the benchmark runs.
2
+
3
+ A benchmark that runs with Harbor and the Nullius standard reads its data two
4
+ ways: Harbor from its registry or hub, the standard from the upstream it pins
5
+ (a repository at a commit, or a file at a revision) or from the Hugging Face
6
+ dataset its driver downloads (``data-source.json``: ``repository`` and
7
+ ``revision``). Each row names the source, the pin and who fetches it - always
8
+ the harness, on a run, never the person - and the files a definition
9
+ checks by their sha256. Nothing here reads the network: it reads the
10
+ catalogue entry and the definitions beside it.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import os
16
+ import re
17
+ from typing import List, Optional
18
+
19
+ _GITHUB = re.compile(r"^https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$")
20
+ _HF = re.compile(r"^https://huggingface\.co/datasets/([^/]+/[^/]+)/resolve/([^/]+)/")
21
+ _HF_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
22
+ _REVISION = re.compile(r"^[^/\s]+$")
23
+
24
+
25
+ def _size(n) -> str:
26
+ if not isinstance(n, int) or n <= 0:
27
+ return ""
28
+ for unit, scale in (("GB", 10 ** 9), ("MB", 10 ** 6), ("kB", 10 ** 3)):
29
+ if n >= scale:
30
+ return f"{n / scale:.1f} {unit}".replace(".0 ", " ")
31
+ return f"{n} B"
32
+
33
+
34
+ def _repo(upstream: dict) -> dict:
35
+ """The upstream repository as a row's source: GitHub's owner/repo, linked at
36
+ the pinned commit; its release tag, when it has one, beside the commit."""
37
+ url, commit = upstream.get("repository", ""), upstream.get("tested", "")
38
+ m = _GITHUB.match(url)
39
+ pin = " · ".join(x for x in (upstream.get("tag"), commit[:7]) if x)
40
+ return {"from": "GitHub" if m else "Repository", "name": m.group(1) if m else url,
41
+ "href": f"{url.rstrip('/')}/tree/{commit}" if m and commit else url, "pin": pin}
42
+
43
+
44
+ def _files(defined: List[dict]) -> List[dict]:
45
+ """The files a definition's datasets name, each with its size when known."""
46
+ out = []
47
+ for d in defined:
48
+ where = d.get("path") or d.get("url") or ""
49
+ if where:
50
+ out.append({"name": d.get("path") or os.path.basename(where), "size": _size(d.get("bytes")),
51
+ "checked": bool(d.get("sha256"))})
52
+ return out
53
+
54
+
55
+ def _data_source(here) -> Optional[tuple]:
56
+ """The Hugging Face dataset a driver downloads itself, ``data-source.json``
57
+ beside its definition: (repository, revision), or None."""
58
+ if not here:
59
+ return None
60
+ try:
61
+ with open(os.path.join(here, "data-source.json")) as f:
62
+ src = json.load(f)
63
+ repo, rev = src["repository"], src["revision"]
64
+ except (OSError, ValueError, TypeError, KeyError):
65
+ return None
66
+ ok = isinstance(repo, str) and _HF_REPO.match(repo) and isinstance(rev, str) and _REVISION.match(rev)
67
+ return (repo, rev) if ok else None
68
+
69
+
70
+ def _own(upstream: dict, defined: List[dict], who: str, source: Optional[tuple] = None) -> dict:
71
+ """A row for a harness that reads the upstream itself: from Hugging Face when
72
+ its driver names a dataset there (``source``) or every file it names is
73
+ there, else from the repository it clones."""
74
+ hf = [_HF.match(d.get("url") or "") for d in defined]
75
+ if source or (defined and all(hf)):
76
+ name, rev = source or (hf[0].group(1), hf[0].group(2))
77
+ row = {"from": "Hugging Face", "name": name, "href": f"https://huggingface.co/datasets/{name}/tree/{rev}",
78
+ "pin": rev[:7], "by": f"{who} downloads it on the first run and keeps it"}
79
+ else:
80
+ row = dict(_repo(upstream), by=f"{who} fetches the repository at this commit on the first run and keeps it")
81
+ row["files"] = _files(defined)
82
+ return row
83
+
84
+
85
+ def _matching(defined: List[dict], key: str, only: bool) -> List[dict]:
86
+ """The datasets of a definition that are this catalogue dataset: by id
87
+ (``-`` and ``_`` alike), or all of them when the entry has a single dataset."""
88
+ norm = key.replace("-", "_")
89
+ hit = [d for d in defined if str(d.get("id", "")).replace("-", "_") == norm]
90
+ return hit or (list(defined) if only else [])
91
+
92
+
93
+ def of(dataset, harness: List[str], standard: Optional[dict] = None, only: bool = True) -> List[dict]:
94
+ """One row per way ``dataset`` runs, in the order the page names them
95
+ (Nullius, Harbor): ``way``, ``label``, ``from``, ``name``, ``href``,
96
+ ``pin``, ``by`` and ``files``. ``standard`` is the raw definition beside
97
+ the entry; ``only`` says the entry has one dataset."""
98
+ rows = []
99
+ if standard:
100
+ rows.append(dict(_own(standard.get("upstream") or {}, _matching(standard.get("datasets") or [], dataset.key, only), "Nullius",
101
+ _data_source(standard.get("_here"))),
102
+ way="standard", label="Nullius"))
103
+ if "harbor" in harness and (dataset.hub or dataset.registry):
104
+ rows.append({"way": "harbor", "label": "Harbor", "from": "Harbor hub" if dataset.hub else "Harbor registry",
105
+ "name": dataset.hub or dataset.registry, "href": "", "pin": "",
106
+ "by": "Harbor downloads it when a run needs it", "files": []})
107
+ return rows
@@ -0,0 +1,125 @@
1
+ """Whole-entry catalogue overlays, shared by the catalogue and the standard.
2
+
3
+ An explicit root is a single isolated tree. Otherwise the user's folder wins
4
+ over the bundled folder of the same name, including when its entry is invalid.
5
+ We never combine files from different copies of an entry.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ import shutil
11
+ import stat
12
+ import tempfile
13
+
14
+ from nullius import paths
15
+
16
+
17
+ def roots(bundled: str, root: str | None = None) -> tuple:
18
+ if root is not None:
19
+ return (os.fspath(root),)
20
+ user = paths.catalogue_dir()
21
+ return (bundled,) if os.path.realpath(user) == os.path.realpath(bundled) else (bundled, user)
22
+
23
+
24
+ def folders(group: str, bundled: str, root: str | None = None) -> dict:
25
+ """Selected entry folders, sorted by key; an empty folder still wins."""
26
+ found = {}
27
+ for base in roots(bundled, root):
28
+ for directory in (("features", "capabilities") if group == "capabilities" else (group,)):
29
+ parent = os.path.join(base, directory)
30
+ try:
31
+ names = os.listdir(parent)
32
+ except FileNotFoundError:
33
+ continue
34
+ for name in sorted(names):
35
+ folder = os.path.join(parent, name)
36
+ if name.startswith((".", "_")) or not os.path.isdir(folder):
37
+ continue
38
+ if os.listdir(folder) == ["__pycache__"]:
39
+ continue
40
+ found[name] = folder
41
+ return dict(sorted(found.items()))
42
+
43
+
44
+ def entry_dir(group: str, key: str, bundled: str, root: str | None = None) -> str | None:
45
+ return folders(group, bundled, root).get(key)
46
+
47
+
48
+ def writable_root(bundled: str, root: str | None = None) -> str:
49
+ """The destination, refusing paths that would mutate package resources."""
50
+ target = os.fspath(root) if root is not None else paths.catalogue_dir()
51
+ ensure_writable(target, bundled)
52
+ return target
53
+
54
+
55
+ def ensure_writable(target: str, bundled: str) -> None:
56
+ bundle = os.path.realpath(bundled)
57
+ try:
58
+ common = os.path.commonpath((os.path.realpath(target), bundle))
59
+ except ValueError: # different drives on Windows cannot overlap
60
+ return
61
+ if common == bundle:
62
+ raise ValueError("the bundled catalogue is read-only; use the user catalogue")
63
+
64
+
65
+ def writable_dir(group: str, key: str, bundled: str, root: str | None = None) -> str:
66
+ """Copy a selected bundled folder in full before returning its user copy.
67
+
68
+ An existing user folder is left intact. Staging keeps an interrupted copy
69
+ from shadowing a bundled entry with half of its files.
70
+ """
71
+ if not key or key in (".", "..") or os.path.basename(key) != key:
72
+ raise ValueError("an entry key is a folder name")
73
+ target = os.path.join(writable_root(bundled, root), group, key)
74
+ ensure_writable(target, bundled)
75
+ source = entry_dir(group, key, bundled, root)
76
+ if source is None or os.path.realpath(source) == os.path.realpath(target):
77
+ return target
78
+ parent = os.path.dirname(target)
79
+ os.makedirs(parent, exist_ok=True)
80
+ with tempfile.TemporaryDirectory(prefix=".catalogue-", dir=parent) as stage:
81
+ copied = os.path.join(stage, key)
82
+ shutil.copytree(source, copied)
83
+ # Package resources may be read-only. The copy belongs to its user.
84
+ for folder, _, files in os.walk(copied):
85
+ os.chmod(folder, os.stat(folder).st_mode | stat.S_IWUSR)
86
+ for name in files:
87
+ path = os.path.join(folder, name)
88
+ os.chmod(path, os.stat(path).st_mode | stat.S_IWUSR)
89
+ os.rename(copied, target)
90
+ return target
91
+
92
+
93
+ def _restore_targets(group: str, key: str, bundled: str) -> list[str]:
94
+ if group not in ("benchmarks", "agents", "capabilities"):
95
+ raise ValueError("unknown catalogue")
96
+ if not key or key in (".", "..") or "/" in key or "\\" in key:
97
+ raise ValueError("an entry key is a folder name")
98
+ if entry_dir(group, key, bundled, root=bundled) is None:
99
+ raise ValueError("this entry has no bundled copy")
100
+ user = writable_root(bundled)
101
+ groups = ("features", "capabilities") if group == "capabilities" else (group,)
102
+ targets = [os.path.join(user, directory, key) for directory in groups
103
+ if os.path.isdir(os.path.join(user, directory, key))]
104
+ if not targets:
105
+ raise ValueError("this entry has no user copy")
106
+ for target in targets:
107
+ ensure_writable(target, bundled)
108
+ return targets
109
+
110
+
111
+ def can_restore(group: str, key: str, bundled: str) -> bool:
112
+ """Whether removing the user override can reveal a bundled entry."""
113
+ try:
114
+ return bool(_restore_targets(group, key, bundled))
115
+ except (OSError, ValueError):
116
+ return False
117
+
118
+
119
+ def use_bundled(group: str, key: str, bundled: str) -> None:
120
+ """Remove complete user overrides, including a capability's legacy alias."""
121
+ for target in _restore_targets(group, key, bundled):
122
+ if os.path.islink(target):
123
+ os.unlink(target)
124
+ else:
125
+ shutil.rmtree(target)