nullius-eval 0.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. _nullius_app/.env.example +54 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +16 -0
  5. _nullius_app/static/brand/favicon.svg +1 -0
  6. _nullius_app/static/brand/mark-dark.svg +1 -0
  7. _nullius_app/static/brand/marks.svg +34 -0
  8. _nullius_app/static/brand/tokens.json +51 -0
  9. _nullius_app/static/brand/typesafe.png +0 -0
  10. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  11. _nullius_app/static/builder/command-builder-core.js +744 -0
  12. _nullius_app/static/builder/command-builder.css +248 -0
  13. _nullius_app/static/builder/command-builder.js +994 -0
  14. _nullius_app/static/builder/model-picker.js +113 -0
  15. _nullius_app/static/catalogue/add.js +46 -0
  16. _nullius_app/static/catalogue/agents/README.md +19 -0
  17. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  18. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  19. _nullius_app/static/catalogue/agents/marks.json +8 -0
  20. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  21. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  22. _nullius_app/static/catalogue/catalogue.css +453 -0
  23. _nullius_app/static/catalogue/catalogue.js +87 -0
  24. _nullius_app/static/catalogue/marks.svg +25 -0
  25. _nullius_app/static/catalogue/new.js +78 -0
  26. _nullius_app/static/catalogue/versions.js +27 -0
  27. _nullius_app/static/compare/comparison.css +117 -0
  28. _nullius_app/static/controls.js +117 -0
  29. _nullius_app/static/disclosures.js +70 -0
  30. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  31. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  32. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  33. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  36. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/README.md +11 -0
  38. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  39. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  40. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  41. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  42. _nullius_app/static/fonts.css +49 -0
  43. _nullius_app/static/notifications.js +34 -0
  44. _nullius_app/static/onboarding.css +78 -0
  45. _nullius_app/static/onboarding.js +158 -0
  46. _nullius_app/static/runs/board.css +354 -0
  47. _nullius_app/static/runs/board.js +127 -0
  48. _nullius_app/static/runs/launches.css +87 -0
  49. _nullius_app/static/runs/live-board.css +186 -0
  50. _nullius_app/static/runs/live-board.js +387 -0
  51. _nullius_app/static/runs/path-suggestions.js +119 -0
  52. _nullius_app/static/runs/start.css +38 -0
  53. _nullius_app/static/settings/autosave.js +145 -0
  54. _nullius_app/static/settings/judges.js +61 -0
  55. _nullius_app/static/settings/modal.css +32 -0
  56. _nullius_app/static/settings/modal.js +91 -0
  57. _nullius_app/static/settings/navigation.js +55 -0
  58. _nullius_app/static/settings/providers.js +78 -0
  59. _nullius_app/static/settings/sandbox.js +83 -0
  60. _nullius_app/static/settings/settings.css +229 -0
  61. _nullius_app/static/style.css +1751 -0
  62. _nullius_app/static/traces/conversation.css +49 -0
  63. _nullius_app/static/traces/conversation.js +215 -0
  64. _nullius_app/templates/agents/entry.html +145 -0
  65. _nullius_app/templates/agents/kits.html +55 -0
  66. _nullius_app/templates/agents/standard.html +58 -0
  67. _nullius_app/templates/base.html +477 -0
  68. _nullius_app/templates/builder/new_run.html +55 -0
  69. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  70. _nullius_app/templates/catalogue/benchmark.html +301 -0
  71. _nullius_app/templates/catalogue/benchmarks.html +92 -0
  72. _nullius_app/templates/catalogue/new.html +141 -0
  73. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  74. _nullius_app/templates/compare/_detail.html +37 -0
  75. _nullius_app/templates/compare/compare.html +298 -0
  76. _nullius_app/templates/compare/compare_picker.html +51 -0
  77. _nullius_app/templates/compare/compare_task.html +329 -0
  78. _nullius_app/templates/runs/_execution.html +131 -0
  79. _nullius_app/templates/runs/_runtime.html +5 -0
  80. _nullius_app/templates/runs/ascii-mark.html +485 -0
  81. _nullius_app/templates/runs/feasibility.html +127 -0
  82. _nullius_app/templates/runs/index.html +609 -0
  83. _nullius_app/templates/runs/job.html +446 -0
  84. _nullius_app/templates/runs/preparing.html +14 -0
  85. _nullius_app/templates/runs/reconcile.html +124 -0
  86. _nullius_app/templates/runs/setup.html +272 -0
  87. _nullius_app/templates/runs/start.html +51 -0
  88. _nullius_app/templates/settings/settings.html +508 -0
  89. _nullius_app/templates/traces/_charts.html +165 -0
  90. _nullius_app/templates/traces/_conversation.html +37 -0
  91. _nullius_app/templates/traces/_judgement.html +72 -0
  92. _nullius_app/templates/traces/artifact.html +15 -0
  93. _nullius_app/templates/traces/trial.html +838 -0
  94. _nullius_app/templates/workspaces/workspaces.html +54 -0
  95. _nullius_app/web/__init__.py +1 -0
  96. _nullius_app/web/agents.py +230 -0
  97. _nullius_app/web/builder.py +392 -0
  98. _nullius_app/web/catalogue.py +374 -0
  99. _nullius_app/web/compare.py +489 -0
  100. _nullius_app/web/core.py +17 -0
  101. _nullius_app/web/execute.py +205 -0
  102. _nullius_app/web/feasibility.py +60 -0
  103. _nullius_app/web/filters.py +224 -0
  104. _nullius_app/web/judge.py +623 -0
  105. _nullius_app/web/judge_tasks.py +33 -0
  106. _nullius_app/web/onboarding.py +20 -0
  107. _nullius_app/web/reconcile.py +47 -0
  108. _nullius_app/web/runs.py +461 -0
  109. _nullius_app/web/settings.py +210 -0
  110. _nullius_app/web/shared.py +468 -0
  111. _nullius_app/web/traces.py +306 -0
  112. _nullius_app/web/workspaces.py +84 -0
  113. nullius/__init__.py +32 -0
  114. nullius/__main__.py +5 -0
  115. nullius/catalogue/__init__.py +0 -0
  116. nullius/catalogue/agents/codex/agent.json +202 -0
  117. nullius/catalogue/agents/codex/config.toml +25 -0
  118. nullius/catalogue/agents/goose/agent.json +120 -0
  119. nullius/catalogue/agents/goose/config.yaml +20 -0
  120. nullius/catalogue/agents/jiuwenswarm/agent.json +115 -0
  121. nullius/catalogue/agents/jiuwenswarm/agent.py +316 -0
  122. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  123. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  124. nullius/catalogue/agents/opencode/agent.json +124 -0
  125. nullius/catalogue/agents/opencode/opencode.json +32 -0
  126. nullius/catalogue/agents/reference/agent.json +89 -0
  127. nullius/catalogue/agents/reference/agent.py +209 -0
  128. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  129. nullius/catalogue/agents/terminus-2/run.py +139 -0
  130. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  131. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  132. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  133. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  134. nullius/catalogue/benchmarks/humaneval/benchmark.json +37 -0
  135. nullius/catalogue/benchmarks/humaneval/driver.py +208 -0
  136. nullius/catalogue/benchmarks/humaneval/standard.json +114 -0
  137. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  138. nullius/catalogue/benchmarks/locomo/benchmark.json +42 -0
  139. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  140. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  141. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  142. nullius/catalogue/benchmarks/longmemeval/benchmark.json +64 -0
  143. nullius/catalogue/benchmarks/longmemeval/driver.py +323 -0
  144. nullius/catalogue/benchmarks/longmemeval/standard.json +551 -0
  145. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +74 -0
  146. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  147. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +105 -0
  148. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  149. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  150. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  151. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  152. nullius/catalogue/benchmarks/tau3-bench/driver.py +605 -0
  153. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +111 -0
  154. nullius/catalogue/benchmarks/tau3-bench/standard.json +495 -0
  155. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  156. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +70 -0
  157. nullius/catalogue/benchmarks/terminal-bench/driver.py +189 -0
  158. nullius/catalogue/benchmarks/terminal-bench/standard.json +160 -0
  159. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  160. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  161. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  162. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  163. nullius/catalogue/capabilities/mem0/capability.py +170 -0
  164. nullius/catalogue/catalog.py +413 -0
  165. nullius/catalogue/citation.py +116 -0
  166. nullius/catalogue/harbor_entry.py +367 -0
  167. nullius/catalogue/sources.py +85 -0
  168. nullius/catalogue/storage.py +125 -0
  169. nullius/cli.py +83 -0
  170. nullius/compare/__init__.py +0 -0
  171. nullius/compare/cli.py +265 -0
  172. nullius/compare/comparison.py +184 -0
  173. nullius/execution.py +1624 -0
  174. nullius/gitrefs.py +128 -0
  175. nullius/harbor/__init__.py +0 -0
  176. nullius/harbor/feed.py +228 -0
  177. nullius/harbor/source.py +582 -0
  178. nullius/hosts.py +339 -0
  179. nullius/importers/__init__.py +140 -0
  180. nullius/importers/atif.py +387 -0
  181. nullius/importers/cli.py +135 -0
  182. nullius/importers/otel.py +274 -0
  183. nullius/judge/__init__.py +14 -0
  184. nullius/judge/batch.py +122 -0
  185. nullius/judge/client.py +416 -0
  186. nullius/judge/floors.json +86 -0
  187. nullius/judge/lanes.py +89 -0
  188. nullius/judge/questions.py +309 -0
  189. nullius/judge/reconcile.py +172 -0
  190. nullius/judge/state.py +224 -0
  191. nullius/judge/store.py +167 -0
  192. nullius/migrate.py +268 -0
  193. nullius/onboarding.py +112 -0
  194. nullius/paths.py +84 -0
  195. nullius/prerequisites.py +111 -0
  196. nullius/progress.py +294 -0
  197. nullius/providers.py +115 -0
  198. nullius/recipes/__init__.py +0 -0
  199. nullius/recipes/launch.py +78 -0
  200. nullius/redact.py +72 -0
  201. nullius/runs/__init__.py +0 -0
  202. nullius/runs/artifacts.py +67 -0
  203. nullius/runs/classify.py +189 -0
  204. nullius/runs/cli.py +316 -0
  205. nullius/runs/feasibility.py +442 -0
  206. nullius/runs/launches.py +139 -0
  207. nullius/runs/live.py +705 -0
  208. nullius/runs/parse.py +991 -0
  209. nullius/runs/verifier.py +241 -0
  210. nullius/sandbox.py +146 -0
  211. nullius/secrets.py +192 -0
  212. nullius/settings.py +456 -0
  213. nullius/standard/__init__.py +15 -0
  214. nullius/standard/acp.py +301 -0
  215. nullius/standard/cancellation.py +19 -0
  216. nullius/standard/cli.py +650 -0
  217. nullius/standard/conform.py +149 -0
  218. nullius/standard/container.py +1381 -0
  219. nullius/standard/definitions.py +1577 -0
  220. nullius/standard/egress.py +192 -0
  221. nullius/standard/fidelity.py +255 -0
  222. nullius/standard/forward.py +92 -0
  223. nullius/standard/hostreq.py +65 -0
  224. nullius/standard/howto.py +220 -0
  225. nullius/standard/install.py +397 -0
  226. nullius/standard/job.py +541 -0
  227. nullius/standard/jsonl.py +129 -0
  228. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  229. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  230. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  231. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  232. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  233. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  234. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  235. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  236. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  237. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  238. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  239. nullius/standard/match.py +66 -0
  240. nullius/standard/mcp_bridge.py +62 -0
  241. nullius/standard/mcp_machine.py +193 -0
  242. nullius/standard/merge.py +209 -0
  243. nullius/standard/names.py +30 -0
  244. nullius/standard/payload.py +255 -0
  245. nullius/standard/ports.py +64 -0
  246. nullius/standard/prove.py +357 -0
  247. nullius/standard/recipe.py +663 -0
  248. nullius/standard/relay.py +629 -0
  249. nullius/standard/run.py +1367 -0
  250. nullius/standard/shard.py +71 -0
  251. nullius/traces/__init__.py +0 -0
  252. nullius/traces/behavior.py +131 -0
  253. nullius/traces/charts.py +898 -0
  254. nullius/traces/conversation.py +159 -0
  255. nullius/traces/otel.py +608 -0
  256. nullius/traces/trace.py +1477 -0
  257. nullius/traces/trajectory.py +586 -0
  258. nullius/workspace.py +409 -0
  259. nullius/workspace_cli.py +119 -0
  260. nullius_eval-0.1.0rc1.dist-info/METADATA +93 -0
  261. nullius_eval-0.1.0rc1.dist-info/RECORD +268 -0
  262. nullius_eval-0.1.0rc1.dist-info/WHEEL +5 -0
  263. nullius_eval-0.1.0rc1.dist-info/entry_points.txt +2 -0
  264. nullius_eval-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
  265. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  266. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  267. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  268. nullius_eval-0.1.0rc1.dist-info/top_level.txt +2 -0
@@ -0,0 +1,220 @@
1
+ """How to add an agent or a capability, as the Agents and Capabilities tabs show it:
2
+ the commands to type and the files to write, in order.
3
+
4
+ Each step is ``{title, text, code, lang, file}``: ``code`` is what is typed or
5
+ written, ``file`` where it goes. The definitions shown are real: the suite
6
+ checks every ``agent.json`` and ``capability.json`` here with
7
+ ``definitions.problems``, so the page never shows one that would be refused.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import os
13
+ import shlex
14
+
15
+ from nullius import paths
16
+
17
+ CLI = "nullius standard"
18
+
19
+ AGENT_JSON = {
20
+ "schema": "nullius.agent/v1",
21
+ "key": "my-agent",
22
+ "label": "My agent",
23
+ "summary": "What the agent is, and how it is driven: its ACP adapter, its model through the relay.",
24
+ "upstream": {"package": "@acme/my-agent-acp", "version": "1.4.2", "license": "MIT"},
25
+ "install": {"npm": ["@acme/my-agent-acp@1.4.2"]},
26
+ "start": {"acp": ["{install}/node_modules/.bin/my-agent-acp"]},
27
+ "model": {
28
+ "apis": ["chat"],
29
+ "env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "OPENAI_MODEL": "{model}"},
30
+ },
31
+ "offers": [],
32
+ "proven": {},
33
+ }
34
+
35
+ # What an agent may add: an option a run can set (its default the vendor's own,
36
+ # filled into a template as {opt_<name>}), and its own non-interactive command,
37
+ # for cli fidelity to compare with ACP.
38
+ AGENT_OPTIONS = {
39
+ "model": {
40
+ "apis": ["chat"],
41
+ "env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "OPENAI_MODEL": "{model}",
42
+ "MY_AGENT_REASONING": "{opt_reasoning_effort}"},
43
+ },
44
+ "options": {"reasoning_effort": {
45
+ "summary": "How hard the model reasons. The default is the agent's own, as its documentation or its source says.",
46
+ "values": ["low", "medium", "high"], "default": "medium"}},
47
+ "fidelity": {"command": ["{install}/node_modules/.bin/my-agent", "run", "--yes", "--", "{prompt}"]},
48
+ }
49
+
50
+ FEATURE_JSON = {
51
+ "schema": "nullius.capability/v1",
52
+ "key": "my-memory",
53
+ "label": "My memory",
54
+ "summary": "What the memory keeps, and what it gives back for a question.",
55
+ "upstream": {"package": "my-memory", "version": "0.3.1", "license": "MIT"},
56
+ "kind": "memory",
57
+ "install": {"pip": ["my-memory==0.3.1"]},
58
+ "start": {"command": ["{install}/.venv/bin/python", "{here}/capability.py"]},
59
+ "offers": [],
60
+ "proven": {},
61
+ }
62
+
63
+ # The same memory installed from its own repository, calling a model: the parts that change.
64
+ FEATURE_FROM_REPO = {
65
+ "upstream": {"repository": "https://github.com/acme/my-memory", "tested": "0123456789abcdef0123456789abcdef01234567",
66
+ "license": "MIT"},
67
+ "install": {"python": "3.12", "project": "packages/memory", "extras": ["embeddings"],
68
+ "indexes": [{"url": "https://download.pytorch.org/whl/cpu", "packages": ["torch"]}],
69
+ "prefetch": ["BAAI/bge-small-en-v1.5"]},
70
+ "env": {"HF_HUB_OFFLINE": "1"},
71
+ "model": {"apis": ["chat"], "env": {"API_BASE": "{base_url}", "API_KEY": "{key}", "MODEL_NAME": "{model}"}},
72
+ }
73
+
74
+ FEATURE_PY = '''"""My memory behind cap.memory/v1: one JSON line in, one JSON line out."""
75
+ import json
76
+ import sys
77
+
78
+ from my_memory import Store # your library
79
+
80
+ OUT, sys.stdout = sys.stdout, sys.stderr # stdout: the replies only
81
+ store = Store() # one process per question
82
+
83
+ for line in sys.stdin:
84
+ req = json.loads(line)
85
+ try:
86
+ if req["op"] == "add": # session: {id, date, turns, text}
87
+ s = req["session"]
88
+ store.add(s["text"], id=s["id"], date=s["date"])
89
+ out = {}
90
+ elif req["op"] == "recall": # question: the text
91
+ hits = store.search(req["question"], k=5)
92
+ out = {"memory": "\\n\\n".join(h.text for h in hits)}
93
+ else:
94
+ out = {"error": "unknown op " + req["op"]}
95
+ except Exception as e:
96
+ out = {"error": f"{type(e).__name__}: {e}"}
97
+ print(json.dumps(out), file=OUT, flush=True)
98
+ '''
99
+
100
+ TRY_FEATURE = """printf '%s\\n' \\
101
+ '{"op":"add","session":{"id":"s1","date":"2026-01-01","turns":[{"role":"user","text":"My sister moved to Lisbon."}],"text":"user: My sister moved to Lisbon."}}' \\
102
+ '{"op":"recall","question":"Where did my sister move?"}' \\
103
+ | python3 {wrapper}
104
+ # {}
105
+ # {"memory": "user: My sister moved to Lisbon."}"""
106
+
107
+
108
+ def _json(d) -> str:
109
+ return json.dumps(d, indent=2)
110
+
111
+
112
+ def steps(kind: str) -> list:
113
+ agent = os.path.join(paths.catalogue_dir(), "agents", "my-agent", "agent.json")
114
+ capability = os.path.join(paths.catalogue_dir(), "capabilities", "my-memory")
115
+ definition = os.path.join(capability, "capability.json")
116
+ wrapper = os.path.join(capability, "capability.py")
117
+ if kind == "agent":
118
+ return [
119
+ {"title": "Create its folder and definition", "lang": "shell",
120
+ "text": "The key is lower case letters, digits and dashes; it names the folder in your catalogue.",
121
+ "code": f"{CLI} new agent my-agent\n# wrote {agent}"},
122
+ {"title": "Fill in agent.json", "lang": "json", "file": agent,
123
+ "text": "<b>install</b> pins the package (npm <code>name@version</code>, or pip <code>name==version</code>). "
124
+ "<b>start.acp</b> is the program that speaks ACP on stdin and stdout; <code>{install}</code> is where it was installed. "
125
+ "<b>model.env</b> tells it where its model is: the relay's address, a placeholder key and the run's model - "
126
+ "use the variable names the agent itself reads. Leave <b>offers</b> and <b>proven</b> empty: step 4 writes them.",
127
+ "code": _json(AGENT_JSON)},
128
+ {"title": "Check the definition", "lang": "shell", "text": "Every field is checked; a problem is named with its field.",
129
+ "code": f"{CLI} check\n# ok"},
130
+ {"title": "Prove its ports", "lang": "shell",
131
+ "text": "Runs the kit of every port it can take and writes what passed into agent.json. "
132
+ "<code>--unsandboxed</code> is your yes to running the agent on this machine as you. Or use <i>Prove its ports</i> on its page.",
133
+ "code": f"{CLI} conform my-agent --model gpt-5-mini --unsandboxed\n"
134
+ "# agent.task/v1: passed 3/3\n# agent.workspace/v1: passed 2/2\n# agent.tools/v1: passed 2/2\n# agent.conversation/v1: passed 2/2"},
135
+ {"title": "Make sure its calls went through the relay", "lang": "shell",
136
+ "text": "Each kit trial records the agent's model calls. An empty file means the agent called its vendor directly: "
137
+ "it ignored <code>model.env</code>, and needs the relay declared as a provider in its own config file "
138
+ "(<code>model.files</code>).",
139
+ "code": "cat jobs_standard/*__kit__agent-task-v1__my-agent/*/agent/calls.jsonl | head -3"},
140
+ {"title": "Its options, and its own command", "lang": "json", "file": agent,
141
+ "text": "<b>options</b> are what a run may set of the agent's behaviour (<code>--option name=value</code>), each with its "
142
+ "values and a <b>default</b> that is the vendor's own, written into its variables or config file as "
143
+ "<code>{opt_name}</code>; the job records every value. Declare the settings other agents declare "
144
+ "(<code>reasoning_effort</code>) when it has them. Write a setting explicitly when the adapter and the "
145
+ "agent's own command would each fill it in their own way. <b>fidelity.command</b> is the agent's own "
146
+ "non-interactive command, carrying <code>{prompt}</code>.",
147
+ "code": _json(AGENT_OPTIONS)},
148
+ {"title": "Show ACP sends what its own command sends", "lang": "shell",
149
+ "text": "Runs the workspace kit's two units over ACP and through <b>fidelity.command</b>, twice each, and compares "
150
+ "every field of the requests its model was sent; what differs between two runs of one way is noise. "
151
+ "The verdict is written into agent.json beside the kits' proofs.",
152
+ "code": f"{CLI} fidelity my-agent --model gpt-5-mini --unsandboxed\n# equal: 20 fields\n# verdict: same"},
153
+ {"title": "Run a benchmark with it", "lang": "shell",
154
+ "text": "Or from the Run builder: runner <i>Nullius</i>, a benchmark, then your agent on the Agent step.",
155
+ "code": f"{CLI} run BENCHMARK my-agent --model gpt-5-mini --limit 2 --unsandboxed"},
156
+ {"title": "A new release: add it as a version", "lang": "shell",
157
+ "text": "The older version stays, with its proofs; the new one is offered once its own kits pass. "
158
+ "Check the pins it was given (a package that moved changes its name too), then prove it. "
159
+ "Set <b>default</b> in agent.json to make it the one a bare <code>my-agent</code> names.",
160
+ "code": f"{CLI} new version my-agent 1.5.0\n# wrote {agent}: my-agent@1.5.0 beside 1.4.2\n"
161
+ f"{CLI} conform my-agent@1.5.0 --model gpt-5-mini --unsandboxed"},
162
+ ]
163
+ return [
164
+ {"title": "Create its folder, definition and wrapper", "lang": "shell",
165
+ "text": "The key is lower case letters, digits and dashes; it names the folder in your catalogue.",
166
+ "code": f"{CLI} new capability my-memory\n# wrote {definition}\n# wrote {wrapper}"},
167
+ {"title": "Write capability.py around your memory", "lang": "python", "file": wrapper,
168
+ "text": "Nullius sends <code>add</code> once per session of the history, then <code>recall</code> with the question, "
169
+ "and gives what comes back to the benchmark's reader. Replace the three marked calls with your library's.",
170
+ "code": FEATURE_PY},
171
+ {"title": "Try it by hand", "lang": "shell", "text": "Two lines in, two lines out, before Nullius is involved (with your library installed).",
172
+ "code": TRY_FEATURE.replace("{wrapper}", shlex.quote(wrapper))},
173
+ {"title": "Fill in capability.json", "lang": "json", "file": definition,
174
+ "text": "<b>install.pip</b> pins its packages; <b>start.command</b> runs the wrapper in that environment. "
175
+ "Leave <b>offers</b> and <b>proven</b> empty: step 6 writes them.",
176
+ "code": _json(FEATURE_JSON)},
177
+ {"title": "Or: installed from its repository, calling a model", "lang": "json",
178
+ "text": "Replace <b>upstream</b> and <b>install</b> with these when the memory lives in a repository: it is cloned at that "
179
+ "commit and its Python project installed (CPU torch first, the embedder fetched once). A memory that calls a model "
180
+ "gets a <b>model</b> block with the variable names it reads: its calls then go through the relay, as an agent's.",
181
+ "code": _json(FEATURE_FROM_REPO)},
182
+ {"title": "Check, then prove its port", "lang": "shell", "text": "Or use <i>Prove its ports</i> on its page.",
183
+ "code": f"{CLI} check\n# ok\n{CLI} conform my-memory --model gpt-5-mini\n# cap.memory/v1: passed 2/2"},
184
+ {"title": "Run a capability benchmark with it", "lang": "shell",
185
+ "text": "Or from the Run builder: runner <i>Nullius</i>, a benchmark that takes its port, then your capability on "
186
+ "the step named after its kind (Memory for a memory).",
187
+ "code": f"{CLI} run BENCHMARK my-memory --model gpt-5-mini --limit 10"},
188
+ ]
189
+
190
+
191
+ # The prompt the Add dialog builds for a coding assistant - any of them, so it
192
+ # names a skill's file as well as its command. Three slots, filled on the page
193
+ # with what was typed: {name}, {source}, {remark} - what the assistant should
194
+ # know beyond what it reads from the source; the Remark line is left out when
195
+ # there is none (``static/catalogue/add.js``).
196
+ _KINDS = ("benchmark", "agent", "capability")
197
+
198
+
199
+ def prompt(kind: str) -> str:
200
+ if kind not in _KINDS:
201
+ raise ValueError(f"unknown catalogue kind: {kind}")
202
+ folder = "capabilities" if kind == "capability" else kind + "s"
203
+ target = os.path.join(paths.catalogue_dir(), folder, "<key>")
204
+ specific = {
205
+ "benchmark": "First ask me how it should run: Harbor or the Nullius standard (its Original setup). Recommend a route.\n",
206
+ "agent": "Ask before running unsandboxed. Verify native capabilities and supported extensions.\n",
207
+ "capability": "Verify its callable operations and proven ports.\n",
208
+ }[kind]
209
+ return (f'Add the {kind} "{{name}}" to Nullius.\nSource: {{source}}\nRemark: {{remark}}\n\n'
210
+ f"Follow /import-{kind}, or .claude/skills/import-{kind}/SKILL.md in the source checkout, "
211
+ "with docs/standard.md and AGENTS.md. If unavailable, ask for the checkout.\n"
212
+ f"Write to {target}, even when the skill names package paths. "
213
+ "For an existing bundled entry, copy its whole folder there first; never edit installed files. "
214
+ f"On another machine, use app/nullius/catalogue/{folder}/<key> in a checkout and tell me to update Nullius.\n"
215
+ "Use pinned upstream code; ask before implementing behavior of our own. "
216
+ f"{specific}Report what fits, what passed, and what is blocked or untested.")
217
+
218
+
219
+ def prompts() -> dict:
220
+ return {kind: prompt(kind) for kind in _KINDS}
@@ -0,0 +1,397 @@
1
+ """Where what a run needs is installed, and installing it once.
2
+
3
+ Everything goes under ``STANDARD_DIR`` (``standard/`` in the persistent user data directory):
4
+
5
+ - ``agents/<key>/<digest>/`` and ``capabilities/<key>/<digest>/``: the packages a
6
+ definition's ``install`` names (``npm`` into ``node_modules``, ``pip`` and
7
+ ``wheels`` - a wheel by its URL, checked by its sha256 - into a ``uv`` environment), the digest taken over that list so a re-pin installs
8
+ beside the old one rather than over it;
9
+ - ``drivers/<key>/<digest>/``: a benchmark's ``driver.install`` the same way -
10
+ the environment its upstream's own code needs, which its driver imports
11
+ (``driver_argv``), never this application's and never an agent's. Its
12
+ ``project`` is installed from the benchmark's upstream clone (below), the
13
+ same commit, rather than fetched a second time;
14
+ - ``upstreams/<key>/<commit>/``: a benchmark's upstream repository at its
15
+ tested commit;
16
+ - ``datasets/<sha256>/<file>``: a dataset served outside its upstream, under
17
+ ``STANDARD_DATA_DIR`` instead when that is set - datasets can be gigabytes
18
+ where installs are not, so they may sit on a bigger disk than the installs.
19
+
20
+ A folder is complete when its ``.installed`` marker is; a failed install
21
+ leaves no marker and is tried again from scratch. Two runs installing the same
22
+ thing at once take turns (a lock file beside the folder): the second finds the
23
+ first one's marker rather than removing its half-made folder. A dataset file is checked
24
+ against its sha256 before a run reads it.
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import contextlib
29
+ import fcntl
30
+ import hashlib
31
+ import json
32
+ import os
33
+ import re
34
+ import shutil
35
+ import subprocess
36
+ from typing import Callable, Dict, List, Optional
37
+
38
+ from nullius.standard import definitions
39
+
40
+ from nullius import paths
41
+
42
+ HOME = os.environ.get("STANDARD_DIR") or os.path.join(paths.data_dir(), "standard")
43
+ MARK = ".installed"
44
+
45
+
46
+ def data_dir(home: str = HOME, env=os.environ) -> str:
47
+ """Where downloaded datasets are kept: ``STANDARD_DATA_DIR``, else
48
+ ``datasets/`` in the standard's home. A dataset already downloaded
49
+ elsewhere is not moved: setting the variable downloads it once more."""
50
+ return env.get("STANDARD_DATA_DIR") or os.path.join(home, "datasets")
51
+
52
+
53
+ class InstallFailed(Exception):
54
+ pass
55
+
56
+
57
+ # A variable an install step is never given: a package's install script runs
58
+ # as published, and a run's process holds the model key (under --from-run as
59
+ # OPENAI_API_KEY, API_TOKEN, LLM_API_KEY) - the relay is the only holder meant.
60
+ SECRET = re.compile(r"(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?)$", re.I) # the last word: GIT_CONFIG_KEY_0 is git's own
61
+
62
+
63
+ def install_env(env=None) -> dict:
64
+ """This process's environment less every variable named like a secret."""
65
+ return {k: v for k, v in (os.environ if env is None else env).items() if not SECRET.search(k)}
66
+
67
+
68
+ def git_env() -> dict:
69
+ """What a fetch of an upstream runs with: the installer's environment and
70
+ no prompt. A private repository - a fork of the user's - is fetched with
71
+ this machine's own git credentials (its credential helper, its ssh agent,
72
+ in its HOME); without them the fetch fails, said, rather than waiting on a
73
+ terminal for a password."""
74
+ return dict(install_env(), GIT_TERMINAL_PROMPT="0")
75
+
76
+
77
+ def _run(cmd: List[str], cwd: str, log: str, env: Optional[dict] = None):
78
+ with open(log, "a") as f:
79
+ f.write("$ " + " ".join(cmd) + "\n")
80
+ f.flush()
81
+ try:
82
+ r = subprocess.run(cmd, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, env=install_env() if env is None else env)
83
+ except FileNotFoundError:
84
+ raise InstallFailed(f"{cmd[0]} is not on this machine")
85
+ if r.returncode != 0:
86
+ with open(log, errors="replace") as f:
87
+ tail = f.read().strip().splitlines()[-3:]
88
+ raise InstallFailed(f"{' '.join(cmd[:3])}...: exit {r.returncode}: {' | '.join(tail)[-300:]}")
89
+
90
+
91
+ @contextlib.contextmanager
92
+ def _locked(where: str):
93
+ os.makedirs(os.path.dirname(where), exist_ok=True)
94
+ with open(where + ".lock", "w") as f:
95
+ fcntl.flock(f, fcntl.LOCK_EX)
96
+ try:
97
+ yield
98
+ finally:
99
+ fcntl.flock(f, fcntl.LOCK_UN)
100
+
101
+
102
+ def _digest(obj) -> str:
103
+ return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest()[:12]
104
+
105
+
106
+ def packages(d: dict, home: str = HOME, npm_in: Optional[Callable[[List[str], str], List[str]]] = None,
107
+ src: Optional[str] = None) -> str:
108
+ """The folder an agent's or a capability's packages are installed in,
109
+ installing them first when they are not. A definition installing nothing
110
+ gets an empty folder all the same, so ``{install}`` always names one.
111
+
112
+ ``npm_in``: how to run ``npm`` where this machine has none - the docker
113
+ backend's base image, which carries Node (``container.install_argv``):
114
+ given the command and the folder, the argument list that runs it there.
115
+ A host is not asked for Node to run an agent it will only start in a
116
+ container. Without it, a machine with no ``npm`` is refused, said.
117
+
118
+ ``src``: a checkout of the definition's upstream at ``upstream.tested``
119
+ already on this machine, which ``project`` is installed from instead of
120
+ a clone of its own (a benchmark's driver: its upstream clone)."""
121
+ inst = d.get("install") or {}
122
+ up = (d.get("upstream") or {}) if inst.get("project") else {}
123
+ where = packages_dir(d, home)
124
+ if os.path.exists(os.path.join(where, MARK)):
125
+ return where
126
+ with _locked(where):
127
+ return _packages(inst, where, up, npm_in, src)
128
+
129
+
130
+ def packages_dir(d: dict, home: str = HOME) -> str:
131
+ """Where ``packages`` installs a definition's packages, installed or not."""
132
+ inst = d.get("install") or {}
133
+ kind = {"agent": "agents", "driver": "drivers"}.get(d.get("_kind"), "capabilities")
134
+ up = (d.get("upstream") or {}) if inst.get("project") else {}
135
+ return os.path.join(home, kind, d["key"], _digest(dict(inst, _at=up.get("tested")) if up else inst))
136
+
137
+
138
+ def ready(where: Optional[str]) -> bool:
139
+ """Whether an install folder is complete (its marker is there): a run says
140
+ the stage was there already rather than that it took no time."""
141
+ return bool(where) and os.path.exists(os.path.join(where, MARK))
142
+
143
+
144
+ HF = "hf" # inside an install: the models prefetched from the hub (HF_HUB_CACHE at run time)
145
+ WHEELS = "wheels" # inside an install: the wheels it names by URL, as downloaded
146
+ SRC = "src" # inside an install: the definition's upstream, at its tested commit, for install.project
147
+
148
+
149
+ def _packages(inst: dict, where: str, up: Optional[dict] = None, npm_in: Optional[Callable] = None,
150
+ src: Optional[str] = None) -> str:
151
+ if os.path.exists(os.path.join(where, MARK)):
152
+ return where # another run finished it while this one waited
153
+ if os.path.isdir(where):
154
+ shutil.rmtree(where)
155
+ os.makedirs(where)
156
+ log = os.path.join(where, "install.log")
157
+ if inst.get("npm"):
158
+ cmd = ["npm", "install", "--no-audit", "--no-fund", "--prefix", where] + list(inst["npm"])
159
+ if shutil.which("npm"):
160
+ _run(cmd, where, log)
161
+ elif npm_in is None:
162
+ raise InstallFailed("npm is not on this machine: install Node.js 22 here, or run on the docker backend, "
163
+ "which installs with the Node in its base image")
164
+ else:
165
+ _run(npm_in(cmd, where), where, log)
166
+ for scratch in (".npm-home", ".npm-cache"): # what npm kept of its own, in the container
167
+ shutil.rmtree(os.path.join(where, scratch), ignore_errors=True)
168
+ if inst.get("pip") or inst.get("project") or inst.get("indexes") or inst.get("wheels"):
169
+ py = os.path.join(where, ".venv", "bin", "python")
170
+ # uv's own interpreter, never the system's: an own image mounts the one the
171
+ # environment links to (container.python_homes), and /usr/bin is the image's
172
+ _run(["uv", "venv", "--quiet", "--python-preference", "only-managed", os.path.join(where, ".venv")]
173
+ + (["--python", inst["python"]] if inst.get("python") else []), where, log)
174
+ for x in inst.get("indexes") or []: # first: the project's resolution then keeps them
175
+ _run(["uv", "pip", "install", "--quiet", "--python", py, "--index-url", x["url"]] + list(x["packages"]), where, log)
176
+ wheels = [] # each checked by its sha256 before it is installed
177
+ for x in inst.get("wheels") or []:
178
+ os.makedirs(os.path.join(where, WHEELS), exist_ok=True)
179
+ wheels.append(os.path.join(where, WHEELS, x["url"].rsplit("/", 1)[1]))
180
+ _fetch(x["url"], x["sha256"], wheels[-1], "wheel")
181
+ locked = bool(inst.get("project") and inst.get("locked"))
182
+ if (inst.get("pip") or wheels) and not locked:
183
+ # resolved together: a wheel's dependencies are pip's pins
184
+ _run(["uv", "pip", "install", "--quiet", "--python", py] + list(inst.get("pip") or []) + wheels, where, log)
185
+ if inst.get("project"):
186
+ if src is None:
187
+ src = os.path.join(where, SRC)
188
+ os.makedirs(src)
189
+ for cmd in (["git", "init", "--quiet"], ["git", "fetch", "--quiet", "--depth", "1", up["repository"], up["tested"]],
190
+ ["git", "checkout", "--quiet", "FETCH_HEAD"]):
191
+ _run(cmd, src, log, env=git_env())
192
+ proj = os.path.normpath(os.path.join(src, inst["project"]))
193
+ extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else ""
194
+ if locked:
195
+ # the project's own lock file, its pins as published, and pip's packages
196
+ # resolved beside them in one go: a conflict fails the install, said
197
+ lock = os.path.join(where, "requirements.lock")
198
+ _run(["uv", "export", "--frozen", "--no-dev", "--no-emit-project", "--no-hashes", "--quiet",
199
+ "--project", proj, "-o", lock] + [w for x in inst.get("extras") or [] for w in ("--extra", x)], where, log)
200
+ _run(["uv", "pip", "install", "--quiet", "--python", py, "-r", lock] + list(inst.get("pip") or []) + wheels, where, log)
201
+ _run(["uv", "pip", "install", "--quiet", "--python", py, "--no-deps", "-e", proj + extras], where, log)
202
+ else:
203
+ _run(["uv", "pip", "install", "--quiet", "--python", py, "-e", proj + extras], where, log)
204
+ if inst.get("prefetch"):
205
+ env = dict(install_env(), HF_HUB_CACHE=os.path.join(where, HF))
206
+ _run([py, "-c", "import sys\nfrom huggingface_hub import snapshot_download\nfor m in sys.argv[1:]: snapshot_download(m)"]
207
+ + list(inst["prefetch"]), where, log, env=env)
208
+ with open(os.path.join(where, MARK), "w") as f:
209
+ json.dump(inst, f)
210
+ return where
211
+
212
+
213
+ def process_env(d: dict, where: str, home: str, options: Optional[dict] = None) -> dict:
214
+ """What an install adds to its process's environment: the definition's own
215
+ ``env`` rendered (``{opt_<name>}`` from ``options``, the run's values), and
216
+ ``HF_HUB_CACHE`` when it prefetched models."""
217
+ slots = {"install": where, "home": home, "here": d.get("_here", ""),
218
+ **{"opt_" + k: v for k, v in (options or {}).items()}}
219
+ out = {}
220
+ if (d.get("install") or {}).get("prefetch"):
221
+ out["HF_HUB_CACHE"] = os.path.join(where, HF)
222
+ for k, v in (d.get("env") or {}).items():
223
+ for s, val in slots.items():
224
+ v = v.replace("{" + s + "}", val)
225
+ out[k] = v
226
+ return out
227
+
228
+
229
+ def prune(d: dict, home: str) -> List[str]:
230
+ """Remove what the definition declares as ``caches`` from a unit's home,
231
+ once the unit has ended and its trial is written: the paths it removed.
232
+ A path is resolved under ``home`` and refused when it would leave it
233
+ (a symlink the agent made included); the definition's check already
234
+ holds them to ``{home}/...``."""
235
+ root, gone = os.path.realpath(home), []
236
+ for c in d.get("caches") or []:
237
+ if not isinstance(c, str) or not c.startswith("{home}/"):
238
+ continue
239
+ path = os.path.realpath(os.path.join(home, c[len("{home}/"):]))
240
+ if not path.startswith(root + os.sep) or not os.path.lexists(path):
241
+ continue
242
+ if os.path.isdir(path):
243
+ shutil.rmtree(path, ignore_errors=True)
244
+ else:
245
+ with contextlib.suppress(OSError):
246
+ os.remove(path)
247
+ gone.append(path)
248
+ return gone
249
+
250
+
251
+ def bin_dirs(where: str) -> List[str]:
252
+ """The folders of an install whose programs a start command may name."""
253
+ return [p for p in (os.path.join(where, "node_modules", ".bin"), os.path.join(where, ".venv", "bin")) if os.path.isdir(p)]
254
+
255
+
256
+ def driver_packages(bench: dict, home: str = HOME) -> Optional[str]:
257
+ """The folder a benchmark's ``driver.install`` is installed in, installing
258
+ it first when it is not (once per digest, as an agent's packages); None
259
+ for a driver that declares none and runs on what the machine has. Its
260
+ ``project`` comes from the benchmark's upstream clone, cloned first when
261
+ it is not there: the same commit, fetched once."""
262
+ d = definitions.driver_install(bench)
263
+ if not d["install"]:
264
+ return None
265
+ src = upstream(bench, home) if d["install"].get("project") else None
266
+ return packages(d, home, src=src)
267
+
268
+
269
+ def driver_dir(bench: dict, home: str = HOME) -> Optional[str]:
270
+ """Where ``driver_packages`` installs, installed or not; None when it declares nothing."""
271
+ d = definitions.driver_install(bench)
272
+ return packages_dir(d, home) if d["install"] else None
273
+
274
+
275
+ def driver_argv(bench: dict, home: str = HOME) -> tuple:
276
+ """How a benchmark's driver is started: its ``driver.command`` with
277
+ ``{here}`` (the definition's folder) and ``{install}`` (its
278
+ ``driver.install`` folder, installed first when it is not) filled, and
279
+ what that install adds to the driver's environment (``HF_HUB_CACHE``
280
+ when it prefetched models). Every place that starts a driver - a run, a
281
+ shard, a proof's checks - starts it here."""
282
+ where = driver_packages(bench, home)
283
+ extra = process_env(definitions.driver_install(bench), where, home) if where else {}
284
+ return words(bench["driver"]["command"], {"here": bench["_here"], "install": where or ""}), extra
285
+
286
+
287
+ def upstream(bench: dict, home: str = HOME) -> Optional[str]:
288
+ """A benchmark's upstream at its tested commit, cloned once; None when the
289
+ definition has no upstream repository (its data sits beside it)."""
290
+ u = bench.get("upstream") or {}
291
+ where = upstream_dir(bench, home)
292
+ if where is None:
293
+ return None
294
+ if os.path.exists(os.path.join(where, MARK)):
295
+ return where
296
+ with _locked(where):
297
+ return _upstream(u, where)
298
+
299
+
300
+ def upstream_dir(bench: dict, home: str = HOME) -> Optional[str]:
301
+ """Where ``upstream`` clones a benchmark's upstream; None when it has none."""
302
+ u = bench.get("upstream") or {}
303
+ return os.path.join(home, "upstreams", bench["key"], u["tested"]) if u.get("repository") else None
304
+
305
+
306
+ def _upstream(u: dict, where: str) -> str:
307
+ if os.path.exists(os.path.join(where, MARK)):
308
+ return where
309
+ if os.path.isdir(where):
310
+ shutil.rmtree(where)
311
+ os.makedirs(where)
312
+ log = os.path.join(os.path.dirname(where), u["tested"] + ".log")
313
+ _run(["git", "init", "--quiet"], where, log)
314
+ _run(["git", "fetch", "--quiet", "--depth", "1", u["repository"], u["tested"]], where, log, env=git_env())
315
+ _run(["git", "checkout", "--quiet", "FETCH_HEAD"], where, log)
316
+ open(os.path.join(where, MARK), "w").close()
317
+ return where
318
+
319
+
320
+ def sha256_of(path: str) -> str:
321
+ h = hashlib.sha256()
322
+ with open(path, "rb") as f:
323
+ for chunk in iter(lambda: f.read(1 << 20), b""):
324
+ h.update(chunk)
325
+ return h.hexdigest()
326
+
327
+
328
+ def dataset(bench: dict, ds: dict, up: Optional[str], progress: Optional[Callable] = None) -> Optional[str]:
329
+ """The dataset's file, checked by its sha256: in the upstream when there
330
+ is one, else beside the definition. None for a dataset with no file (its
331
+ driver makes its units). ``progress``: told a download's bytes as they
332
+ come (``download``)."""
333
+ if ds.get("url"):
334
+ return download(ds["url"], ds["sha256"], progress=progress)
335
+ if not ds.get("path"):
336
+ return None
337
+ path = os.path.join(up or bench["_here"], ds["path"])
338
+ if not os.path.isfile(path):
339
+ raise InstallFailed(f"dataset {ds['id']}: {ds['path']} is not in {'the upstream' if up else 'the definition folder'}")
340
+ got = sha256_of(path)
341
+ if got != ds["sha256"]:
342
+ raise InstallFailed(f"dataset {ds['id']}: sha256 {got[:12]}..., the definition says {ds['sha256'][:12]}...")
343
+ return path
344
+
345
+
346
+ def download_path(url: str, sha256: str, home: str = HOME) -> str:
347
+ """Where ``download`` keeps a dataset, downloaded or not."""
348
+ return os.path.join(data_dir(home), sha256, os.path.basename(url.split("?", 1)[0]) or "data")
349
+
350
+
351
+ def download(url: str, sha256: str, home: str = HOME, progress: Optional[Callable] = None) -> str:
352
+ """A dataset served outside its upstream, downloaded once under its
353
+ digest (``data_dir``), and checked by it. ``progress(done, total)`` is
354
+ told the bytes so far after each chunk (``total`` from the server's
355
+ length, None when it gives none), and once more with ``final=True``."""
356
+ where = download_path(url, sha256, home)
357
+ if os.path.isfile(where):
358
+ return where
359
+ with _locked(where):
360
+ if not os.path.isfile(where):
361
+ _fetch(url, sha256, where, "dataset", progress)
362
+ return where
363
+
364
+
365
+ def _fetch(url: str, sha256: str, where: str, what: str, progress: Optional[Callable] = None) -> None:
366
+ """``url`` downloaded to ``where``, kept only when it is the file ``sha256`` names."""
367
+ import urllib.request
368
+ part = where + ".part"
369
+ try:
370
+ with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "nullius"}), timeout=60) as r, open(part, "wb") as f:
371
+ total = int(r.headers.get("Content-Length") or 0) or None
372
+ done = 0
373
+ for chunk in iter(lambda: r.read(1 << 20), b""):
374
+ f.write(chunk)
375
+ done += len(chunk)
376
+ if progress:
377
+ progress(done, total)
378
+ if progress:
379
+ progress(done, total, final=True)
380
+ except OSError as e:
381
+ raise InstallFailed(f"{what} {url}: {e}")
382
+ got = sha256_of(part)
383
+ if got != sha256:
384
+ os.remove(part)
385
+ raise InstallFailed(f"{what} {url}: sha256 {got[:12]}..., the definition says {sha256[:12]}...")
386
+ os.replace(part, where)
387
+
388
+
389
+ def words(cmd: List[str], slots: Dict[str, str]) -> List[str]:
390
+ """A definition's command with its slots filled: each slot is replaced
391
+ inside its word, and the word stays one argument - no shell reads it."""
392
+ out = []
393
+ for w in cmd:
394
+ for k, v in slots.items():
395
+ w = w.replace("{" + k + "}", v)
396
+ out.append(w)
397
+ return out