nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,278 @@
1
+ """A standard runner snapshot made from this runtime and its effective catalogue.
2
+
3
+ Only backend Python modules, definition documents and their local resources
4
+ travel. The installation's neighbours, user configuration, licences and SSH
5
+ keys are never inputs. Source checkouts and installed wheels use the same
6
+ content address, including executable bits; neither Git nor the viewer is
7
+ needed on the target. Credential transport belongs to ``execution``.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import ast
12
+ import hashlib
13
+ import json
14
+ import os
15
+ import re
16
+ import shutil
17
+ import stat
18
+ import tempfile
19
+ from pathlib import Path
20
+
21
+ from nullius import paths
22
+ from nullius.standard import definitions
23
+
24
+ PACKAGE = Path(__file__).resolve().parents[1]
25
+ FORMAT = 1
26
+ _KINDS = {"benchmarks": ("benchmark.json", "standard.json"),
27
+ "agents": ("agent.json",), "capabilities": ("capability.json",)}
28
+ _DOCUMENTS = {"feature.json", "benchmark.json", "standard.json", "agent.json", "capability.json",
29
+ "upstream-settings.json", "images.json", "data-source.json"}
30
+ _PRIVATE = {"credentials", "credentials.json", "secret", "secret.json", "secrets", "secrets.json", "settings.json",
31
+ "id_rsa", "id_ed25519", "id_ecdsa", "id_dsa", "node_modules", "__pycache__"}
32
+ _KEY = re.compile(r"^[a-z][a-z0-9-]{0,39}$")
33
+ _HERE = re.compile(r"\{here\}/([A-Za-z0-9_./-]+)")
34
+
35
+
36
+ def _safe(relative: str) -> bool:
37
+ p = Path(relative)
38
+ return (bool(relative) and not p.is_absolute() and "\\" not in relative
39
+ and all(not s.startswith(".") and s.lower() not in _PRIVATE for s in p.parts)
40
+ and p.suffix.lower() not in {".pem", ".key", ".p12", ".pfx", ".pyc"})
41
+
42
+
43
+ def _read(root: Path, relative: str) -> tuple[bytes, int]:
44
+ """Read a regular file below root, refusing symlinks at every segment."""
45
+ if not _safe(relative):
46
+ raise ValueError(f"not a runner resource: {relative}")
47
+ p = root
48
+ for segment in Path(relative).parts:
49
+ p = p / segment
50
+ if p.is_symlink():
51
+ raise ValueError(f"runner resources cannot be symlinks: {relative}")
52
+ mode = p.stat().st_mode
53
+ if not stat.S_ISREG(mode):
54
+ raise ValueError(f"runner resource is not a file: {relative}")
55
+ return p.read_bytes(), 0o755 if mode & 0o111 else 0o644
56
+
57
+
58
+ def _python(root: Path):
59
+ """Backend/glue code only: never follow a directory link or copy caches."""
60
+ for directory, folders, files in os.walk(root, followlinks=False):
61
+ folders[:] = sorted(n for n in folders if _safe(n) and n != "_resources"
62
+ and not (Path(directory) / n).is_symlink())
63
+ for name in sorted(files):
64
+ if name.endswith(".py") and _safe(name):
65
+ yield (Path(directory) / name).relative_to(root).as_posix()
66
+
67
+
68
+ def _resources(d: dict) -> set[str]:
69
+ """Local files named by commands, model templates and image contexts.
70
+
71
+ Walk version blocks too. Licence declarations name runtime destinations,
72
+ not resources; they are deliberately not interpreted as local paths.
73
+ """
74
+ out: set[str] = set()
75
+ def visit(value):
76
+ if isinstance(value, str):
77
+ out.update(_HERE.findall(value))
78
+ elif isinstance(value, list):
79
+ for v in value:
80
+ visit(v)
81
+ elif isinstance(value, dict):
82
+ for k, v in value.items():
83
+ if k != "needs_secrets":
84
+ visit(v)
85
+ model = value.get("model")
86
+ if isinstance(model, dict) and isinstance(model.get("files"), dict):
87
+ out.update(v for v in model["files"].values() if isinstance(v, str))
88
+ environments = value.get("environments")
89
+ if isinstance(environments, dict):
90
+ for environment in environments.values():
91
+ if isinstance(environment, dict):
92
+ if isinstance(environment.get("dockerfile"), str):
93
+ out.add(environment["dockerfile"])
94
+ out.update(v for v in environment.get("files", []) if isinstance(v, str))
95
+ visit(d)
96
+ return out
97
+
98
+
99
+ def _dataset_resources(d: dict) -> set[str]:
100
+ """Dataset files installed beside each effective benchmark version.
101
+
102
+ Upstream checkout paths and downloaded datasets belong to the installer,
103
+ not the catalogue snapshot. Resolve versions to retain inherited upstreams
104
+ and datasets exactly as the runner sees them.
105
+ """
106
+ out = set()
107
+ for version in definitions.versions(d):
108
+ bench = definitions.resolve(d, version)
109
+ if bench is None or (bench.get("upstream") or {}).get("repository"):
110
+ continue
111
+ for dataset in bench.get("datasets") or []:
112
+ if isinstance(dataset, dict) and not dataset.get("url"):
113
+ path = dataset.get("path")
114
+ if isinstance(path, str) and path:
115
+ out.add(path)
116
+ return out
117
+
118
+
119
+ def _imports(root: Path, relative: str, content: bytes) -> set[str]:
120
+ """Follow local Python imports from a declared driver without sweeping its folder."""
121
+ if not relative.endswith(".py"):
122
+ return set()
123
+ try:
124
+ tree = ast.parse(content)
125
+ except (SyntaxError, ValueError):
126
+ return set() # Python itself reports invalid source when run
127
+ found = set()
128
+ for node in ast.walk(tree):
129
+ names = []
130
+ base = Path(relative).parent
131
+ if isinstance(node, ast.Import):
132
+ names = [n.name for n in node.names]
133
+ elif isinstance(node, ast.ImportFrom):
134
+ if node.level:
135
+ for _ in range(node.level - 1):
136
+ base = base.parent
137
+ prefix = (node.module + ".") if node.module else ""
138
+ names = ([node.module] if node.module else []) + [prefix + n.name for n in node.names if n.name != "*"]
139
+ for name in names:
140
+ for origin in (base, Path()):
141
+ module = origin / name.replace(".", "/")
142
+ for candidate in (module.with_suffix(".py"), module / "__init__.py"):
143
+ r = candidate.as_posix()
144
+ if _safe(r) and (root / r).is_file():
145
+ found.add(r)
146
+ for parent in module.parents:
147
+ r = (parent / "__init__.py").as_posix()
148
+ if _safe(r) and (root / r).is_file():
149
+ found.add(r)
150
+ return found
151
+
152
+
153
+ def _entry(root: Path, *, kit: bool = False) -> dict[str, tuple[bytes, int]]:
154
+ if root.is_symlink():
155
+ raise ValueError("catalogue entries cannot be symlinks")
156
+ files = {}
157
+ resources = set(_python(root)) if kit else set()
158
+ for name in sorted(_DOCUMENTS):
159
+ if (root / name).exists():
160
+ content = _read(root, name)
161
+ d = json.loads(content[0])
162
+ if name in {"standard.json", "agent.json", "capability.json", "feature.json", "benchmark.json"} and not isinstance(d, dict):
163
+ raise ValueError(f"{name}: expected a definition object")
164
+ files[name] = content
165
+ if name in {"standard.json", "agent.json", "capability.json", "feature.json"}:
166
+ resources.update(_resources(d))
167
+ if name == "standard.json":
168
+ resources.update(_dataset_resources(d))
169
+ while resources - files.keys():
170
+ name = min(resources - files.keys())
171
+ files[name] = _read(root, name)
172
+ resources.update(_imports(root, name, files[name][0]))
173
+ return files
174
+
175
+
176
+ def _cached(destination: Path, marker: str, files: dict) -> bool:
177
+ """Do not reuse or transfer a cache into which another file was written."""
178
+ if not destination.exists():
179
+ return False
180
+ manifest = destination / "manifest.json"
181
+ expected = {"manifest.json", *("app/nullius/" + name for name in files)}
182
+ found = set()
183
+ if destination.is_symlink():
184
+ raise ValueError("runner snapshot cache cannot be a symlink")
185
+ for directory, folders, names in os.walk(destination, followlinks=False):
186
+ for name in folders + names:
187
+ p = Path(directory) / name
188
+ if p.is_symlink():
189
+ raise ValueError("runner snapshot cache contains a symlink; remove " + str(destination))
190
+ found.update((Path(directory) / name).relative_to(destination).as_posix() for name in names)
191
+ valid = found == expected and manifest.read_text() == marker
192
+ if valid:
193
+ for name, value in files.items():
194
+ if _read(destination / "app/nullius", name) != value:
195
+ valid = False
196
+ break
197
+ if not valid:
198
+ raise ValueError("runner snapshot cache has changed; remove " + str(destination))
199
+ return True
200
+
201
+
202
+ def build(cache: str, *, package: Path | None = None, catalogue: str | None = None) -> tuple[str, str]:
203
+ """Return (content ID, snapshot root), with ``app/nullius`` below it.
204
+
205
+ A user entry replaces the whole bundled entry, as the normal catalogue
206
+ loader does. Nothing is created in either input tree. The staged tree is
207
+ replaced atomically, and the address covers the exact bytes being sent.
208
+ """
209
+ package = Path(package) if package is not None else PACKAGE
210
+ if package.is_symlink():
211
+ raise ValueError("the runtime package cannot be a symlink")
212
+ files = {}
213
+ for relative in _python(package):
214
+ pieces = Path(relative).parts
215
+ if ((len(pieces) > 2 and pieces[0] == "catalogue" and pieces[1] in _KINDS)
216
+ or pieces[:2] == ("standard", "kits")):
217
+ continue
218
+ files[relative] = _read(package, relative)
219
+ if "standard/cli.py" not in files or "__init__.py" not in files:
220
+ raise ValueError("the standard runtime package is incomplete")
221
+ bundled = package / "catalogue"
222
+ user = Path(catalogue if catalogue is not None else paths.catalogue_dir())
223
+ if user.is_symlink() or bundled.is_symlink():
224
+ raise ValueError("catalogue roots cannot be symlinks")
225
+ for kind in _KINDS:
226
+ entries = {}
227
+ for catalogue_root in (bundled, user):
228
+ for directory in (("features", "capabilities") if kind == "capabilities" else (kind,)):
229
+ base = catalogue_root / directory
230
+ if base.is_symlink():
231
+ raise ValueError("catalogue folders cannot be symlinks")
232
+ if not base.is_dir():
233
+ continue
234
+ for entry in sorted(base.iterdir()):
235
+ if (_KEY.fullmatch(entry.name) and entry.is_dir()
236
+ and sorted(p.name for p in entry.iterdir()) != ["__pycache__"]):
237
+ entries[entry.name] = entry
238
+ for name, entry in sorted(entries.items()):
239
+ try:
240
+ contents = _entry(entry)
241
+ except json.JSONDecodeError:
242
+ continue # Unfinished JSON drafts are not runnable catalogue entries.
243
+ files.update({f"catalogue/{kind}/{name}/{r}": value for r, value in contents.items()})
244
+ kits = package / "standard" / "kits"
245
+ if kits.is_dir():
246
+ if kits.is_symlink():
247
+ raise ValueError("the kit folder cannot be a symlink")
248
+ for entry in sorted(kits.iterdir()):
249
+ if _KEY.fullmatch(entry.name) and (entry / "standard.json").is_file():
250
+ files.update({f"standard/kits/{entry.name}/{r}": value for r, value in _entry(entry, kit=True).items()})
251
+ digest = hashlib.sha256(f"nullius-standard-payload-v{FORMAT}\0".encode())
252
+ manifest = {}
253
+ for name, (content, mode) in sorted(files.items()):
254
+ manifest[name] = {"sha256": hashlib.sha256(content).hexdigest(), "mode": mode}
255
+ identity = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode()
256
+ digest.update(identity)
257
+ key = "v%d-%s" % (FORMAT, digest.hexdigest())
258
+ destination = Path(cache) / key
259
+ marker = json.dumps({"format": FORMAT, "files": manifest}, sort_keys=True, indent=2) + "\n"
260
+ if _cached(destination, marker, files):
261
+ return key, str(destination)
262
+ Path(cache).mkdir(parents=True, exist_ok=True)
263
+ staging = Path(tempfile.mkdtemp(prefix=".payload-", dir=cache))
264
+ try:
265
+ for name, (content, mode) in sorted(files.items()):
266
+ target = staging / "app" / "nullius" / name
267
+ target.parent.mkdir(parents=True, exist_ok=True)
268
+ target.write_bytes(content)
269
+ target.chmod(mode)
270
+ (staging / "manifest.json").write_text(marker)
271
+ try:
272
+ staging.rename(destination)
273
+ except OSError:
274
+ if not _cached(destination, marker, files):
275
+ raise
276
+ finally:
277
+ shutil.rmtree(staging, ignore_errors=True)
278
+ return key, str(destination)
@@ -0,0 +1,130 @@
1
+ """Compare a benchmark's image pins with registry tags, without pulling or
2
+ changing anything. Read images.json, or string literals in driver.py without
3
+ executing it. Registry V2 HEAD requests read manifest digests; anonymous
4
+ Bearer challenges are supported, private credentials are never loaded.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import ast
9
+ import json
10
+ import re
11
+ from http.client import HTTPException
12
+ from pathlib import Path
13
+ from urllib.error import HTTPError, URLError
14
+ from urllib.parse import urlencode, urlsplit
15
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
16
+
17
+ TAG = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
18
+ DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
19
+ REPOSITORY = re.compile(r"[a-z0-9]+(?:[._-]+[a-z0-9]+)*(?:/[a-z0-9]+(?:[._-]+[a-z0-9]+)*)*")
20
+ ACCEPT = ", ".join(("application/vnd.oci.image.index.v1+json", "application/vnd.oci.image.manifest.v1+json",
21
+ "application/vnd.docker.distribution.manifest.list.v2+json",
22
+ "application/vnd.docker.distribution.manifest.v2+json"))
23
+
24
+
25
+ def references(folder: str) -> list:
26
+ """(name, reference) in file order. images.json takes precedence."""
27
+ root = Path(folder)
28
+ if (root / "images.json").exists():
29
+ data = json.loads((root / "images.json").read_text())
30
+ if not isinstance(data, dict) or not all(isinstance(v, str) for v in data.values()):
31
+ raise ValueError("images.json must map names to pinned image references")
32
+ return list(data.items())
33
+ tree = ast.parse((root / "driver.py").read_text())
34
+ strings = sorted((n for n in ast.walk(tree) if isinstance(n, ast.Constant) and isinstance(n.value, str)
35
+ and "@sha256:" in n.value), key=lambda n: (n.lineno, n.col_offset))
36
+ return [(s, s) for s in dict.fromkeys(n.value for n in strings)]
37
+
38
+
39
+ def target(reference: str, tag: str | None = None) -> tuple:
40
+ """Registry, repository, tag and pinned digest. Docker Hub is implicit;
41
+ --tag overrides an embedded tag, otherwise an untagged pin uses latest."""
42
+ name, _, digest = reference.rpartition("@")
43
+ if not DIGEST.fullmatch(digest):
44
+ raise ValueError("expected an image pinned by sha256 digest")
45
+ embedded = None
46
+ if ":" in name.rsplit("/", 1)[-1]:
47
+ name, embedded = name.rsplit(":", 1)
48
+ tag = tag if tag is not None else embedded or "latest"
49
+ if not TAG.fullmatch(tag):
50
+ raise ValueError("invalid image tag")
51
+ first, slash, rest = name.partition("/")
52
+ if slash and ("." in first or ":" in first or first == "localhost"):
53
+ registry, repository = first, rest
54
+ else:
55
+ registry, repository = "docker.io", name
56
+ if not re.fullmatch(r"[A-Za-z0-9.-]+(?::[0-9]+)?", registry) or not REPOSITORY.fullmatch(repository):
57
+ raise ValueError("invalid registry or repository name")
58
+ if registry in ("docker.io", "index.docker.io", "registry-1.docker.io"):
59
+ registry = "registry-1.docker.io"
60
+ if "/" not in repository:
61
+ repository = "library/" + repository
62
+ return registry, repository, tag, digest
63
+
64
+
65
+ class _NoRedirect(HTTPRedirectHandler):
66
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
67
+ # A registry's bearer token must not follow a redirect to another service.
68
+ raise ValueError("registry redirect not followed")
69
+
70
+
71
+ def _token(challenge: str, repository: str, open_) -> str:
72
+ kind, _, fields = challenge.partition(" ")
73
+ params = dict(re.findall(r'(\w+)="([^"\r\n]*)"', fields))
74
+ realm = params.get("realm", "")
75
+ url = urlsplit(realm)
76
+ if kind.lower() != "bearer" or url.scheme != "https" or not url.hostname or url.username or url.password or url.fragment:
77
+ raise ValueError("registry requires unsupported authentication")
78
+ query = urlencode({"service": params.get("service", ""), "scope": f"repository:{repository}:pull"})
79
+ try:
80
+ with open_(Request(realm + ("&" if url.query else "?") + query), timeout=15) as response:
81
+ data = json.loads(response.read(65537))
82
+ except HTTPError as e:
83
+ e.close()
84
+ raise ValueError(f"token service returned HTTP {e.code}; anonymous access unavailable") from None
85
+ except (ValueError, UnicodeError):
86
+ raise ValueError("invalid token service response") from None
87
+ token = (data.get("token") or data.get("access_token")) if isinstance(data, dict) else None
88
+ if not isinstance(token, str) or not token or any(not 33 <= ord(c) <= 126 for c in token):
89
+ raise ValueError("token service returned no usable bearer token")
90
+ return token
91
+
92
+
93
+ def current(registry: str, repository: str, tag: str, open_=None) -> str:
94
+ """Ask only for manifest headers. A missing tag raises HTTPError(404);
95
+ auth-service failures must not be mistaken for a missing manifest."""
96
+ open_ = open_ or build_opener(_NoRedirect()).open
97
+ url = f"https://{registry}/v2/{repository}/manifests/{tag}"
98
+ headers = {"Accept": ACCEPT}
99
+ try:
100
+ response = open_(Request(url, headers=headers, method="HEAD"), timeout=15)
101
+ except HTTPError as e:
102
+ if e.code != 401:
103
+ raise
104
+ challenge = e.headers.get("WWW-Authenticate", "")
105
+ e.close()
106
+ headers["Authorization"] = "Bearer " + _token(challenge, repository, open_)
107
+ response = open_(Request(url, headers=headers, method="HEAD"), timeout=15)
108
+ with response:
109
+ digest = response.headers.get("Docker-Content-Digest", "")
110
+ if not DIGEST.fullmatch(digest):
111
+ raise ValueError("registry returned no valid sha256 manifest digest")
112
+ return digest
113
+
114
+
115
+ def check(reference: str, tag: str | None = None) -> tuple:
116
+ """(state, detail): same, moved, gone, or not checked. Never include
117
+ response bodies, authentication URLs or credentials in diagnostics."""
118
+ try:
119
+ registry, repository, tag, pinned = target(reference, tag)
120
+ found = current(registry, repository, tag)
121
+ return ("same", f"{tag}: {pinned}") if found == pinned else ("moved", f"{tag}: {pinned} -> {found}")
122
+ except HTTPError as e:
123
+ e.close()
124
+ if e.code == 404:
125
+ return "gone", f"{tag}: registry returned HTTP 404"
126
+ return "not checked", f"registry returned HTTP {e.code}"
127
+ except (URLError, OSError, HTTPException):
128
+ return "not checked", "registry connection failed or timed out"
129
+ except ValueError as e:
130
+ return "not checked", str(e)
@@ -0,0 +1,70 @@
1
+ """The ports of version 1: what each one promises, and the kit that proves it.
2
+
3
+ A port's name carries its version. Its contract never changes under the same
4
+ name: a different contract is a new version, and a definition naming the old
5
+ one keeps meaning the old one. ``label`` is what a page calls it, in words a
6
+ person needs no glossary for; the port's name stays beside it. ``kit`` is the folder under ``standard/kits/``
7
+ holding the tiny benchmark that proves the port, or None while a port is
8
+ defined ahead of its first benchmark.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import re
13
+
14
+ from . import names
15
+
16
+ NAME = re.compile(r"^(agent|cap)\.[a-z][a-z0-9-]{0,31}/v[1-9][0-9]{0,2}$")
17
+
18
+ PORTS = {
19
+ "agent.task/v1": {
20
+ "label": "One task",
21
+ "family": "agent",
22
+ "summary": "One prompt in, one turn; the answer is the agent's final message.",
23
+ "kit": "agent-task-v1",
24
+ },
25
+ "agent.workspace/v1": {
26
+ "label": "Workspace files",
27
+ "family": "agent",
28
+ "summary": "The session's working directory is the unit's workspace; the agent reads and writes files there.",
29
+ "kit": "agent-workspace-v1",
30
+ },
31
+ "agent.tools/v1": {
32
+ "label": "MCP tools",
33
+ "family": "agent",
34
+ "summary": "The agent calls the tools of the MCP servers given to its session.",
35
+ "kit": "agent-tools-v1",
36
+ },
37
+ "agent.conversation/v1": {
38
+ "label": "Conversation",
39
+ "family": "agent",
40
+ "summary": "Several prompts in one session, each answered in its own turn, the earlier ones remembered.",
41
+ "kit": "agent-conversation-v1",
42
+ },
43
+ "agent.search/v1": {
44
+ "label": "Search",
45
+ "family": "agent",
46
+ "summary": "Search an HTTP collection, retrieve a source and answer with its evidence using the agent's available tools.",
47
+ "kit": "agent-search-v1",
48
+ },
49
+ "agent.browser/v1": {
50
+ "label": "Browser",
51
+ "family": "agent",
52
+ "summary": "A browser reached through an MCP server Nullius starts.",
53
+ "kit": None,
54
+ },
55
+ "cap.memory/v1": {
56
+ "label": "Memory",
57
+ "family": "capability",
58
+ "summary": "add {session} then recall {question} -> {memory}: what it kept that bears on the question.",
59
+ "kit": "cap-memory-v1",
60
+ },
61
+ }
62
+
63
+
64
+ def family(port: str) -> str:
65
+ """``agent`` or ``capability``: the side of a pairing a port belongs to."""
66
+ return "capability" if names.port(port).startswith("cap.") else port.split(".", 1)[0]
67
+
68
+
69
+ def known(port: str) -> bool:
70
+ return names.port(port) in PORTS