nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/hosts.py ADDED
@@ -0,0 +1,368 @@
1
+ """The remote machines a recipe can run on.
2
+
3
+ A host is one of two things, both spelled in the ``HOSTS`` setting, comma
4
+ separated:
5
+
6
+ - an alias from ``~/.ssh/config`` — ``bench`` — where ssh itself knows the
7
+ address, user, port and key. Nullius reads the config for the *names* and
8
+ what it says beside them (HostName, User, Port); it never opens a key.
9
+ - a manual entry — ``user@host:port`` optionally followed by ``:/path/to/key``
10
+ — for a machine the config does not name. An optional ``Name=`` prefix
11
+ supplies its display label without changing the SSH target. The key stays a path.
12
+
13
+ ``test`` asks the host what a run needs: that ssh gets in without a prompt,
14
+ and whether ``uv``, ``docker``, ``podman`` and ``harbor`` answer there, and
15
+ whether its docker has ``userns-remap``. It runs ssh with
16
+ ``BatchMode=yes`` so a password prompt is a refusal, not a hang, and it never
17
+ sends anything but that one command. Nothing is stored beyond the host list.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import os
22
+ import re
23
+ import subprocess
24
+ import time
25
+ from dataclasses import dataclass
26
+ from typing import Callable, Dict, List, Optional
27
+
28
+ from nullius import paths
29
+
30
+ _ALIAS = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
31
+ _MANUAL = re.compile(r"^(?P<user>[A-Za-z0-9._-]+)@(?P<host>[A-Za-z0-9.-]+|\[[0-9A-Fa-f:.]+\])(?::(?P<port>[0-9]{1,5}))?(?::(?P<key>(?:/|~/)[^\s,:]+))?$")
32
+ PROBE = ("printf 'ok\\n'; for c in uv docker podman harbor; do command -v \"$c\" >/dev/null 2>&1 && printf '%s=yes\\n' \"$c\" || printf '%s=no\\n' \"$c\"; done; "
33
+ "command -v harbor >/dev/null 2>&1 && printf 'harbor_version=%s\\n' \"$(harbor --version 2>/dev/null | sed -n 1p)\"; "
34
+ "for c in docker podman; do command -v $c >/dev/null 2>&1 && ($c ps >/dev/null 2>&1 && printf '%s_ok=yes\\n' $c || printf '%s_ok=no\\n' $c); done; "
35
+ "docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q name=userns && printf 'docker_userns=yes\\n'; "
36
+ "printf 'hostname=%s\\n' \"$(hostname)\"")
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class Host:
41
+ entry: str # as written in the setting
42
+ kind: str # "alias" | "manual"
43
+ alias: Optional[str] = None
44
+ user: Optional[str] = None
45
+ host: Optional[str] = None
46
+ port: Optional[int] = None
47
+ key: Optional[str] = None # a path, never its contents
48
+ display_name: Optional[str] = None # optional name for a manual connection
49
+
50
+ @property
51
+ def label(self) -> str:
52
+ if self.display_name:
53
+ return self.display_name
54
+ if self.kind == "alias":
55
+ return self.alias or self.entry
56
+ return f"{self.user}@{self.host}" + (f":{self.port}" if self.port else "")
57
+
58
+ def argv(self, command: str, timeout: float = 8.0) -> List[str]:
59
+ """The ssh argv: BatchMode so a prompt is a refusal, a connect
60
+ timeout, and the one command."""
61
+ argv = ["ssh", "-o", "BatchMode=yes", "-o", f"ConnectTimeout={int(timeout)}", "-o", "StrictHostKeyChecking=accept-new"]
62
+ if self.kind == "alias":
63
+ target = self.alias
64
+ else:
65
+ if self.port:
66
+ argv += ["-p", str(self.port)]
67
+ if self.key:
68
+ # that key alone: an agent holding several offers each first, and a server that counts
69
+ # them (MaxAuthTries, six by default) hangs up before the named one is tried
70
+ argv += ["-i", os.path.expanduser(self.key), "-o", "IdentitiesOnly=yes"]
71
+ target = f"{self.user}@{self.host}"
72
+ return argv + [target, command]
73
+
74
+ def to_dict(self) -> dict:
75
+ return {"entry": self.entry, "kind": self.kind, "label": self.label, "alias": self.alias, "user": self.user,
76
+ "host": self.host, "port": self.port, "key": self.key}
77
+
78
+
79
+ def parse(entry: str) -> Host:
80
+ """One entry of the setting → a Host; ValueError when it is neither form."""
81
+ e = (entry or "").strip()
82
+ target, display_name = e, None
83
+ if "=" in e.split("@", 1)[0]:
84
+ display_name, target = e.split("=", 1)
85
+ display_name, target = display_name.strip(), target.strip()
86
+ if not display_name or len(display_name) > 64 or any(c in ",=@" or not c.isprintable() for c in display_name):
87
+ raise ValueError("Host name must be 1–64 printable characters without commas, @ or =")
88
+ m = _MANUAL.match(target)
89
+ if m:
90
+ port = int(m.group("port")) if m.group("port") else None
91
+ if port is not None and not 1 <= port <= 65535:
92
+ raise ValueError(f"{e!r}: port out of range")
93
+ return Host(entry=e, kind="manual", user=m.group("user"), host=m.group("host"), port=port, key=m.group("key"), display_name=display_name)
94
+ if display_name is None and _ALIAS.match(e):
95
+ return Host(entry=e, kind="alias", alias=e)
96
+ raise ValueError(f"{e!r} is not a host: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]")
97
+
98
+
99
+ def parse_list(text: str) -> List[Host]:
100
+ out, seen = [], set()
101
+ for raw in (text or "").split(","):
102
+ if not raw.strip():
103
+ continue
104
+ h = parse(raw)
105
+ if h.entry not in seen:
106
+ seen.add(h.entry)
107
+ out.append(h)
108
+ return out
109
+
110
+
111
+ def validate_list(text: str) -> Optional[str]:
112
+ """None when every entry parses, else what was wrong."""
113
+ try:
114
+ parse_list(text)
115
+ return None
116
+ except ValueError as e:
117
+ return str(e)
118
+
119
+
120
+ # --- ~/.ssh/config --------------------------------------------------------
121
+
122
+ def ssh_config_path() -> str:
123
+ return os.path.join(os.path.expanduser("~"), ".ssh", "config")
124
+
125
+
126
+ def scan_ssh_config(path: Optional[str] = None) -> List[dict]:
127
+ """The ``Host`` aliases of the ssh config with what the config says beside
128
+ them - hostname, user, port. Patterns (``*``, ``?``, ``!``) are skipped:
129
+ they are not machines. Identity files are not read and not reported."""
130
+ path = path or ssh_config_path()
131
+ try:
132
+ with open(path, encoding="utf-8", errors="replace") as fh:
133
+ lines = fh.read().splitlines()
134
+ except OSError:
135
+ return []
136
+ hosts: List[dict] = []
137
+ current: List[dict] = []
138
+ for raw in lines:
139
+ line = raw.split("#", 1)[0].strip()
140
+ if not line:
141
+ continue
142
+ m = re.match(r"^(\w+)\s*[=\s]\s*(.*)$", line)
143
+ if not m:
144
+ continue
145
+ key, value = m.group(1).lower(), m.group(2).strip()
146
+ if key == "host":
147
+ current = []
148
+ for name in value.split():
149
+ if any(c in name for c in "*?!"):
150
+ continue
151
+ if _ALIAS.match(name):
152
+ entry = {"alias": name, "hostname": None, "user": None, "port": None}
153
+ hosts.append(entry)
154
+ current.append(entry)
155
+ elif key == "match":
156
+ current = []
157
+ elif key in ("hostname", "user", "port") and current:
158
+ for entry in current:
159
+ if entry[key] is None:
160
+ entry[key] = value
161
+ return hosts
162
+
163
+
164
+
165
+ def subprocess_run(argv: List[str], timeout: float) -> tuple:
166
+ """(returncode, stdout, stderr); a timeout is a return code of its own."""
167
+ try:
168
+ proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL)
169
+ return proc.returncode, proc.stdout, proc.stderr
170
+ except subprocess.TimeoutExpired:
171
+ return 124, "", f"no answer within {int(timeout)} s"
172
+ except OSError as e:
173
+ return 127, "", str(e)
174
+
175
+
176
+ _PROGRAM = re.compile(r"^[a-z0-9][a-z0-9._+-]{0,39}$")
177
+
178
+
179
+ def test(host: Host, run: Optional[Callable] = None, timeout: float = 12.0, programs: Optional[List[str]] = None) -> dict:
180
+ """What the host answers: reachable or why not, and whether uv, docker,
181
+ podman and harbor are there (``engine``: which of docker and podman the
182
+ standard's container backend runs there, or None) - and, given ``programs`` (a benchmark's host
183
+ requirements), which of them are not on its PATH (``missing``). Nothing
184
+ but the probe command is sent. ``run`` is looked up at call time so a
185
+ test can stand in for ssh."""
186
+ programs = [p for p in programs or [] if _PROGRAM.match(p)]
187
+ probe = PROBE + ("; for p in " + " ".join(programs) + '; do command -v "$p" >/dev/null 2>&1 || printf \'missing=%s\\n\' "$p"; done'
188
+ if programs else "")
189
+ started = time.monotonic()
190
+ rc, out, err = (run or subprocess_run)(host.argv(probe, timeout=min(timeout, 8.0)), timeout)
191
+ ms = int((time.monotonic() - started) * 1000)
192
+ result: Dict = {"entry": host.entry, "label": host.label, "ok": False, "latency_ms": ms, "uv": None, "docker": None,
193
+ "docker_ok": None, "docker_userns": False, "podman": None, "podman_ok": None, "engine": None,
194
+ "harbor": None, "harbor_version": None, "hostname": None, "error": None,
195
+ **({"missing": []} if programs else {})}
196
+ if rc != 0 or "ok" not in out.split():
197
+ text = (err or out or "").strip().splitlines()
198
+ result["error"] = _reason(rc, text[-1] if text else "")
199
+ return result
200
+ for line in out.splitlines():
201
+ k, _, v = line.partition("=")
202
+ if k in ("uv", "docker", "podman", "harbor", "docker_ok", "podman_ok", "docker_userns"):
203
+ result[k] = v == "yes"
204
+ elif k == "harbor_version":
205
+ result["harbor_version"] = v.strip().removeprefix("harbor ").strip() or None
206
+ elif k == "hostname":
207
+ result["hostname"] = v.strip() or None
208
+ elif k == "missing" and programs and v.strip() in programs:
209
+ result["missing"].append(v.strip())
210
+ # the standard's container backend's engine there: docker when the probe finds it, else podman
211
+ result["engine"] = "docker" if result["docker"] else "podman" if result["podman"] else None
212
+ result["ok"] = True
213
+ return result
214
+
215
+
216
+ def _reason(rc: int, last: str) -> str:
217
+ low = last.lower()
218
+ if rc == 124:
219
+ return last or "no answer"
220
+ if "permission denied" in low:
221
+ return "refused: no usable key"
222
+ if "could not resolve" in low or "name or service not known" in low:
223
+ return "the host name does not resolve"
224
+ if "connection refused" in low:
225
+ return "connection refused"
226
+ if "host key verification failed" in low:
227
+ return "host key verification failed"
228
+ if rc == 255:
229
+ return last or "ssh could not connect"
230
+ return last or f"ssh exited {rc}"
231
+
232
+
233
+ # --- getting a key onto a host ------------------------------------------
234
+ #
235
+ # A manual host needs a key ssh can use without a prompt. Three ways in, none
236
+ # of which stores a password: a key already on this machine (a path), a key
237
+ # file handed to the page (saved under the user's Nullius config, mode 0600), or a
238
+ # password used once to put Nullius's own public key into the host's
239
+ # authorized_keys — the password goes to ssh through a pty and nowhere else,
240
+ # and the entry that comes back names the key, not the password.
241
+
242
+ # Optional explicit overrides; defaults are resolved at use time so a
243
+ # NULLIUS_HOME profile does not share the real user's key store.
244
+ KEY_DIR: Optional[str] = None
245
+ OWN_KEY: Optional[str] = None
246
+ _KEY_HEADER = re.compile(rb"^-----BEGIN (?:OPENSSH |RSA |EC |DSA |ENCRYPTED )?PRIVATE KEY-----")
247
+
248
+
249
+ def _key_name(user: str, host: str) -> str:
250
+ return re.sub(r"[^A-Za-z0-9._@-]+", "_", f"{user}@{host}")
251
+
252
+
253
+ def save_key(user: str, host: str, data: bytes, key_dir: Optional[str] = None) -> str:
254
+ """A private key handed to the page, written where ssh can read it and
255
+ nobody else: <config>/ssh/<user@host>, mode 0600. The path it is at."""
256
+ if not _KEY_HEADER.match(data.lstrip()) or len(data) > 20000:
257
+ raise ValueError("not a private key file (expected -----BEGIN … PRIVATE KEY-----)")
258
+ if b"ENCRYPTED" in data[:200]:
259
+ raise ValueError("the key has a passphrase; ssh could not use it without a prompt")
260
+ key_dir = key_dir or KEY_DIR or os.path.join(paths.config_dir(), "ssh")
261
+ os.makedirs(key_dir, mode=0o700, exist_ok=True)
262
+ path = os.path.join(key_dir, _key_name(user, host))
263
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
264
+ with os.fdopen(fd, "wb") as fh:
265
+ fh.write(data if data.endswith(b"\n") else data + b"\n")
266
+ return path
267
+
268
+
269
+ def forget_keys(old: str, new: str, key_dir: Optional[str] = None) -> List[tuple]:
270
+ """Delete each key file that ``save_key`` wrote for an entry of the host
271
+ list ``old`` when no entry of the list ``new`` names it. Only the file
272
+ <config>/ssh/<user@host> of that entry goes: Nullius's own key and a key
273
+ the user named by path stay. The files not deleted, each as (path, reason)."""
274
+ key_dir = os.path.realpath(key_dir or KEY_DIR or os.path.join(paths.config_dir(), "ssh"))
275
+ try:
276
+ before = parse_list(old)
277
+ except ValueError: # a list edited by hand into a wrong one names no file to delete
278
+ return []
279
+ kept = {os.path.realpath(os.path.expanduser(h.key)) for h in parse_list(new) if h.key}
280
+ failed = []
281
+ for h in before:
282
+ path = h.key and os.path.realpath(os.path.expanduser(h.key))
283
+ if not path or path in kept or path != os.path.join(key_dir, _key_name(h.user, h.host)):
284
+ continue
285
+ try:
286
+ os.unlink(path)
287
+ except FileNotFoundError:
288
+ pass
289
+ except OSError as e:
290
+ failed.append((path, e.strerror or str(e)))
291
+ return failed
292
+
293
+
294
+ def own_key(path: Optional[str] = None, run: Optional[Callable] = None) -> str:
295
+ """Nullius's own key pair on this machine, made on first use with no
296
+ passphrase. The private key's path; the public key sits beside it."""
297
+ path = path or OWN_KEY or os.path.join(KEY_DIR or os.path.join(paths.config_dir(), "ssh"), "id_ed25519")
298
+ if not os.path.exists(path):
299
+ os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
300
+ rc, out, err = (run or subprocess_run)(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "nullius", "-f", path], 30)
301
+ if rc != 0:
302
+ raise RuntimeError(f"ssh-keygen failed: {(err or out).strip() or rc}")
303
+ return path
304
+
305
+
306
+ def pty_run(argv: List[str], answer: str, timeout: float = 30.0) -> tuple:
307
+ """Run argv on a pty, type ``answer`` at the first password prompt, and
308
+ return (returncode, what was printed after that). Nothing typed is kept."""
309
+ import pty
310
+ import select
311
+ import signal
312
+ import termios
313
+ pid, fd = pty.fork()
314
+ if pid == 0:
315
+ os.execvp(argv[0], argv)
316
+ try: # no echo of what is typed, whatever the child does
317
+ attrs = termios.tcgetattr(fd); attrs[3] &= ~termios.ECHO; termios.tcsetattr(fd, termios.TCSANOW, attrs)
318
+ except termios.error:
319
+ pass
320
+ deadline = time.monotonic() + timeout
321
+ seen, after, sent = b"", b"", False
322
+ try:
323
+ while True:
324
+ ready, _, _ = select.select([fd], [], [], 0.5)
325
+ if ready:
326
+ try:
327
+ chunk = os.read(fd, 4096)
328
+ except OSError:
329
+ break
330
+ if not chunk:
331
+ break
332
+ if sent:
333
+ after += chunk
334
+ else:
335
+ seen += chunk
336
+ if b"assword" in seen:
337
+ os.write(fd, answer.encode() + b"\n"); sent = True
338
+ if time.monotonic() > deadline:
339
+ os.kill(pid, signal.SIGKILL)
340
+ after += b"\nno answer within %d s" % int(timeout)
341
+ break
342
+ finally:
343
+ os.close(fd)
344
+ _, status = os.waitpid(pid, 0)
345
+ rc = os.waitstatus_to_exitcode(status)
346
+ if not sent:
347
+ after = seen
348
+ return rc, after.replace(answer.encode(), b"").decode("utf-8", "replace")
349
+
350
+
351
+ def enrol(user: str, host: str, port: Optional[int], password: str, key: Optional[str] = None, run_pty: Optional[Callable] = None) -> Host:
352
+ """Put Nullius's public key on the host, using the password once, and
353
+ return the entry that reaches the host with that key from now on."""
354
+ key = key or own_key()
355
+ with open(key + ".pub", encoding="utf-8") as fh:
356
+ pub = fh.read().strip()
357
+ argv = ["ssh", "-o", "PubkeyAuthentication=no", "-o", "NumberOfPasswordPrompts=1", "-o", "ConnectTimeout=8", "-o", "StrictHostKeyChecking=accept-new"]
358
+ if port:
359
+ argv += ["-p", str(port)]
360
+ cmd = f"umask 077; mkdir -p ~/.ssh; grep -qxF '{pub}' ~/.ssh/authorized_keys 2>/dev/null || echo '{pub}' >> ~/.ssh/authorized_keys; echo enrolled"
361
+ rc, out = (run_pty or pty_run)(argv + [f"{user}@{host}", cmd], password)
362
+ if rc != 0 or "enrolled" not in out:
363
+ last = [l for l in out.strip().splitlines() if l.strip()]
364
+ reason = _reason(rc, last[-1] if last else "")
365
+ if "refused" in reason:
366
+ reason = "the password was not accepted"
367
+ raise RuntimeError(reason)
368
+ return parse(f"{user}@{host}" + (f":{port}" if port else "") + f":{key}")
@@ -0,0 +1,140 @@
1
+ """Importers for run trees this app did not produce.
2
+
3
+ The app walks one shape: a job directory whose subdirectories each carry a
4
+ ``result.json``. Everything else — a run tree another harness wrote — is
5
+ converted into that shape rather than taught to the walker: a job directory
6
+ is all the app can walk, so emit one. That keeps `parse.py` a reader of one
7
+ format no matter how many harnesses there are.
8
+
9
+ An importer knows how one harness lays out a whole run on disk; what a
10
+ dataset's verifier report means is `nullius.runs.verifier`'s. A new dataset
11
+ under a harness already read needs nothing here.
12
+
13
+ **Adding one** is a module in this package exposing three names:
14
+
15
+ NAME str what to pass to --harness
16
+ detect (path) -> bool does this directory look like ours?
17
+ convert (src, out_root, job_name="", command="") -> list[str]
18
+
19
+ `convert` returns job directories, plural: one run tree is not always one
20
+ job. A directory that accumulates runs across sittings holds several, and
21
+ pooling them would report the idle time between as though it were work.
22
+
23
+ and optionally ``LABEL``, how a person should see it named — the picker shows
24
+ it, and ``NAME.upper()`` is a poor guess for anything but an acronym —
25
+ ``check``, ``(path) -> None``, which reads the tree the way `convert` would
26
+ and raises ``SystemExit`` for what `convert` would refuse, writing
27
+ nothing (without it, :func:`problem` converts into a scratch directory to
28
+ learn the same thing) — and ``trees``, ``(root) -> list[str]``, which names
29
+ the directories under a root the app opened that are one `convert` each.
30
+ Without it a claimed root is one tree, which is how the CLI reads it. The
31
+ picker uses the same boundaries so both surfaces produce the same jobs.
32
+
33
+ **Refusing a tree** is `convert` raising ``SystemExit`` with a message
34
+ that names the file and what is wrong with it — never a job with no trials,
35
+ which reads as a run that did nothing, and never a traceback, which reads as
36
+ the app breaking. The message is what a person sees, on the picker or from
37
+ the CLI, so it is redacted at construction: a tree is arbitrary bytes from
38
+ wherever a run was made, and a credential in a bad file's path or content
39
+ must not travel into an error just because the file was bad.
40
+
41
+ **Removing one is deleting its module**, plus its tests. Registration is by
42
+ discovery rather than a table here, so there is no second place that keeps
43
+ naming a harness after its file is gone — the failure mode a hand-maintained
44
+ registry has, where the import survives the deletion and the app stops
45
+ loading. Nothing outside a harness module mentions the harness.
46
+
47
+ An empty registry is a valid state - the CLI says ``no importers
48
+ registered`` and every run tree is refused as unrecognised.
49
+ """
50
+ from __future__ import annotations
51
+
52
+ import importlib
53
+ import pkgutil
54
+ import shutil
55
+ import tempfile
56
+ from types import ModuleType
57
+
58
+ from nullius.redact import redact
59
+
60
+ # A module must expose all three to count. A half-written one is skipped
61
+ # rather than half-registered, so an editor's stray file cannot break `--list`.
62
+ _CONTRACT = ("NAME", "detect", "convert")
63
+
64
+
65
+ def available() -> dict[str, ModuleType]:
66
+ """Every importer in this package, keyed by its NAME."""
67
+ found: dict[str, ModuleType] = {}
68
+ for info in pkgutil.iter_modules(__path__):
69
+ if info.name.startswith("_"):
70
+ continue
71
+ try:
72
+ module = importlib.import_module(f"{__name__}.{info.name}")
73
+ except ImportError:
74
+ continue
75
+ if all(hasattr(module, attr) for attr in _CONTRACT):
76
+ found[getattr(module, "NAME")] = module
77
+ return dict(sorted(found.items()))
78
+
79
+
80
+ def get(name: str) -> ModuleType | None:
81
+ """The importer called *name*, or None."""
82
+ return available().get((name or "").strip().lower())
83
+
84
+
85
+ def detect(path: str) -> ModuleType | None:
86
+ """The importer that recognises *path*, or None when nothing does.
87
+
88
+ First match wins in name order, which is stable rather than clever: two
89
+ harnesses claiming one directory is a conflict to fix in `detect`, not to
90
+ arbitrate here by scoring.
91
+ """
92
+ for module in available().values():
93
+ try:
94
+ if module.detect(path):
95
+ return module
96
+ except OSError:
97
+ continue
98
+ return None
99
+
100
+
101
+ def trees(module: ModuleType, root: str) -> list[str]:
102
+ """The trees under *root* that *module* converts one job each.
103
+
104
+ The importer's own ``trees`` when it has one; otherwise *root* itself
105
+ when the importer claims it, and nothing when it does not. A caller
106
+ converts each entry as it would the root a person named on the CLI, so
107
+ the two agree on what one job is.
108
+ """
109
+ if hasattr(module, "trees"):
110
+ return list(module.trees(root))
111
+ return [root] if module.detect(root) else []
112
+
113
+
114
+ def problem(path: str) -> str | None:
115
+ """Why the importer that claims *path* cannot read it — or None.
116
+
117
+ None also when nothing claims *path*: an unclaimed root is one for the
118
+ Harbor reader, and whether it holds a job is that reader's question.
119
+ The message is redacted whichever way it arrived, so a caller can show
120
+ it as it is.
121
+ """
122
+ module = detect(path)
123
+ if module is None:
124
+ return None
125
+ try:
126
+ if hasattr(module, "check"):
127
+ module.check(path)
128
+ else:
129
+ scratch = tempfile.mkdtemp(prefix="nullius-check-")
130
+ try:
131
+ module.convert(path, scratch)
132
+ finally:
133
+ shutil.rmtree(scratch, ignore_errors=True)
134
+ except (OSError, SystemExit) as e:
135
+ return redact(str(e) or f"{module.NAME} could not read it")
136
+ return None
137
+
138
+
139
+ def labels() -> str:
140
+ return ", ".join(getattr(m, "LABEL", name) for name, m in available().items())