nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/gitrefs.py ADDED
@@ -0,0 +1,128 @@
1
+ """What a git repository offers - its default branch, its branches and its
2
+ tags, each at its commit - read with ``git ls-remote``.
3
+
4
+ One lister for the picker that names a version of a repository: the Harbor a
5
+ recipe pins (``harbor/source.py``). It asks the repository itself, as this machine's git
6
+ would: with the machine's own credentials - an ssh key, a credential helper -
7
+ and never a token of its own, so a private repository lists when this
8
+ machine can clone it, and there is no API rate limit to meet. Nothing is
9
+ prompted for (``GIT_TERMINAL_PROMPT=0``, ssh in batch mode) and a repository
10
+ that does not answer within the timeout is said to, not waited on.
11
+
12
+ ``ls-remote`` carries no dates, so the order is the names': tags by version,
13
+ newest first, an annotated tag read through to the commit it points at
14
+ (its ``^{}`` line); branches with the default first, then by name.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import os
19
+ import re
20
+ import subprocess
21
+ from typing import Callable, List, Optional
22
+
23
+ GIT = "git" # the program; a test or a probe points it elsewhere
24
+ TIMEOUT = 30.0
25
+ LISTED = 50 # the most of each kind a picker is given; the totals say how many there are
26
+ COMMIT = re.compile(r"^[0-9a-f]{40}$")
27
+ _SLUG = re.compile(r"^([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)$")
28
+ _URL = re.compile(r"^(?:https?|ssh|git|file)://\S+$")
29
+ _SCP = re.compile(r"^[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+:\S+$")
30
+ _GITHUB = re.compile(r"^(?:https://github\.com/|ssh://git@github\.com/|git@github\.com:)([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?)(?:\.git)?/?$")
31
+
32
+
33
+ def url_of(text: str) -> Optional[str]:
34
+ """The URL git is given for what a person typed: ``owner/name`` is that
35
+ repository on GitHub over https; an https, ssh, git or file URL, or an
36
+ scp-like ``user@host:path``, is taken as it is. Anything else - an
37
+ option, a word with spaces - is None."""
38
+ text = (text or "").strip()
39
+ if not text or text.startswith("-") or any(c.isspace() or ord(c) < 32 for c in text):
40
+ return None
41
+ m = _SLUG.match(text)
42
+ if m and not text.startswith("."):
43
+ name = m.group(2)[:-4] if m.group(2).endswith(".git") else m.group(2)
44
+ return f"https://github.com/{m.group(1)}/{name}.git"
45
+ return text if _URL.match(text) or _SCP.match(text) else None
46
+
47
+
48
+ def label_of(url: str) -> str:
49
+ """How a repository is named on a page: ``owner/name`` on GitHub, else its URL."""
50
+ m = _GITHUB.match(url or "")
51
+ return m.group(1) if m else url
52
+
53
+
54
+ def env() -> dict:
55
+ """The environment git runs in: this process's, with no prompt possible."""
56
+ out = dict(os.environ, GIT_TERMINAL_PROMPT="0")
57
+ if not out.get("GIT_SSH_COMMAND") and not out.get("GIT_SSH"):
58
+ out["GIT_SSH_COMMAND"] = "ssh -o BatchMode=yes"
59
+ return out
60
+
61
+
62
+ def run_git(args: List[str], timeout: float = TIMEOUT) -> tuple:
63
+ """``git <args>``: (returncode, stdout, stderr); a missing git and a
64
+ timeout are return codes of their own."""
65
+ try:
66
+ p = subprocess.run([GIT, *args], capture_output=True, text=True, timeout=timeout, env=env(), stdin=subprocess.DEVNULL)
67
+ except OSError as e:
68
+ return 127, "", f"{GIT}: {e.strerror or e}"
69
+ except subprocess.TimeoutExpired:
70
+ return 124, "", f"no answer within {int(timeout)} s"
71
+ return p.returncode, p.stdout, p.stderr
72
+
73
+
74
+ def version_key(name: str):
75
+ """A tag or a version name as something to sort by: its numbers as
76
+ numbers, from the first digit on (``v1.10`` after ``v1.9``, ``harbor-0.3``
77
+ beside ``0.3``), a pre-release before its release; a name with no digit
78
+ sorts below every one that has one."""
79
+ m = re.search(r"\d", name)
80
+ if not m:
81
+ return (0, [], name)
82
+ parts = [(1, int(p)) if p.isdigit() else (0, p.lower()) for p in re.findall(r"\d+|[A-Za-z]+", name[m.start():])]
83
+ return (1, parts + [(0.5, "")], name)
84
+
85
+
86
+ def parse(text: str) -> dict:
87
+ """``git ls-remote --symref`` output as the default branch, the branches
88
+ and the tags, each ``{name, commit}``, in the order a picker shows them."""
89
+ default, branches, tags, peeled = "", {}, {}, {}
90
+ for line in text.splitlines():
91
+ left, _, ref = line.partition("\t")
92
+ if left.startswith("ref: ") and ref.strip() == "HEAD":
93
+ target = left[5:].strip()
94
+ if target.startswith("refs/heads/"):
95
+ default = target[len("refs/heads/"):]
96
+ continue
97
+ sha, ref = left.strip(), ref.strip()
98
+ if not COMMIT.match(sha):
99
+ continue
100
+ if ref.startswith("refs/heads/"):
101
+ branches[ref[len("refs/heads/"):]] = sha
102
+ elif ref.startswith("refs/tags/"):
103
+ name = ref[len("refs/tags/"):]
104
+ if name.endswith("^{}"):
105
+ peeled[name[:-3]] = sha # an annotated tag: the commit it names
106
+ else:
107
+ tags[name] = sha
108
+ tags.update({k: v for k, v in peeled.items() if k in tags})
109
+ order = sorted(branches, key=lambda n: (n != default, n))
110
+ return {"default_branch": default,
111
+ "branches": [{"name": n, "commit": branches[n]} for n in order],
112
+ "tags": [{"name": n, "commit": tags[n]} for n in sorted(tags, key=version_key, reverse=True)]}
113
+
114
+
115
+ def ls_remote(url: str, run: Optional[Callable] = None, listed: int = LISTED) -> dict:
116
+ """What ``url`` offers: ``{default_branch, branches, branches_total,
117
+ tags, tags_total, error}``, at most ``listed`` of each kind. On any
118
+ failure the lists are empty and ``error`` is git's own last word."""
119
+ out = {"default_branch": "", "branches": [], "branches_total": 0, "tags": [], "tags_total": 0, "error": ""}
120
+ if url_of(url) is None:
121
+ return dict(out, error="not a repository URL")
122
+ rc, text, err = (run or run_git)(["ls-remote", "--symref", url, "HEAD", "refs/heads/*", "refs/tags/*"])
123
+ if rc != 0:
124
+ said = [l.strip() for l in (err or text or "").splitlines() if l.strip()]
125
+ return dict(out, error=(said[-1] if said else f"git exited {rc}").removeprefix("fatal: "))
126
+ got = parse(text)
127
+ return dict(out, default_branch=got["default_branch"], branches=got["branches"][:listed], branches_total=len(got["branches"]),
128
+ tags=got["tags"][:listed], tags_total=len(got["tags"]))
File without changes
nullius/harbor/feed.py ADDED
@@ -0,0 +1,228 @@
1
+ """The live feed for any Harbor: ``events.jsonl`` written from Harbor's own trial hooks.
2
+
3
+ A run's ``harbor`` script (``source.wrapper``) runs this file, sent beside it
4
+ as ``harbor-feed.py``, with the pinned
5
+ Harbor's Python instead of the ``harbor`` command: it registers the hooks
6
+ every Harbor ``Job`` offers (``on_trial_started`` … ``on_trial_ended``), then
7
+ hands its arguments to Harbor's own command line, unchanged. Each job then
8
+ writes the feed ``nullius.runs.live`` folds into the job page's Live view -
9
+ the same lines, in the same schema - whatever Harbor it is: a PyPI release or
10
+ a repository at a commit.
11
+
12
+ What a Harbor without its own feed cannot give is left out, never made up:
13
+ no agent-setup phase (its hooks go from the environment to the agent), no
14
+ step or token counts while the agent runs; the trial's own result fills
15
+ the totals at its end. A phase's limit is the trial's own (its
16
+ ``*_timeout_sec`` properties, multipliers applied), read from the trial
17
+ object the feed keeps by name since the hooks do not carry it. A Harbor that writes the feed itself
18
+ (its ``Job`` has a live feed of its own) is left alone, so a job never has two
19
+ writers.
20
+
21
+ Nothing here imports Harbor until :func:`main` runs inside Harbor's
22
+ environment, so this module loads in this application like any other; a
23
+ failure of the feed is said once on stderr and never stops the run.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import json
28
+ import os
29
+ import sys
30
+ import threading
31
+ from datetime import datetime, timezone
32
+
33
+ FEED = "events.jsonl"
34
+ TRIALS: dict = {} # trial name -> the Harbor trial object, kept as each is made, for its limits
35
+ LIMITS = {"env_setup": "environment_build_timeout_sec", "agent_run": "agent_timeout_sec", "verify": "verifier_timeout_sec"}
36
+ # an exception type naming the infrastructure, not the agent: the board's infra_error - the provider's
37
+ # (Api*, a refused key, a model name it does not serve) as the classifier counts them (nullius.runs.classify)
38
+ INFRA = ("Api", "Authentication", "ModelNotFound", "RateLimit", "Setup", "Environment", "Docker", "Build", "Connection", "Network")
39
+
40
+
41
+ class Writer:
42
+ """One job's feed: a line per event, ``seq`` strictly increasing, each
43
+ line flushed as written so a reader folds it as it grows."""
44
+
45
+ def __init__(self, path: str, job: str):
46
+ self.path, self.job, self.seq = path, job, 0
47
+ self.lock = threading.Lock()
48
+
49
+ def emit(self, event: str, trial=None, **data) -> None:
50
+ with self.lock:
51
+ self.seq += 1
52
+ line = {"seq": self.seq, "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
53
+ "job": self.job, "event": event, "trial": trial}
54
+ line.update({k: v for k, v in data.items() if v is not None})
55
+ with open(self.path, "a", encoding="utf-8") as fh:
56
+ fh.write(json.dumps(line, default=str) + "\n")
57
+
58
+
59
+ def outcome_of(result) -> tuple:
60
+ """``(outcome, reward, evidence)`` of a finished trial, in the feed's words:
61
+ the reward when the verifier gave one, else what the exception says."""
62
+ rewards = getattr(getattr(result, "verifier_result", None), "rewards", None)
63
+ reward = rewards.get("reward") if isinstance(rewards, dict) else None
64
+ info = getattr(result, "exception_info", None)
65
+ kind = (getattr(info, "exception_type", "") or "") if info is not None else ""
66
+ if "Cancel" in kind:
67
+ return "cancelled", reward, kind
68
+ if reward is not None:
69
+ return ("resolved" if reward >= 1 else "unresolved"), reward, f"reward {reward}"
70
+ if "Timeout" in kind:
71
+ return "timeout", None, kind
72
+ if kind:
73
+ return ("infra_error" if any(w in kind for w in INFRA) else "agent_error"), None, kind
74
+ return "unresolved", None, "no reward"
75
+
76
+
77
+ def _seconds(timing):
78
+ start, end = getattr(timing, "started_at", None), getattr(timing, "finished_at", None)
79
+ return (end - start).total_seconds() if start is not None and end is not None else None
80
+
81
+
82
+ def _agent_name(agent) -> str:
83
+ return getattr(agent, "name", None) or getattr(agent, "import_path", None) or "agent"
84
+
85
+
86
+ def _task_name(config) -> str:
87
+ task = getattr(config, "task", None)
88
+ try:
89
+ return task.get_task_id().get_name()
90
+ except Exception:
91
+ return str(getattr(task, "path", "") or "")
92
+
93
+
94
+ def _said_once(e) -> None:
95
+ if not _SAID:
96
+ _SAID.append(e)
97
+ print(f"nullius: the live feed stopped: {e!r}; the run goes on", file=sys.stderr)
98
+
99
+
100
+ _SAID: list = []
101
+
102
+
103
+ def _safe(hook):
104
+ """A hook whose failure is said once and never reaches Harbor's trial."""
105
+ async def run(event):
106
+ try:
107
+ await hook(event)
108
+ except Exception as e:
109
+ _said_once(e)
110
+ return run
111
+
112
+
113
+ def attach(job, writer: Writer) -> None:
114
+ """Register the feed's hooks on a Harbor ``Job`` and say the job started:
115
+ its trials, each queued in the order Harbor will run them."""
116
+ def who(event) -> dict:
117
+ agent = getattr(event.config, "agent", None)
118
+ return {"task": event.task_name, "agent": _agent_name(agent), "model": getattr(agent, "model_name", None)}
119
+
120
+ def limit(trial_name, name):
121
+ try:
122
+ v = getattr(TRIALS.get(trial_name), LIMITS[name], None)
123
+ return float(v) if isinstance(v, (int, float)) and v > 0 else None
124
+ except Exception:
125
+ return None
126
+
127
+ def phase(name):
128
+ async def hook(event):
129
+ writer.emit("trial.phase", event.trial_name, phase=name, timeout_sec=limit(event.trial_name, name), **who(event))
130
+ return hook
131
+
132
+ async def ended(event):
133
+ result = event.result
134
+ outcome, reward, evidence = outcome_of(result)
135
+ usage = getattr(result, "agent_result", None)
136
+ writer.emit("trial.end", event.trial_name, outcome=outcome, reward=reward, evidence=evidence,
137
+ exception_type=getattr(getattr(result, "exception_info", None), "exception_type", None),
138
+ durations={"env_setup": _seconds(result.environment_setup), "agent_setup": _seconds(result.agent_setup),
139
+ "agent_run": _seconds(result.agent_execution), "verify": _seconds(result.verifier)},
140
+ n_input_tokens=getattr(usage, "n_input_tokens", None), n_cache_tokens=getattr(usage, "n_cache_tokens", None),
141
+ n_output_tokens=getattr(usage, "n_output_tokens", None), cost_usd=getattr(usage, "cost_usd", None),
142
+ **who(event))
143
+
144
+ async def cancelled(event):
145
+ writer.emit("trial.progress", event.trial_name, activity="cancellation requested", **who(event))
146
+
147
+ job.on_environment_started(_safe(phase("env_setup")))
148
+ job.on_agent_started(_safe(phase("agent_run")))
149
+ job.on_verification_started(_safe(phase("verify")))
150
+ job.on_trial_ended(_safe(ended))
151
+ job.on_trial_cancelled(_safe(cancelled))
152
+ configs = list(getattr(job, "_remaining_trial_configs", None) or getattr(job, "_trial_configs", None) or [])
153
+ writer.emit("job.start", n_total=len(getattr(job, "_trial_configs", None) or configs),
154
+ n_concurrent=getattr(job.config, "n_concurrent_trials", None),
155
+ agents=sorted({_agent_name(c.agent) for c in configs}), tasks=sorted({_task_name(c) for c in configs}))
156
+ for position, c in enumerate(configs):
157
+ writer.emit("trial.queued", c.trial_name, task=_task_name(c), agent=_agent_name(c.agent),
158
+ model=getattr(c.agent, "model_name", None), position=position)
159
+
160
+
161
+ def keep_trials(Trial) -> None:
162
+ """Wrap ``Trial.__init__`` so every trial is kept by its name: the hooks
163
+ name a trial, and its limits are on the object."""
164
+ made = Trial.__init__
165
+
166
+ def init(self, *args, **kwargs):
167
+ made(self, *args, **kwargs)
168
+ try:
169
+ TRIALS[self.config.trial_name] = self
170
+ except Exception as e:
171
+ _said_once(e)
172
+
173
+ Trial.__init__ = init
174
+
175
+
176
+ def install(Job, Trial=None) -> bool:
177
+ """Wrap ``Job.run`` so each job writes the feed (and ``Trial`` so its
178
+ phases carry their limits). False, and nothing changed, when this Harbor
179
+ writes its own."""
180
+ if hasattr(Job, "_init_live_feed"):
181
+ return False
182
+ if Trial is not None:
183
+ keep_trials(Trial)
184
+ run = Job.run
185
+ warn = _said_once
186
+
187
+ async def run_with_feed(self, *args, **kwargs):
188
+ writer = None
189
+ try:
190
+ os.makedirs(str(self.job_dir), exist_ok=True)
191
+ writer = Writer(os.path.join(str(self.job_dir), FEED), self.config.job_name)
192
+ attach(self, writer)
193
+ except Exception as e: # the feed is a view: never the run's undoing
194
+ warn(e)
195
+ result = await run(self, *args, **kwargs)
196
+ if writer is not None:
197
+ try:
198
+ stats = getattr(result, "stats", None)
199
+ writer.emit("job.end", n_completed=getattr(stats, "n_completed_trials", None),
200
+ n_errored=getattr(stats, "n_errored_trials", None), n_cancelled=getattr(stats, "n_cancelled_trials", None))
201
+ except Exception as e:
202
+ warn(e)
203
+ return result
204
+
205
+ Job.run = run_with_feed
206
+ return True
207
+
208
+
209
+ def main() -> None:
210
+ """``python feed.py <harbor arguments>``: Harbor's command line, with the feed."""
211
+ try:
212
+ from harbor.job import Job
213
+ try:
214
+ from harbor.trial.trial import Trial
215
+ except Exception:
216
+ Trial = None
217
+ install(Job, Trial)
218
+ except Exception as e:
219
+ print(f"nullius: no live feed with this Harbor: {e!r}; the run goes on", file=sys.stderr)
220
+ from importlib.metadata import entry_points
221
+ # the `harbor` command as this Harbor declares it, whatever module it lives in
222
+ command = next(iter(entry_points(group="console_scripts", name="harbor"))).load()
223
+ sys.argv = ["harbor"] + sys.argv[1:]
224
+ sys.exit(command())
225
+
226
+
227
+ if __name__ == "__main__":
228
+ main()