nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/progress.py ADDED
@@ -0,0 +1,294 @@
1
+ """What a run is doing, one line per stage: read by a person in a terminal,
2
+ and by the run's page.
3
+
4
+ A run spends minutes before its first unit - a host connected to, ``uv``
5
+ installed, this repository's ``app/`` sent, the docker base image built, an
6
+ agent installed, a dataset downloaded - and a log that only says "running"
7
+ looks stuck all that time. Each of those says where it is in one line::
8
+
9
+ [nullius] stage=image state=start detail="building nullius/base:229ac6f71272 (first time on this machine)" at=2026-10-12T14:03:41Z
10
+ [nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...
11
+ [nullius] stage=unit i=3 of=10 unit=HumanEval/2 state=done outcome=passed reward=1.0 at=...
12
+
13
+ ``key=value`` words after the prefix; a value with a space, a quote or an
14
+ ``=`` is a JSON string. ``state`` is ``start``, ``done``, ``cached`` (there
15
+ already: nothing to do), ``failed`` or ``progress`` (a download's bytes so
16
+ far); ``at`` is when, in UTC, on the machine that said it. ``stage=plan``
17
+ names the stages to come (``stages=image,install,units``), so a page can
18
+ show them waiting before they start.
19
+
20
+ The standard's ``cli`` writes them on stderr - its stdout stays the job's
21
+ path and the units, as before - and into the job's ``progress.jsonl``, one
22
+ JSON object per line, so a mirrored job carries them. ``execution`` writes
23
+ its own for what it does on a host before the run starts (connect, uv, the
24
+ tasks, app/), into the run's log; and reads every one that comes through the
25
+ part's log, so the page follows a run on another machine as it goes
26
+ (``fold``).
27
+
28
+ Nothing here is a secret: a detail names an image, a package, a file.
29
+ """
30
+ from __future__ import annotations
31
+
32
+ import contextlib
33
+ import json
34
+ import os
35
+ import re
36
+ import threading
37
+ import time
38
+ from datetime import datetime, timezone
39
+ from typing import IO, Iterable, List, Optional
40
+
41
+ PREFIX = "[nullius]"
42
+ FILE = "progress.jsonl"
43
+ #: where every reporter also writes its lines, unless it was given a stream: the cli sets it to stderr
44
+ ECHO: Optional[IO] = None
45
+ #: seconds between two lines of one download's bytes
46
+ EVERY = 5.0
47
+
48
+ # What the page calls each stage. One not here is shown by its own name.
49
+ LABELS = {"connect": "Connect", "uv": "uv", "tasks": "Send the tasks", "app": "Send app/", "harbor": "Harbor",
50
+ "image": "Base image", "environment": "Environment", "upstream": "Benchmark source", "dataset": "Dataset",
51
+ "driver": "Driver environment", "install": "Install", "runtime": "Agent runtime", "units": "Units"}
52
+
53
+ _WORD = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=("(?:[^"\\]|\\.)*"|\S*)')
54
+ _BARE = re.compile(r'^[^\s"=]+$')
55
+
56
+
57
+ def _value(v) -> str:
58
+ if isinstance(v, bool):
59
+ return "true" if v else "false"
60
+ if isinstance(v, float):
61
+ v = round(v, 3)
62
+ return str(int(v)) if v == int(v) and abs(v) >= 10 else str(v)
63
+ s = str(v)
64
+ return s if _BARE.match(s) else json.dumps(s, ensure_ascii=False)
65
+
66
+
67
+ def stamp(t: Optional[float] = None) -> str:
68
+ return datetime.fromtimestamp(time.time() if t is None else t, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
69
+
70
+
71
+ def line(stage: str, state: Optional[str] = None, t: Optional[float] = None, **fields) -> str:
72
+ """One stage's line: the prefix, ``stage``, the fields in the order given
73
+ (``state`` second), ``at`` last. A field that is None is left out."""
74
+ words = [f"stage={_value(stage)}"] + ([f"state={state}"] if state else [])
75
+ words += [f"{k}={_value(v)}" for k, v in fields.items() if v is not None]
76
+ return " ".join([PREFIX] + words + [f"at={stamp(t)}"])
77
+
78
+
79
+ def _plain(s: str):
80
+ if re.fullmatch(r"-?\d+", s):
81
+ return int(s)
82
+ if re.fullmatch(r"-?\d+\.\d*(?:[eE][-+]?\d+)?", s):
83
+ return float(s)
84
+ return {"true": True, "false": False}.get(s, s)
85
+
86
+
87
+ def parse(text: str) -> Optional[dict]:
88
+ """The event a line says, or None for any other line. ``t`` is ``at`` in
89
+ seconds since the epoch, when it is one."""
90
+ s = text.strip()
91
+ if not s.startswith(PREFIX + " "):
92
+ return None
93
+ ev = {}
94
+ for k, v in _WORD.findall(s[len(PREFIX):]):
95
+ if v.startswith('"'):
96
+ try:
97
+ ev[k] = json.loads(v)
98
+ except ValueError:
99
+ ev[k] = v.strip('"')
100
+ else:
101
+ ev[k] = _plain(v)
102
+ if not isinstance(ev.get("stage"), str) or not ev["stage"]:
103
+ return None
104
+ try:
105
+ ev["t"] = datetime.strptime(str(ev.get("at", "")), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp()
106
+ except ValueError:
107
+ pass
108
+ return ev
109
+
110
+
111
+ def read(job_dir: str) -> List[dict]:
112
+ """A job's ``progress.jsonl``: every event it holds, a line that is not
113
+ one left out; [] when there is none."""
114
+ out = []
115
+ try:
116
+ with open(os.path.join(job_dir, FILE), encoding="utf-8", errors="replace") as fh:
117
+ for raw in fh:
118
+ try:
119
+ ev = json.loads(raw)
120
+ except ValueError:
121
+ continue
122
+ if isinstance(ev, dict) and ev.get("stage"):
123
+ out.append(ev)
124
+ except OSError:
125
+ pass
126
+ return out
127
+
128
+
129
+ def refusal(events: Iterable[dict]) -> Optional[str]:
130
+ """Why the run was refused before its first unit, when it was: the detail
131
+ of its ``refused`` event; None otherwise."""
132
+ for ev in events:
133
+ if ev.get("stage") == "refused":
134
+ return str(ev.get("detail") or "refused")
135
+ return None
136
+
137
+
138
+ class Reporter:
139
+ """Says each stage: a line on ``echo`` (``ECHO`` when none is given;
140
+ nowhere when that is None too) and an event in ``<job_dir>/progress.jsonl``
141
+ (none without a job). Safe from several threads: units run at once."""
142
+
143
+ def __init__(self, job_dir: Optional[str] = None, echo: Optional[IO] = None):
144
+ self.job_dir = job_dir
145
+ self.echo = echo
146
+ self._lock = threading.Lock()
147
+
148
+ def emit(self, stage: str, state: Optional[str] = None, **fields) -> dict:
149
+ t = time.time()
150
+ text = line(stage, state, t, **fields)
151
+ ev = dict({"stage": stage}, **({"state": state} if state else {}),
152
+ **{k: v for k, v in fields.items() if v is not None}, at=stamp(t), t=round(t, 3))
153
+ with self._lock:
154
+ out = self.echo or ECHO
155
+ if out is not None:
156
+ try:
157
+ out.write(text + "\n")
158
+ out.flush()
159
+ except (OSError, ValueError):
160
+ pass
161
+ if self.job_dir:
162
+ try:
163
+ os.makedirs(self.job_dir, exist_ok=True)
164
+ with open(os.path.join(self.job_dir, FILE), "a", encoding="utf-8") as fh:
165
+ fh.write(json.dumps(ev) + "\n")
166
+ except OSError:
167
+ pass
168
+ return ev
169
+
170
+ @contextlib.contextmanager
171
+ def stage(self, stage: str, *, cached: bool = False, start: Optional[str] = None, **fields):
172
+ """``with reporter.stage("install", agent="codex@2.2.2"):`` says it
173
+ started, then that it is done with its seconds - or failed, with what
174
+ was raised, which goes on up. ``cached``: there already, said once as
175
+ such. ``start``: the start line's detail (what is being done)."""
176
+ if cached:
177
+ self.emit(stage, "cached", **fields)
178
+ yield
179
+ return
180
+ t0 = time.monotonic()
181
+ self.emit(stage, "start", **dict(fields, **({"detail": start} if start else {})))
182
+ try:
183
+ yield
184
+ except BaseException as e:
185
+ self.emit(stage, "failed", seconds=round(time.monotonic() - t0, 1), **dict(fields, detail=str(e)[:300] or type(e).__name__))
186
+ raise
187
+ self.emit(stage, "done", seconds=round(time.monotonic() - t0, 1), **fields)
188
+
189
+ def download(self, stage: str, **fields):
190
+ """A callback for a download's ``(done, total)`` bytes, said at most
191
+ every ``EVERY`` seconds - and once more at the end, whatever the time."""
192
+ last = [0.0]
193
+
194
+ def say(done: int, total: Optional[int], final: bool = False):
195
+ now = time.monotonic()
196
+ if final or now - last[0] >= EVERY:
197
+ last[0] = now
198
+ self.emit(stage, "progress", bytes=int(done), total=int(total) if total else None, **fields)
199
+ return say
200
+
201
+
202
+ def _key(ev: dict) -> str:
203
+ """One stage of the list: its name, and what it is of when there are
204
+ several (an environment's name, the agent installed)."""
205
+ of = ev.get("name") or ev.get("agent") or ""
206
+ return f"{ev['stage']}:{of}" if of else str(ev["stage"])
207
+
208
+
209
+ def label(ev: dict) -> str:
210
+ base = LABELS.get(str(ev["stage"]), str(ev["stage"]))
211
+ of = ev.get("name") or ev.get("agent")
212
+ return f"{base} {of}" if of else base
213
+
214
+
215
+ def fold(events: Iterable[dict]) -> dict:
216
+ """What the page draws, from the events in the order they came:
217
+
218
+ - ``stages``: each stage once, in the order planned (``stage=plan``) or
219
+ first said - ``waiting`` until it starts, then ``running`` (with ``t``,
220
+ when it started), ``done`` or ``cached`` with its ``seconds``, or
221
+ ``failed`` with its detail. A planned stage still waiting when a later
222
+ one has started was not needed (a benchmark with nothing to download),
223
+ and is left out.
224
+ - ``units``: ``n`` to run, the ``noun``, how many ended and how: passed,
225
+ failed, scored, errors; ``running`` the ones started and not ended. A
226
+ run of several parts counts every part's: ``n`` grows as each begins.
227
+ - ``current``: the stage running now, else None; ``ended``: ``done``,
228
+ ``refused`` (with ``detail``) or None while the run goes on.
229
+ """
230
+ stages: dict = {}
231
+ units = None
232
+ ended, detail = None, None
233
+ started_units = 0
234
+ for ev in events:
235
+ st = ev.get("stage")
236
+ state = ev.get("state")
237
+ if st == "plan":
238
+ # a run of several parts (each agent, each dataset) says each part's plan: the one after an end goes on
239
+ ended, detail = None, None
240
+ for name in str(ev.get("stages") or "").split(","):
241
+ name = name.strip()
242
+ if name and name not in stages:
243
+ stages[name] = {"key": name, "stage": name, "label": LABELS.get(name, name), "state": "waiting"}
244
+ continue
245
+ if st == "unit":
246
+ units = units or {"n": None, "noun": "units", "done": 0, "passed": 0, "failed": 0, "scored": 0, "errors": 0, "running": 0}
247
+ if state == "start":
248
+ started_units += 1
249
+ elif state == "done":
250
+ units["done"] += 1
251
+ outcome = ev.get("outcome")
252
+ bucket = {"passed": "passed", "failed": "failed", "scored": "scored"}.get(outcome, "errors")
253
+ units[bucket] += 1
254
+ units["running"] = max(0, started_units - units["done"])
255
+ continue
256
+ if st in ("done", "refused"):
257
+ ended, detail = st, ev.get("detail")
258
+ continue
259
+ key = _key(ev)
260
+ if key not in stages:
261
+ # a planned stage, now said with what it is of: the plan's entry becomes it
262
+ if key != st and st in stages and stages[st]["state"] == "waiting":
263
+ stages = {(key if k == st else k): v for k, v in stages.items()}
264
+ stages[key]["key"] = key
265
+ stages.setdefault(key, {"key": key, "stage": st, "label": label(ev), "state": "waiting"})
266
+ row = stages[key]
267
+ row["label"] = label(ev)
268
+ if ev.get("detail") is not None:
269
+ row["detail"] = ev["detail"]
270
+ if state == "start":
271
+ row.update(state="running", t=ev.get("t"))
272
+ elif state in ("done", "cached", "failed"):
273
+ row["state"] = state
274
+ if ev.get("seconds") is not None:
275
+ row["seconds"] = ev["seconds"]
276
+ elif state == "progress":
277
+ if row["state"] == "waiting":
278
+ row.update(state="running", t=ev.get("t"))
279
+ row["bytes"], row["total"] = ev.get("bytes"), ev.get("total")
280
+ if st == "units":
281
+ units = units or {"n": None, "noun": "units", "done": 0, "passed": 0, "failed": 0, "scored": 0, "errors": 0, "running": 0}
282
+ if ev.get("n") is not None and state == "start":
283
+ units["n"] = (units["n"] or 0) + ev["n"] # each part's units: the run's are all of them
284
+ if ev.get("noun"):
285
+ units["noun"] = ev["noun"]
286
+ rows = list(stages.values())
287
+ seen = [i for i, r in enumerate(rows) if r["state"] != "waiting"]
288
+ last = seen[-1] if seen else -1
289
+ rows = [r for i, r in enumerate(rows) if r["state"] != "waiting" or i > last]
290
+ if ended: # a run that ended has nothing running, nor anything still to come
291
+ rows = [dict(r, state="failed") if r["state"] == "running" and ended == "refused" else r
292
+ for r in rows if r["state"] != "waiting"]
293
+ current = next((r for r in rows if r["state"] == "running"), None)
294
+ return {"stages": rows, "units": units, "current": current["key"] if current else None, "ended": ended, "detail": detail}
nullius/providers.py ADDED
@@ -0,0 +1,335 @@
1
+ """Where a run's model calls go, and the builder's steps.
2
+
3
+ The builder's three providers (command-builder-core.js, ``PROVIDERS``), named
4
+ the same here: what each is called, where it is served, the setting its key is
5
+ read from, and what makes it one a run can start on here. The third,
6
+ ``custom``, is any number of OpenAI-compatible endpoints: the one at
7
+ MODEL_API_URL, and one per id in Settings (``settings.endpoint_ids``). A
8
+ recipe names one by ``provider: custom`` and its ``endpoint`` id; the builder
9
+ and the judge pickers by a choice, ``custom:<id>`` (``choice``, ``split``). Also the builder's
10
+ step numbers, so an error from a compiled recipe names the step that edits it.
11
+ Settings' Test (``test``) asks a provider with free GETs only, and keeps the last
12
+ result per choice (``tests``, ``remember``) for its card and Review.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import http.client
17
+ import json
18
+ import os
19
+ import secrets
20
+ import socket
21
+ import ssl
22
+ import threading
23
+ import time
24
+ import urllib.error
25
+ import urllib.parse
26
+ import urllib.request
27
+ from typing import List
28
+
29
+ from nullius import paths, settings
30
+
31
+ # The builder's steps (command-builder-core.js, STEP): an error names the step that edits it.
32
+ STEP = {"benchmark": 0, "runner": 1, "dataset": 2, "setup": 3, "agent": 4, "model": 5, "limits": 6, "where": 7, "review": 8}
33
+
34
+ # Harbor's card on the builder's Runner step: compiled by the builder's own core
35
+ # (static/builder/command-builder-core.js), not by the application.
36
+ HARBOR = {"key": "harbor", "label": "Harbor", "builtin": True,
37
+ "summary": "Task directories in containers, with the oracle and nop bracket."}
38
+
39
+ # A recipe whose harness is neither Harbor nor the standard: a runner of a benchmark's own scripts,
40
+ # no longer offered - those scripts run as the benchmark's Original setup on the Nullius runner.
41
+ GONE = ("The Native runner is no longer offered: the benchmark's own code runs as its Original setup "
42
+ "on the Nullius runner.")
43
+
44
+ PROVIDERS = ("openrouter", "openai", "custom")
45
+ PROVIDER_LABEL = {"openrouter": "OpenRouter", "openai": "OpenAI", "custom": settings.DEFAULT_ENDPOINT_NAME}
46
+ # Where a provider is served: OpenRouter and OpenAI at their own endpoint,
47
+ # fixed; an OpenAI-compatible endpoint has no default and must be named
48
+ # (Settings: MODEL_API_URL, or an endpoint's own).
49
+ PROVIDER_BASE = {"openrouter": "https://openrouter.ai/api/v1", "openai": "https://api.openai.com/v1", "custom": ""}
50
+ # The setting a run's key is read from, per provider (web/execute.py,
51
+ # credentials_for): a key of its own for OpenRouter and for OpenAI, the
52
+ # endpoint's key for an OpenAI-compatible one.
53
+ PROVIDER_KEY = {"openrouter": "OPENROUTER_API_KEY", "openai": "OPENAI_API_KEY", "custom": "MODEL_API_KEY"}
54
+ # What makes a provider one a run can start on here: its key, for OpenRouter
55
+ # and OpenAI; its endpoint, for an OpenAI-compatible one, whose key is optional.
56
+ PROVIDER_NEEDS = dict(PROVIDER_KEY, custom="MODEL_API_URL")
57
+
58
+
59
+ def endpoint_vars(endpoint: str = "") -> tuple:
60
+ """(URL setting, key setting) of an OpenAI-compatible endpoint: MODEL_API_URL
61
+ and MODEL_API_KEY for none named, else that endpoint's own. An id that is not
62
+ one names settings that are never set, so a run on it never falls back to another."""
63
+ return ("MODEL_API_URL", "MODEL_API_KEY") if not endpoint else (f"MODEL_{endpoint}_API_URL", f"MODEL_{endpoint}_API_KEY")
64
+
65
+
66
+ def choice(provider: str, endpoint: str = "") -> str:
67
+ """A provider as the builder and the judge pickers offer it: ``custom:<id>`` for a named endpoint."""
68
+ return f"custom:{endpoint}" if provider == "custom" and endpoint else provider
69
+
70
+
71
+ def split(value: str) -> tuple:
72
+ """(provider, endpoint id) of a choice; a ``custom:`` with no valid id stays whole, so it is no provider."""
73
+ p, _, e = str(value or "").partition(":")
74
+ return (p, e) if p == "custom" and settings.ENDPOINT_ID.match(e) else (str(value or ""), "")
75
+
76
+
77
+ def _with_url(env) -> List[tuple]:
78
+ """(id, URL) of each OpenAI-compatible endpoint whose URL is set; "" is MODEL_API_URL's."""
79
+ return [(e, env[endpoint_vars(e)[0]]) for e in ("", *settings.endpoint_ids(env)) if env.get(endpoint_vars(e)[0])]
80
+
81
+
82
+ def endpoints(env=os.environ) -> List[dict]:
83
+ """The OpenAI-compatible endpoints a run can be sent to, each whose URL is set and
84
+ holds no credential (``broken``): the one at MODEL_API_URL first, then the named
85
+ ones by name. Names only, never a value."""
86
+ out = [{"id": e, "name": (env.get(f"MODEL_{e}_NAME") if e else "") or e or settings.DEFAULT_ENDPOINT_NAME}
87
+ for e, url in _with_url(env) if not settings.endpoint_url_problem(url)]
88
+ return sorted(out, key=lambda x: (x["id"] != "", x["name"].casefold()))
89
+
90
+
91
+ def broken(env=os.environ) -> List[str]:
92
+ """The endpoint choices whose URL is set but is refused (a credential in it, or no
93
+ http(s) URL): never offered, and a recipe on one is told to fix it in Settings."""
94
+ return [choice("custom", e) for e, url in _with_url(env) if settings.endpoint_url_problem(url)]
95
+
96
+
97
+ def label(value: str, env=os.environ) -> str:
98
+ """What a choice is called: the provider's name, or the endpoint's."""
99
+ p, e = split(value)
100
+ if not e:
101
+ return PROVIDER_LABEL.get(p, p)
102
+ return next((x["name"] for x in endpoints(env) if x["id"] == e), env.get(f"MODEL_{e}_NAME") or e)
103
+
104
+
105
+ def needs(value: str) -> str:
106
+ """The setting that makes a choice one a run can start on."""
107
+ p, e = split(value)
108
+ return endpoint_vars(e)[0] if e else PROVIDER_NEEDS.get(p, PROVIDER_NEEDS["custom"])
109
+
110
+
111
+ # OpenRouter names a model with its maker in front (openai/gpt-4o); a
112
+ # benchmark's official judge or simulator is written as its maker spells it.
113
+ _MAKERS = (("openai/", ("gpt-", "o1", "o3", "o4", "chatgpt-", "text-embedding-")),
114
+ ("anthropic/", ("claude-",)), ("google/", ("gemini-",)))
115
+
116
+
117
+ def served_as(model: str, provider: str) -> str:
118
+ """An official model's name as ``provider`` serves it: on OpenRouter a
119
+ bare name gets its maker's prefix (gpt-4o-2024-08-06 is
120
+ openai/gpt-4o-2024-08-06 there); a name already prefixed, an unknown
121
+ maker or another provider leave it as written."""
122
+ if provider != "openrouter" or "/" in model:
123
+ return model
124
+ for prefix, starts in _MAKERS:
125
+ if model.startswith(starts):
126
+ return prefix + model
127
+ return model
128
+
129
+
130
+ def available(env=os.environ) -> List[str]:
131
+ """The choices whose setting is set (``needs``), in the order the builder
132
+ prefers them: OpenRouter, OpenAI, then each OpenAI-compatible endpoint."""
133
+ return [p for p in PROVIDERS[:2] if env.get(PROVIDER_NEEDS[p])] + [choice("custom", e["id"]) for e in endpoints(env)]
134
+
135
+
136
+ def embedding_problem(model: str, provider: str) -> str:
137
+ """Bare OpenAI embedding names work on OpenAI or OpenRouter; qualified names require OpenRouter."""
138
+ allowed = ("openrouter",) if "/" in model else ("openai", "openrouter")
139
+ if provider not in allowed:
140
+ names = " or ".join(PROVIDER_LABEL[p] for p in allowed)
141
+ return f"{model} needs the {names} provider for this search."
142
+ return ""
143
+
144
+
145
+ def _where(provider: str, endpoint: str, env) -> tuple:
146
+ """(base URL, key) a choice is asked at; ValueError, before any request, when it cannot be asked."""
147
+ url_var, key_var = endpoint_vars(endpoint)
148
+ base = PROVIDER_BASE[provider] if provider != "custom" else env.get(url_var, "")
149
+ key = env.get(key_var if provider == "custom" else PROVIDER_KEY[provider], "")
150
+ if not base:
151
+ raise ValueError("Set the provider endpoint in Settings, or enter a model ID.")
152
+ if provider == "openai" and not key:
153
+ raise ValueError("Add the provider key in Settings, or enter a model ID.")
154
+ parsed = urllib.parse.urlsplit(base)
155
+ if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
156
+ raise ValueError("The provider endpoint must be an HTTP or HTTPS base URL.")
157
+ return base.rstrip("/"), key
158
+
159
+
160
+ class _NoRedirect(urllib.request.HTTPRedirectHandler):
161
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
162
+ return None
163
+
164
+
165
+ _MOST = 8 * 1024 * 1024
166
+
167
+
168
+ def _get(url: str, headers: dict, opener=None) -> bytes:
169
+ """One GET, never another method: no redirect followed, so a key stays at its endpoint; at most 8 MiB read."""
170
+ req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": "nullius", **headers})
171
+ with (opener or urllib.request.build_opener(_NoRedirect()).open)(req, timeout=12) as response:
172
+ content = response.read(_MOST + 1)
173
+ if len(content) > _MOST:
174
+ raise ValueError("response too large")
175
+ return content
176
+
177
+
178
+ def _models(content: bytes) -> list[dict]:
179
+ """The model list of a ``{"data": [{"id": ...}]}`` answer, by name; ValueError when there is none."""
180
+ data = json.loads(content)
181
+ rows = data.get("data") if isinstance(data, dict) else None
182
+ if not isinstance(rows, list):
183
+ raise ValueError("missing model list")
184
+ models = {}
185
+ for row in rows:
186
+ if not isinstance(row, dict):
187
+ continue
188
+ model = row.get("id")
189
+ if not isinstance(model, str) or not model or len(model) > 256 or any(c.isspace() or ord(c) < 32 for c in model):
190
+ continue
191
+ label = row.get("name") or row.get("display_name") or model
192
+ models[model] = {"id": model, "name": label[:256] if isinstance(label, str) else model}
193
+ return sorted(models.values(), key=lambda m: (m["name"].casefold(), m["id"]))
194
+
195
+
196
+ def _bearer(key: str) -> dict:
197
+ return {"Authorization": "Bearer " + key} if key else {}
198
+
199
+
200
+ def list_models(provider: str, env=None, *, opener=None) -> list[dict]:
201
+ """Read model IDs from the selected provider, or ``custom:<id>``'s endpoint; credentials never leave its endpoint."""
202
+ env = os.environ if env is None else env
203
+ provider, endpoint = split(provider)
204
+ if provider not in PROVIDERS:
205
+ raise ValueError("Choose a model provider.")
206
+ base, key = _where(provider, endpoint, env)
207
+ try:
208
+ return _models(_get(base + "/models", _bearer(key), opener))
209
+ except (OSError, ValueError, TypeError, http.client.HTTPException):
210
+ # Provider errors can contain keys or private endpoint details; do not echo them.
211
+ raise ValueError("Model list unavailable. Enter a model ID manually.") from None
212
+
213
+
214
+ # Settings' Test, per provider: only requests that cost nothing. A model list everywhere; OpenRouter's is
215
+ # public, so its key is asked at its own key-information endpoint. TypeSafe has no free request: no Test.
216
+ TESTED = ("openrouter", "openai", "anthropic", "custom")
217
+ ANTHROPIC = ("https://api.anthropic.com/v1", "ANTHROPIC_API_KEY")
218
+ _TESTS_LOCK = threading.Lock()
219
+
220
+
221
+ def _unreached(reason) -> str:
222
+ """Why a request reached nothing, in a few fixed words: never the error's own text, which may hold the URL."""
223
+ for kind, said in ((socket.gaierror, "name not resolved"), (ConnectionRefusedError, "connection refused"),
224
+ (TimeoutError, "timed out"), (ssl.SSLError, "TLS failed")):
225
+ if isinstance(reason, kind):
226
+ return said
227
+ return "not reached"
228
+
229
+
230
+ def test(choice: str, env=None, *, opener=None, clock=time.monotonic) -> dict:
231
+ """A provider asked with GETs that cost nothing, never a completion: its model list (and, for OpenRouter,
232
+ its key's information). An OpenAI-compatible endpoint that answers with a key is asked once more with a
233
+ wrong one: an answer then means it does not check keys. ``{at, ok, reached, key, models, ms, error}``,
234
+ ``key`` one of accepted, refused, not checked (the server checks none), not tested; never a key.
235
+ ValueError, before any request, when the provider has no test or its settings are incomplete."""
236
+ env = os.environ if env is None else env
237
+ provider, endpoint = split(choice)
238
+ if provider not in TESTED:
239
+ raise ValueError("This provider has no test.")
240
+ if provider == "anthropic":
241
+ from nullius.standard.relay import ANTHROPIC_VERSION
242
+ base, key = ANTHROPIC[0], env.get(ANTHROPIC[1], "")
243
+ auth, models = {"x-api-key": key, "anthropic-version": ANTHROPIC_VERSION}, base + "/models?limit=1000"
244
+ else:
245
+ base, key = _where(provider, endpoint, env)
246
+ auth, models = _bearer(key), base + "/models"
247
+ if not key and provider != "custom":
248
+ raise ValueError("Add the provider key in Settings.")
249
+ out = {"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "reached": False, "key": "not tested",
250
+ "models": None, "ms": 0, "error": ""}
251
+ start = clock()
252
+ try:
253
+ # OpenRouter's model list is public: its key goes to its key endpoint only
254
+ out["models"] = len(_models(_get(models, {} if provider == "openrouter" else auth, opener)))
255
+ out["reached"] = True
256
+ if provider == "openrouter":
257
+ _get(base + "/key", auth, opener)
258
+ out["key"] = "accepted"
259
+ elif provider != "custom":
260
+ out["key"] = "accepted"
261
+ elif not key:
262
+ out["key"] = "not checked"
263
+ else:
264
+ try:
265
+ _get(models, _bearer("nullius-wrong-" + secrets.token_hex(8)), opener)
266
+ out["key"] = "not checked"
267
+ except urllib.error.HTTPError as e:
268
+ e.close()
269
+ out["key"] = "accepted" if e.code in (401, 403) else "not tested"
270
+ except (OSError, http.client.HTTPException):
271
+ pass
272
+ except urllib.error.HTTPError as e:
273
+ e.close()
274
+ out["reached"] = True
275
+ if e.code in (401, 403):
276
+ out["key"] = "refused"
277
+ else:
278
+ out["error"] = f"HTTP {e.code}"
279
+ except urllib.error.URLError as e:
280
+ out["error"] = _unreached(e.reason)
281
+ except (OSError, http.client.HTTPException) as e:
282
+ out["error"] = _unreached(e)
283
+ except ValueError:
284
+ out.update(reached=True, error="no model list")
285
+ out["ms"] = round((clock() - start) * 1000)
286
+ out["ok"] = out["reached"] and not out["error"] and out["key"] != "refused"
287
+ return out
288
+
289
+
290
+ def _tests_file(env=None) -> str:
291
+ return os.path.join(paths.data_dir(env), "provider-tests.json")
292
+
293
+
294
+ def tests(env=None) -> dict:
295
+ """Each provider's last test by choice; none when the file is missing or unreadable."""
296
+ try:
297
+ with open(_tests_file(env), encoding="utf-8") as f:
298
+ got = json.load(f)
299
+ except (OSError, ValueError):
300
+ return {}
301
+ return got if isinstance(got, dict) else {}
302
+
303
+
304
+ def remember(results: dict, env=None) -> None:
305
+ """Set (a result) or forget (None) the last test of each choice, the file written whole and renamed."""
306
+ with _TESTS_LOCK:
307
+ kept = {k: v for k, v in {**tests(env), **results}.items() if v is not None}
308
+ out = _tests_file(env)
309
+ os.makedirs(os.path.dirname(out), exist_ok=True)
310
+ part = f"{out}.{os.getpid()}.part"
311
+ with open(part, "w", encoding="utf-8") as f:
312
+ json.dump(kept, f, indent=1, sort_keys=True)
313
+ os.replace(part, out)
314
+
315
+
316
+ def mark(result: dict) -> str:
317
+ """ok, warn (it answers but its key is not known good) or bad: the colour of a test's line."""
318
+ return "bad" if not result.get("ok") else "ok" if result.get("key") == "accepted" else "warn"
319
+
320
+
321
+ _KEY_SAID = {"refused": "key refused", "not checked": "does not check keys", "not tested": "key not tested"}
322
+
323
+
324
+ def said(result: dict) -> str:
325
+ """A test in one line: what failed, or the key, the model count and the time it took; then when."""
326
+ when = str(result.get("at", "")).replace("T", " ")[:16] + " UTC"
327
+ if result.get("error"):
328
+ head = ("" if result.get("reached") else "Not reached: ") + result["error"]
329
+ elif result.get("key") == "refused":
330
+ head = "Key refused"
331
+ else:
332
+ n = result.get("models")
333
+ head = " · ".join([("Key accepted" if result.get("key") == "accepted" else "Reached · " + _KEY_SAID.get(result.get("key"), "")),
334
+ f"{n} model{'s' * (n != 1)}", f"{result.get('ms', 0)} ms"])
335
+ return f"{head} · {when}"
File without changes