nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,785 @@
1
+ """The model endpoint every agent, driver and capability of a run is given.
2
+
3
+ It listens on 127.0.0.1, holds the real key and adds it on the way out, so
4
+ what it relays for never holds one: their requests carry a placeholder. Each
5
+ unit has a route of its own, ``http://127.0.0.1:<port>/<tag>/v1``, so a call is
6
+ known by the unit that made it without reading its content.
7
+
8
+ A reply is passed through as it arrives - a streamed one (server-sent events)
9
+ byte for byte - and recorded once it is complete, one line per call in
10
+ ``<record_dir>/<tag>.jsonl``: when it started and ended, the model, the status,
11
+ the tokens, and an abridged request and reply. Three API shapes are read,
12
+ Chat Completions, Responses and Anthropic's Messages, streamed or not. The
13
+ cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an
14
+ estimate from ``prices`` when one is given for exactly that model (the
15
+ user's price list, ``nullius.prices``), marked as an estimate.
16
+
17
+ What the relay changes on the way is kept small, so a call through it costs
18
+ what the same call made directly would: the connection to the provider is
19
+ kept open between calls (a new TLS handshake per call was most of what the
20
+ relay added); the caller's headers go on as they came - an agent's session
21
+ and capability headers are the provider's to read - except the credentials,
22
+ which the relay replaces with its own key, and the headers of the hop itself;
23
+ the provider's headers come back the same way (its request id, its rate
24
+ limits). It never answers for the provider: a 429 reaches the caller and the
25
+ record as it was sent, which is what makes rate limits measurable.
26
+
27
+ One answer is the relay's own: a run with a spending cap (``ledger``) books
28
+ each call before it goes out, and a call that does not fit is not sent. Its
29
+ caller gets a 402 that names the relay and the cap, and the record says
30
+ ``refused``, so it never reads as the provider's error.
31
+ """
32
+ from __future__ import annotations
33
+
34
+ import hashlib
35
+ import http.client
36
+ import json
37
+ import os
38
+ import re
39
+ import select
40
+ import socket
41
+ import threading
42
+ import time
43
+ import urllib.parse
44
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
45
+ from typing import Dict, List, Optional
46
+
47
+ from nullius.redact import redact
48
+ from nullius.standard import ledger as ledgers
49
+
50
+ PLACEHOLDER_KEY = "nullius-relay"
51
+ _TAG = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
52
+ TIMEOUT = 900 # one call, streamed or not: a long reasoning turn is still one call
53
+ # The headers of one hop, which a proxy never passes on (RFC 9110, 7.6.1).
54
+ _HOP = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "proxy-connection", "te", "trailer",
55
+ "trailers", "transfer-encoding", "upgrade"}
56
+ # Not passed on from the caller: the hop's, the ones the relay sets itself
57
+ # (the body it may have rewritten has a length of its own; it asks for an
58
+ # uncompressed reply so it can read the reply), and cookies.
59
+ _NOT_SENT = _HOP | {"host", "content-length", "accept-encoding", "cookie", "cookie2"}
60
+ # A credential, whatever the vendor calls it: the relay's key replaces it.
61
+ # The account headers go with them - the relay's key says whose account it is.
62
+ _CREDENTIAL = re.compile(r"authorization|api[-_]?key|auth|token|secret|password|credential|"
63
+ r"^openai-organization$|^openai-project$", re.I)
64
+ # Not passed back: the hop's, cookies, and what this server writes itself.
65
+ _NOT_RETURNED = _HOP | {"set-cookie", "set-cookie2", "content-length", "date", "server"}
66
+ # A kept connection the provider closed while it sat idle fails like this on
67
+ # its next use, before any of the reply is read: worth one new connection.
68
+ _STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError)
69
+ ANTHROPIC_VERSION = "2023-06-01"
70
+
71
+
72
+ def _cut(text, n=4000):
73
+ if not isinstance(text, str) or len(text) <= n:
74
+ return text
75
+ return text[: n // 4] + f"\n[... {len(text) - n} characters left out ...]\n" + text[-(3 * n // 4):]
76
+
77
+
78
+ def _said(status: int, body: bytes) -> str:
79
+ try:
80
+ e = json.loads(body).get("error") or {}
81
+ m = e.get("message") if isinstance(e, dict) else str(e)
82
+ except Exception:
83
+ m = body[:200].decode("utf-8", "replace")
84
+ return f"HTTP {status}: {m or ''}".strip()[:300]
85
+
86
+
87
+ def _usage(u: Optional[dict]) -> Optional[dict]:
88
+ """Chat Completions', Responses' and Messages' usage, in one shape:
89
+ ``input`` counts every prompt token, ``cached`` those of them a cache
90
+ served. ``cache_write`` counts writes separately; ``cache_write_1h`` is
91
+ their one-hour subset when reported. Anthropic's ``input_tokens`` excludes
92
+ cache reads and writes, so both are added to input, once. Chat Completions
93
+ already includes both in ``prompt_tokens``."""
94
+ if not isinstance(u, dict):
95
+ return None
96
+ inp = u.get("prompt_tokens", u.get("input_tokens")) or 0
97
+ out = u.get("completion_tokens", u.get("output_tokens")) or 0
98
+ det = u.get("prompt_tokens_details") or u.get("input_tokens_details") or {}
99
+ rdet = u.get("completion_tokens_details") or u.get("output_tokens_details") or {}
100
+ cached = (det or {}).get("cached_tokens") or 0
101
+ written = u.get("cache_creation_input_tokens") or 0
102
+ if "cache_read_input_tokens" in u or "cache_creation_input_tokens" in u:
103
+ cached = u.get("cache_read_input_tokens") or 0
104
+ inp += cached + written
105
+ else:
106
+ written = (det or {}).get("cache_write_tokens") or 0
107
+ got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0}
108
+ if written:
109
+ got["cache_write"] = written
110
+ hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0
111
+ if hour:
112
+ got["cache_write_1h"] = min(hour, written)
113
+ if isinstance(u.get("cost"), (int, float)):
114
+ got["cost"] = u["cost"]
115
+ return got
116
+
117
+
118
+ def _text_of_responses(resp: dict) -> (str, List[dict]):
119
+ text, calls = [], []
120
+ for item in resp.get("output") or []:
121
+ if item.get("type") == "message":
122
+ for c in item.get("content") or []:
123
+ if c.get("type") in ("output_text", "text"):
124
+ text.append(c.get("text") or "")
125
+ elif item.get("type") in ("function_call", "custom_tool_call", "local_shell_call"):
126
+ calls.append({"name": item.get("name") or item.get("type"),
127
+ "arguments": _cut(item.get("arguments") or item.get("input") or json.dumps(item.get("action")), 2000)})
128
+ return "".join(text), calls
129
+
130
+
131
+ def _text_of_messages(msg: dict) -> (str, List[dict]):
132
+ text, calls = [], []
133
+ for c in msg.get("content") or []:
134
+ if c.get("type") == "text":
135
+ text.append(c.get("text") or "")
136
+ elif c.get("type") in ("tool_use", "server_tool_use"):
137
+ calls.append({"name": c.get("name"), "arguments": _cut(json.dumps(c.get("input")), 2000)})
138
+ return "".join(text), calls
139
+
140
+
141
+ def read_reply(path: str, body: bytes, streamed: bool) -> dict:
142
+ """What a reply said: its text, its tool calls, its usage."""
143
+ if not streamed:
144
+ try:
145
+ got = json.loads(body)
146
+ except ValueError:
147
+ return {}
148
+ if "choices" in got:
149
+ m = ((got.get("choices") or [{}])[0].get("message")) or {}
150
+ calls = [{"name": (c.get("function") or {}).get("name"), "arguments": _cut((c.get("function") or {}).get("arguments"), 2000)}
151
+ for c in m.get("tool_calls") or []]
152
+ return {"text": m.get("content"), "tool_calls": calls, "usage": _usage(got.get("usage"))}
153
+ if got.get("type") == "message" and isinstance(got.get("content"), list):
154
+ text, calls = _text_of_messages(got)
155
+ return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}
156
+ if got.get("object") == "response" or "output" in got:
157
+ text, calls = _text_of_responses(got)
158
+ return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}
159
+ if got.get("object") == "list" and path.split("?")[0].endswith("/embeddings"): # how many inputs, and the usage: never the vectors
160
+ return {"embeddings": len(got.get("data") or []), "usage": _usage(got.get("usage"))}
161
+ return {}
162
+ text, calls, usage, args = [], {}, None, {}
163
+ told = None # Messages: the usage of message_start, then message_delta's on it
164
+ for line in body.decode("utf-8", "replace").splitlines():
165
+ if not line.startswith("data:"):
166
+ continue
167
+ data = line[5:].strip()
168
+ if not data or data == "[DONE]":
169
+ continue
170
+ try:
171
+ ev = json.loads(data)
172
+ except ValueError:
173
+ continue
174
+ kind = ev.get("type")
175
+ if kind == "message_start":
176
+ told = dict(((ev.get("message") or {}).get("usage")) or {})
177
+ continue
178
+ if kind == "message_delta":
179
+ told = {**(told or {}), **{k: v for k, v in (ev.get("usage") or {}).items() if v is not None}}
180
+ continue
181
+ if kind == "content_block_start":
182
+ b, i = ev.get("content_block") or {}, ev.get("index", 0)
183
+ if b.get("type") in ("tool_use", "server_tool_use"):
184
+ calls[i], args[i] = b.get("name"), ""
185
+ elif b.get("text"):
186
+ text.append(b["text"])
187
+ continue
188
+ if kind == "content_block_delta":
189
+ d, i = ev.get("delta") or {}, ev.get("index", 0)
190
+ if d.get("type") == "text_delta":
191
+ text.append(d.get("text") or "")
192
+ elif d.get("type") == "input_json_delta":
193
+ args[i] = args.get(i, "") + (d.get("partial_json") or "")
194
+ continue
195
+ if kind in ("response.completed", "response.done", "response.incomplete"):
196
+ t, c = _text_of_responses(ev.get("response") or {})
197
+ return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))}
198
+ for ch in ev.get("choices") or []:
199
+ d = ch.get("delta") or {}
200
+ if d.get("content"):
201
+ text.append(d["content"])
202
+ for tc in d.get("tool_calls") or []:
203
+ i = tc.get("index", 0)
204
+ f = tc.get("function") or {}
205
+ if f.get("name"):
206
+ calls[i] = f["name"]
207
+ args[i] = args.get(i, "") + (f.get("arguments") or "")
208
+ if ev.get("usage"):
209
+ usage = _usage(ev["usage"])
210
+ return {"text": "".join(text), "tool_calls": [{"name": calls[i], "arguments": _cut(args.get(i, ""), 2000)} for i in sorted(calls)],
211
+ "usage": _usage(told) if told is not None else usage}
212
+
213
+
214
+ def _request_tail(req: dict) -> dict:
215
+ """The part of a request a reader needs: the model, how many messages,
216
+ the last two, abridged, and the reasoning effort it asked for (chat's
217
+ ``reasoning_effort``, the Responses API's ``reasoning.effort``; None when
218
+ it sent none), so a run shows the effort its agent really sent. A coding
219
+ agent resends its whole context on every call; the trajectory comes from
220
+ the session, not from here."""
221
+ items = req.get("messages") if isinstance(req.get("messages"), list) else req.get("input")
222
+ if isinstance(items, str):
223
+ items = [{"role": "user", "content": items}]
224
+ items = items if isinstance(items, list) else []
225
+ tail = []
226
+ for m in items[-2:]:
227
+ if isinstance(m, dict):
228
+ c = m.get("content")
229
+ if isinstance(c, list):
230
+ c = " ".join(str(x.get("text") or "") for x in c if isinstance(x, dict))
231
+ tail.append({"role": m.get("role") or m.get("type"), "content": _cut(c if isinstance(c, str) else json.dumps(c) if c else "", 2000)})
232
+ effort = req.get("reasoning_effort")
233
+ if effort is None and isinstance(req.get("reasoning"), dict):
234
+ effort = req["reasoning"].get("effort")
235
+ return {"n_messages": len(items), "last": tail, "stream": bool(req.get("stream")),
236
+ "reasoning_effort": effort if isinstance(effort, str) else None}
237
+
238
+
239
+ def _digest(obj) -> str:
240
+ text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True)
241
+ return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16]
242
+
243
+
244
+ def request_fields(req: dict) -> dict:
245
+ """Every top-level field of a request, for ``fidelity`` to compare two
246
+ ways of driving one agent - and no prompt: ``instructions``, each tool
247
+ and each ``input``/``messages`` item are kept as a digest and a length (an
248
+ item with its type and role, digested without its own ``id``), the rest as
249
+ sent, a large value digested too."""
250
+ out = {}
251
+ for k, v in req.items():
252
+ if k in ("input", "messages"):
253
+ items = [{"role": "user", "content": v}] if isinstance(v, str) else v if isinstance(v, list) else []
254
+ items = [{n: x for n, x in i.items() if n != "id"} if isinstance(i, dict) else i for i in items] # an item's own id
255
+ out[k] = [{"type": i.get("type"), "role": i.get("role"), "sha256": _digest(i), "chars": len(json.dumps(i))}
256
+ if isinstance(i, dict) else {"sha256": _digest(i)} for i in items]
257
+ elif k == "tools" and isinstance(v, list):
258
+ out[k] = [{"name": t.get("name") or (t.get("function") or {}).get("name") or t.get("type"), "sha256": _digest(t)}
259
+ if isinstance(t, dict) else {"sha256": _digest(t)} for t in v]
260
+ elif k == "instructions" or len(json.dumps(v)) > 600:
261
+ out[k] = {"sha256": _digest(v), "chars": len(v) if isinstance(v, str) else len(json.dumps(v))}
262
+ else:
263
+ out[k] = v
264
+ return out
265
+
266
+
267
+ def forwarded(headers) -> Dict[str, str]:
268
+ """The caller's request headers the relay passes on: all of them but the
269
+ hop's, the ones it sets itself, cookies and every credential."""
270
+ named = {n.strip().lower() for n in (headers.get("Connection") or "").split(",")} # hop headers by name
271
+ out = {}
272
+ for name, value in headers.items():
273
+ low = name.lower()
274
+ if low in _NOT_SENT or low in named or low.startswith("proxy-") or _CREDENTIAL.search(low):
275
+ continue
276
+ out[name] = value
277
+ return out
278
+
279
+
280
+ def returned(headers) -> List[tuple]:
281
+ """The provider's reply headers passed back to the caller."""
282
+ named = {n.strip().lower() for n in (headers.get("Connection") or "").split(",")}
283
+ return [(n, v) for n, v in headers.items() if n.lower() not in _NOT_RETURNED and n.lower() not in named]
284
+
285
+
286
+ class _Pool:
287
+ """Kept connections to the provider. A connection is taken for one call
288
+ and given back once its reply was read to the end, so no two calls ever
289
+ share one at a time; calls made one after another, from whichever
290
+ thread, reuse it. (The relay's server starts a thread for each of its
291
+ callers' connections, so a pool per thread would rarely be reused.)"""
292
+
293
+ KEEP = 16 # idle connections kept per provider; more are closed
294
+
295
+ def __init__(self, base: str):
296
+ u = urllib.parse.urlsplit(base)
297
+ self.https = u.scheme == "https"
298
+ self.host, self.port = u.hostname, u.port or (443 if self.https else 80)
299
+ self.prefix = u.path.rstrip("/")
300
+ self.idle: List[http.client.HTTPConnection] = []
301
+ self.lock = threading.Lock()
302
+ self.opened = 0 # connections made, for a test to count
303
+
304
+ def take(self, fresh: bool = False):
305
+ if not fresh:
306
+ with self.lock:
307
+ if self.idle:
308
+ return self.idle.pop(), True
309
+ kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection
310
+ with self.lock:
311
+ self.opened += 1
312
+ return kind(self.host, self.port, timeout=TIMEOUT), False
313
+
314
+ def give(self, conn, r):
315
+ """Back to the pool when the reply was read whole and the provider
316
+ keeps the connection open; else closed."""
317
+ if r.isclosed() and not r.will_close:
318
+ with self.lock:
319
+ if len(self.idle) < self.KEEP:
320
+ self.idle.append(conn)
321
+ return
322
+ conn.close()
323
+
324
+ def send(self, method: str, path: str, body: Optional[bytes], headers: Dict[str, str]):
325
+ """The connection and the provider's response, its headers read. A
326
+ kept connection found closed is replaced once - only before any of
327
+ the reply was read, so a call is never sent twice for an answer."""
328
+ conn, reused = self.take()
329
+ while True:
330
+ try:
331
+ if conn.sock is None:
332
+ try:
333
+ conn.connect() # the address, the connection, TLS: a failure here is before anything was sent
334
+ except Exception as e:
335
+ e.unsent = True
336
+ raise
337
+ conn.request(method, self.prefix + path, body=body, headers=headers)
338
+ return conn, conn.getresponse()
339
+ except _STALE:
340
+ conn.close()
341
+ if not reused:
342
+ raise
343
+ conn, reused = self.take(fresh=True)
344
+ except BaseException:
345
+ conn.close()
346
+ raise
347
+
348
+ def close(self):
349
+ with self.lock:
350
+ idle, self.idle = self.idle, []
351
+ for c in idle:
352
+ c.close()
353
+
354
+
355
+ class Relay:
356
+ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None,
357
+ model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False,
358
+ ledger: Optional[str] = None, unit_caps: bool = True):
359
+ """``base``: the provider's endpoint (``https://api.openai.com/v1``).
360
+ ``model``: when given, every call is sent as this model, whatever the
361
+ caller named - the run's model is the recipe's, not the agent's. A
362
+ route of its own may name another (``url(tag, model=...)``): a
363
+ benchmark's judge is asked as the judge, not as the run's model.
364
+ ``side_models``: the models an agent's own side jobs ask for (a
365
+ review, a summary) that it may call besides the run's model - its
366
+ definition's ``model.side_models``, or a setup's: the upstream's
367
+ code asking a model of its own besides the answerer. Each is a name,
368
+ sent as asked, or ``asked=sent``, sent as the provider names it; a
369
+ side call is recorded ``side``. Any other model is replaced by the
370
+ run's, recorded with ``model_asked``. ``key_header``: how the provider takes its key - a
371
+ bearer ``authorization``, or Anthropic's ``x-api-key``.
372
+ ``fields``: each record also holds the request's fields as sent
373
+ (:func:`request_fields`) - ``fidelity``'s, never a normal run's.
374
+ ``ledger``: the file of the run's spending cap (``ledger``), shared
375
+ by every relay of the job: a call is booked there before it is sent,
376
+ and not sent when it does not fit."""
377
+ self.fields = fields
378
+ self.ledger = ledgers.Ledger(ledger) if ledger else None
379
+ # False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them
380
+ self.unit_caps = unit_caps
381
+ self.stopping = threading.Event() # said by stop(): a call waiting for room gives up
382
+ self.base, self.key, self.dir = base.rstrip("/"), key, record_dir
383
+ self.prices, self.model = prices or {}, model
384
+ self.side_models = {a: (b or a) for a, _, b in (str(m).partition("=") for m in side_models or ())}
385
+ self.key_header = key_header.lower()
386
+ self.models: Dict[str, str] = {}
387
+ self.embedding_models: Dict[str, str] = {}
388
+ self.lock = threading.Lock()
389
+ self.pool = _Pool(self.base)
390
+ self.server: Optional[ThreadingHTTPServer] = None
391
+ os.makedirs(record_dir, exist_ok=True)
392
+
393
+ def url(self, tag: str, model: Optional[str] = None, *, embeddings: Optional[str] = None) -> str:
394
+ """A route's model pin, with an optional override only for /embeddings requests on this tag.
395
+ Reusing the URL without new pins preserves them; other tags keep their own routing."""
396
+ if not _TAG.match(tag):
397
+ raise ValueError(f"tag {tag!r}")
398
+ if model:
399
+ self.models[tag] = model
400
+ if embeddings:
401
+ self.embedding_models[tag] = embeddings
402
+ return f"{self.public or self._own()}/{tag}/v1"
403
+
404
+ def _own(self) -> str:
405
+ host, port = self.server.server_address[:2]
406
+ return f"http://{'127.0.0.1' if host in ('0.0.0.0', '') else host}:{port}"
407
+
408
+ def calls(self, tag: str) -> List[dict]:
409
+ try:
410
+ with open(os.path.join(self.dir, tag + ".jsonl")) as f:
411
+ return [json.loads(l) for l in f if l.strip()]
412
+ except OSError:
413
+ return []
414
+
415
+ def _cost(self, model: str, usage: Optional[dict]):
416
+ if not usage:
417
+ return None, None
418
+ if "cost" in usage:
419
+ return usage["cost"], "provider"
420
+ p = self.prices.get(model) # exactly this model: one whose name only starts the same is another
421
+ if not p:
422
+ return None, None
423
+ written = min(usage.get("cache_write") or 0, max(0, usage["input"] - usage["cached"]))
424
+ hour = min(usage.get("cache_write_1h") or 0, written)
425
+ fresh = max(0, usage["input"] - usage["cached"] - written)
426
+ write_rate = p.get("cache_write", p["input"])
427
+ return (fresh * p["input"] + usage["cached"] * p.get("cached", p["input"])
428
+ + (written - hour) * write_rate + hour * p.get("cache_write_1h", write_rate)
429
+ + usage["output"] * p["output"]) / 1e6, "estimate"
430
+
431
+ def _said(self, e: Exception) -> str:
432
+ """An exception as the agent and the record may see it: http.client
433
+ quotes a header value it refuses (a key with a stray CR, a BOM), so
434
+ the key itself is cut out of the text whatever else it says."""
435
+ text = f"{type(e).__name__}: {e}"
436
+ for k in {self.key, self.key.strip()} - {""}:
437
+ text = text.replace(k, "[REDACTED]")
438
+ return redact(text)[:200]
439
+
440
+ def _headers(self, caller) -> Dict[str, str]:
441
+ out = forwarded(caller)
442
+ has = {k.lower() for k in out} # a name in any case is the same header: never send it twice
443
+ for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \
444
+ (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"):
445
+ if name.lower() not in has:
446
+ out[name] = value
447
+ if self.key_header == "x-api-key":
448
+ out["x-api-key"] = self.key
449
+ else:
450
+ out["Authorization"] = "Bearer " + self.key
451
+ return out
452
+
453
+ def _forward(self, h: BaseHTTPRequestHandler):
454
+ """One call of a caller, ``h``: sent on, passed back, recorded."""
455
+ bits = h.path.split("/", 2)
456
+ if len(bits) < 3 or not _TAG.match(bits[1]):
457
+ h.send_error(404, "no route: /<tag>/v1/...")
458
+ return
459
+ tag, rest = bits[1], "/" + bits[2]
460
+ rest = rest[3:] if rest.startswith("/v1/") else rest
461
+ raw = h.rfile.read(int(h.headers.get("Content-Length") or 0))
462
+ try:
463
+ req = json.loads(raw) if raw else {}
464
+ except ValueError:
465
+ req = {}
466
+ asked = req.get("model") if isinstance(req, dict) else None
467
+ side = False
468
+ if isinstance(req, dict) and req:
469
+ if rest.split("?")[0] == "/embeddings" and self.embedding_models.get(tag):
470
+ req["model"] = self.embedding_models[tag]
471
+ elif self.models.get(tag):
472
+ req["model"] = self.models[tag]
473
+ elif self.model and asked != self.model and asked in self.side_models:
474
+ side = True
475
+ req["model"] = self.side_models[asked]
476
+ elif self.model:
477
+ req["model"] = self.model
478
+ if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"):
479
+ req.setdefault("stream_options", {})["include_usage"] = True
480
+ raw = json.dumps(req).encode()
481
+ booking = booked = None
482
+ # a call that names its model is booked, whatever its route, but a token count, which costs nothing and
483
+ # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book
484
+ if self.ledger is not None and h.command == "POST":
485
+ if isinstance(req, dict) and req.get("model") and not _free(rest):
486
+ booking, booked, why, by = self._book(req, raw, rest, tag, h)
487
+ else: # nothing to price: sent as it is, but not once the run's cap is reached
488
+ try:
489
+ why, by = (("the run's spending cap was reached: no further call of this run is sent", "cap")
490
+ if self.ledger.state()["closed"] else (None, None))
491
+ except ledgers.Unavailable as e: # as for a call it would book: not sent, and no cap reached
492
+ why, by = str(e), "ledger"
493
+ if by == "gone": # its caller hung up while it waited: nothing was booked, nothing is sent
494
+ return
495
+ if why:
496
+ self._refuse(h, tag, rest, req, asked, why, by)
497
+ return
498
+ t0 = time.time()
499
+ conn, r, back, body = None, None, [], b""
500
+ cut = False # the reply was cut off: the call may have been billed
501
+ try:
502
+ conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers))
503
+ status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers)
504
+ except Exception as e:
505
+ status, kind = 502, "application/json"
506
+ # the connection was never made (refused, unknown or unreachable address, a timeout or TLS failing as
507
+ # it opened): never sent. Anything else came after the request may have gone out: it may have been
508
+ # billed, so its booking is kept
509
+ cut = not getattr(e, "unsent", False)
510
+ body = json.dumps({"error": {"message": "the relay could not reach the endpoint: " + self._said(e)}}).encode()
511
+ streamed = r is not None and "text/event-stream" in kind
512
+ whole = False # the reply read to its end: its connection can be kept
513
+ if r is not None and not streamed:
514
+ try:
515
+ body, whole = r.read(), True
516
+ except Exception as e: # cut off mid-reply
517
+ status, kind, back, cut = 502, "application/json", [], True
518
+ body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode()
519
+
520
+ def record(body):
521
+ rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None,
522
+ "model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {},
523
+ "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value
524
+ if side:
525
+ rec["side"] = True
526
+ if self.fields and isinstance(req, dict) and req:
527
+ rec["fields"] = request_fields(req)
528
+ cost = None
529
+ if status == 200:
530
+ rec["reply"] = read_reply(rest, body, streamed)
531
+ cost, how = self._cost(rec["model"], rec["reply"].get("usage"))
532
+ if cost is not None:
533
+ rec["cost_usd"], rec["cost_from"] = cost, how
534
+ else:
535
+ rec["error"] = _said(status, body)
536
+ if booking is not None:
537
+ rec["booked_usd"] = booked
538
+ try:
539
+ if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole
540
+ self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output"))
541
+ else: # an error of the provider's, or never reached: not billed
542
+ self.ledger.release(booking)
543
+ except ledgers.Unavailable:
544
+ pass # the booking stays held, against the cap
545
+ with self.lock:
546
+ with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f:
547
+ f.write(json.dumps(rec) + "\n")
548
+
549
+ h.send_response(status)
550
+ if not any(n.lower() == "content-type" for n, _ in back):
551
+ back = back + [("Content-Type", kind)]
552
+ for n, v in back:
553
+ h.send_header(n, v)
554
+ if streamed:
555
+ if not r.getheader("Cache-Control"):
556
+ h.send_header("Cache-Control", "no-cache")
557
+ h.end_headers()
558
+ parts = []
559
+ try:
560
+ while True:
561
+ chunk = r.read1(65536)
562
+ if not chunk:
563
+ whole = True
564
+ break
565
+ parts.append(chunk)
566
+ try:
567
+ h.wfile.write(chunk)
568
+ h.wfile.flush()
569
+ except OSError:
570
+ pass # the caller went away; the record still completes
571
+ except Exception:
572
+ pass
573
+ record(b"".join(parts)) # before the connection closes: the caller's end of stream
574
+ else:
575
+ record(body) # before the caller has the reply, so a reader after it finds the call
576
+ if conn is not None: # read whole: back to the pool before the caller can make its next call
577
+ if whole:
578
+ self.pool.give(conn, r)
579
+ else:
580
+ conn.close()
581
+ conn = None
582
+ h.send_header("Content-Length", str(len(body)))
583
+ h.end_headers()
584
+ try:
585
+ h.wfile.write(body)
586
+ except OSError:
587
+ pass
588
+ if conn is not None:
589
+ if whole:
590
+ self.pool.give(conn, r)
591
+ else:
592
+ conn.close()
593
+
594
+ def _book(self, req: dict, raw: bytes, rest: str, tag: str = "", h: Optional[BaseHTTPRequestHandler] = None) -> tuple:
595
+ """(booking, its amount, None, None) for a call the cap lets out;
596
+ (None, its amount, why, by) for one it does not - ``by`` the run's
597
+ cap (``cap``), the cap of the unit ``tag`` routes for (``unit``),
598
+ ``price`` (a model with no price is not sent either, a capped run
599
+ sending no call it cannot price) or ``ledger`` (it could not be
600
+ read or written) - and those two are no cap reached. ``gone``: its
601
+ caller ``h`` hung up, or this relay stopped, while it waited."""
602
+ row = self.prices.get(req.get("model"))
603
+ if not row:
604
+ return None, None, f"no price for the model {req.get('model')!r}: a run with a spending cap sends no call it cannot price", "price"
605
+ try:
606
+ # a call only the calls in flight keep out waits for them: their bookings are estimates, and settle
607
+ deadline = time.time() + ledgers.WAIT
608
+ least = ledgers.bound(req, raw, row, rest, output=False)
609
+ while True:
610
+ booking, seen = self.ledger.reserve(lambda usual: ledgers.bound(req, raw, row, rest, usual),
611
+ ledgers.unit_of(tag) if self.unit_caps and tag else None, req.get("model"),
612
+ wait=time.time() < deadline, least=least)
613
+ if booking or seen["by"] != "wait":
614
+ break
615
+ if self.stopping.is_set() or _gone(h):
616
+ return None, None, None, "gone" # no one waits for it: not booked later, and the cap not closed in its name
617
+ time.sleep(0.2)
618
+ except ledgers.Unavailable as e: # not a cap reached: the unit is not the cap's, and the run goes on if the ledger reads again
619
+ return None, None, str(e), "ledger"
620
+ amount = seen["amount"]
621
+ return (booking, amount, None, None) if booking else (None, amount, ledgers.refusal(seen, amount), seen["by"])
622
+
623
+ def _refuse(self, h: BaseHTTPRequestHandler, tag: str, rest: str, req, asked, why: str, by: str = "cap") -> None:
624
+ """A call a capped run did not let out - at the run's cap (``by`` is
625
+ ``cap``), at its unit's own (``unit``), or for want of a price
626
+ (``price``): answered here and recorded as the relay's own
627
+ (``refused``), nothing sent."""
628
+ t = time.time()
629
+ rec = {"t0": t, "t1": t, "path": rest, "model": req.get("model") if isinstance(req, dict) else None, "model_asked": asked,
630
+ "status": 402, "refused": by, "error": why, "request": _request_tail(req) if isinstance(req, dict) else {},
631
+ "headers": sorted(k.lower() for k in forwarded(h.headers))}
632
+ with self.lock:
633
+ with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f:
634
+ f.write(json.dumps(rec) + "\n")
635
+ body = json.dumps({"type": "error", "error": {"type": "spending_cap", "code": {"unit": "unit_spending_cap", "price": "unpriced_model", "ledger": "ledger_unavailable"}.get(by, "spending_cap"),
636
+ "message": "nullius relay: " + why}}).encode()
637
+ h.send_response(402)
638
+ for n, v in (("Content-Type", "application/json"), ("X-Nullius-Refused", by), ("Content-Length", str(len(body)))):
639
+ h.send_header(n, v)
640
+ h.end_headers()
641
+ try:
642
+ h.wfile.write(body)
643
+ except OSError:
644
+ pass
645
+
646
+ # The address ``url`` hands out: the listening one unless the caller sets
647
+ # another - a sidecar listens on 0.0.0.0 and is reached as ``relay``.
648
+ public: Optional[str] = None
649
+
650
+ def start(self, host: str = "127.0.0.1", port: int = 0) -> "Relay":
651
+ relay = self
652
+
653
+ class Handler(BaseHTTPRequestHandler):
654
+ def log_message(self, *a):
655
+ pass
656
+
657
+ def do_GET(self):
658
+ self.forward()
659
+
660
+ def do_POST(self):
661
+ self.forward()
662
+
663
+ def forward(self):
664
+ relay._forward(self)
665
+
666
+ self.server = ThreadingHTTPServer((host, port), Handler)
667
+ self.server.daemon_threads = True
668
+ # stop() waits for the serving loop's next look: at the default half-second that idled a worker
669
+ # slot up to 0.5 s at the end of every unit (minutes over a 1319-unit GSM8K run)
670
+ threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True).start()
671
+ return self
672
+
673
+ def stop(self):
674
+ self.stopping.set()
675
+ if self.server:
676
+ self.server.shutdown()
677
+ self.server.server_close()
678
+ self.pool.close()
679
+
680
+
681
+ def _free(rest: str) -> bool:
682
+ """A route that names a model and is not billed: a token count."""
683
+ return rest.split("?")[0].rstrip("/").endswith("/count_tokens")
684
+
685
+
686
+ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool:
687
+ """Whether a caller hung up while its call waited: its end of the connection reads closed."""
688
+ if h is None:
689
+ return False
690
+ try:
691
+ return bool(select.select([h.connection], [], [], 0)[0]) and h.connection.recv(1, socket.MSG_PEEK) == b""
692
+ except (OSError, ValueError):
693
+ return True
694
+
695
+
696
+ def failure(calls: List[dict]) -> Optional[tuple]:
697
+ """When a unit's model calls ended failing, the exception a trial carries
698
+ for it - named so ``nullius.runs.classify`` sorts it as the
699
+ infrastructure it was - and the endpoint's words. None when the last call
700
+ succeeded (a failure the agent recovered from is not the unit's). A call
701
+ the run's spending cap refused is the unit's wherever it falls: once one
702
+ is refused, every later one is."""
703
+ cut = next((c for c in calls if c.get("refused") == "cap"), None)
704
+ if cut:
705
+ return ledgers.ERROR, cut.get("error") or ""
706
+ over = next((c for c in calls if c.get("refused") == "unit"), None)
707
+ if over: # its own cap: the unit's doing, and the run goes on
708
+ return ledgers.UNIT_ERROR, over.get("error") or ""
709
+ if not calls or calls[-1].get("status") == 200:
710
+ return None
711
+ last = calls[-1]
712
+ s, said = last.get("status"), last.get("error") or ""
713
+ if s == 429:
714
+ return "RateLimitError", said
715
+ if s in (401, 403):
716
+ return "AgentAuthenticationError", said
717
+ if s == 404 and "model" in said.lower():
718
+ return "ModelNotFoundError", said
719
+ if s == 502:
720
+ return "NetworkConnectionError", said
721
+ return "ApiError", said
722
+
723
+
724
+ # -- the relay as a process of its own: a container backend's sidecar ----------
725
+ def serve(argv=None) -> int:
726
+ """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL
727
+ --record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME]
728
+ [--public URL] [--prices JSON] [--ledger FILE]``.
729
+
730
+ ``--base`` is the provider's endpoint, ``--route`` a route sent as another
731
+ model (a benchmark's judge), ``--public`` the address ``url`` hands out.
732
+ The real key is read from ``--key-file`` and nowhere else: an argument or
733
+ an environment variable is visible to anyone who lists processes or
734
+ inspects a container. One JSON line, ``{"listening": "http://host:port"}``, says it
735
+ is ready; SIGTERM (or SIGINT) stops it."""
736
+ import argparse
737
+ import signal
738
+ ap = argparse.ArgumentParser(prog="nullius.standard.relay serve")
739
+ ap.add_argument("--listen", default="127.0.0.1:0", help="HOST:PORT")
740
+ ap.add_argument("--base", required=True, help="the provider's endpoint")
741
+ ap.add_argument("--record", required=True, help="the folder each route's calls are recorded in")
742
+ ap.add_argument("--key-file", required=True, help="the file that holds the provider's key")
743
+ ap.add_argument("--model", help="every call sent as this model")
744
+ ap.add_argument("--route", action="append", default=[], help="TAG=MODEL")
745
+ ap.add_argument("--side-model", action="append", default=[], help="a model the agent may call as asked")
746
+ ap.add_argument("--key-header", default="authorization", choices=("authorization", "x-api-key"))
747
+ ap.add_argument("--public", help="the address url() hands out, when not the listening one")
748
+ ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate")
749
+ ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent")
750
+ a = ap.parse_args(argv)
751
+ try:
752
+ with open(a.key_file) as f:
753
+ key = f.read().strip()
754
+ except OSError as e:
755
+ print(json.dumps({"error": f"cannot read --key-file: {e}"}), flush=True)
756
+ return 2
757
+ if not key:
758
+ print(json.dumps({"error": f"--key-file {a.key_file} contains no key"}), flush=True)
759
+ return 2
760
+ host, _, port = a.listen.rpartition(":")
761
+ r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model,
762
+ side_models=a.side_model, key_header=a.key_header, ledger=a.ledger)
763
+ for route in a.route:
764
+ tag, _, model = route.partition("=")
765
+ if not _TAG.match(tag) or not model:
766
+ print(json.dumps({"error": f"--route {route!r}: TAG=MODEL"}), flush=True)
767
+ return 2
768
+ r.models[tag] = model
769
+ r.public = a.public
770
+ stop = threading.Event()
771
+ signal.signal(signal.SIGTERM, lambda *_: stop.set())
772
+ signal.signal(signal.SIGINT, lambda *_: stop.set())
773
+ r.start(host or "127.0.0.1", int(port or 0))
774
+ print(json.dumps({"listening": r._own()}), flush=True)
775
+ while not stop.wait(1):
776
+ pass
777
+ r.stop()
778
+ return 0
779
+
780
+
781
+ if __name__ == "__main__":
782
+ import sys
783
+ if sys.argv[1:2] != ["serve"]:
784
+ sys.exit("python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M]")
785
+ sys.exit(serve(sys.argv[2:]))