nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,192 @@
1
+ """An own image's check's one way out: an HTTP(S) proxy to public addresses only.
2
+
3
+ A task's check may download what it runs (its test runner, a package) once
4
+ the agent has ended. Its container stays on the unit's internal network, which
5
+ has no route off it; this proxy, a container of its own on the default bridge
6
+ and on that network as ``egress`` (``container.Box.online``), is the check's
7
+ door, given to it as ``HTTP_PROXY`` and ``HTTPS_PROXY``. It resolves the
8
+ destination itself and connects only to a *global* address (``ipaddress``'s
9
+ ``is_global``): never a private range (RFC 1918), loopback, link-local
10
+ (``169.254.0.0/16``, where cloud metadata services answer), the shared range
11
+ ``100.64.0.0/10`` (where some clouds put theirs, ``100.100.100.200``), nor the
12
+ docker bridge and the machine behind it. It connects to the address it
13
+ checked, not to the name again, so a name that resolves differently the second
14
+ time gains nothing. ``CONNECT`` is passed through unread (TLS stays the
15
+ client's); a plain ``http://`` request is sent on with ``Connection: close``,
16
+ so one connection is one destination.
17
+
18
+ Why a proxy and not firewall rules: rules on docker's ``DOCKER-USER`` chain
19
+ need root on the machine, and a host without it would have to run the check
20
+ with no filter at all; a proxy needs nothing but docker, so every machine
21
+ runs the check the same way. What it costs is that a download must honour the
22
+ proxy variables - apt, curl, wget, git over HTTPS, pip, uv and npm do - and a
23
+ name cannot be resolved inside, which a client using the proxy never needs.
24
+
25
+ Run as a file on its own (``python3 egress.py --listen 0.0.0.0:3128``),
26
+ mounted alone into the base image: it imports nothing of this application. One
27
+ JSON line per request on its output (``allowed`` or ``refused``, host and
28
+ port), which the trial keeps (``verifier/egress.log``).
29
+ """
30
+ from __future__ import annotations
31
+
32
+ import argparse
33
+ import ipaddress
34
+ import json
35
+ import socket
36
+ import sys
37
+ import threading
38
+ from typing import List, Optional, Tuple
39
+ from urllib.parse import urlsplit
40
+
41
+ HEAD_LIMIT = 65536
42
+ HOP = ("connection", "proxy-connection", "keep-alive", "proxy-authorization", "te", "trailer", "upgrade")
43
+
44
+
45
+ def public(address: str) -> bool:
46
+ """Whether an address is one the check may reach: global, and not multicast."""
47
+ try:
48
+ ip = ipaddress.ip_address(address.split("%", 1)[0])
49
+ except ValueError:
50
+ return False
51
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped:
52
+ ip = ip.ipv4_mapped
53
+ return ip.is_global and not ip.is_multicast
54
+
55
+
56
+ def resolve(host: str, port: int) -> Tuple[Optional[tuple], List[str]]:
57
+ """(the first public address of ``host`` as a socket address, every
58
+ address it resolved to). None when it has no public one."""
59
+ try:
60
+ infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
61
+ except (OSError, UnicodeError):
62
+ return None, []
63
+ seen = [i[4][0] for i in infos]
64
+ for family, _, _, _, sockaddr in infos:
65
+ if public(sockaddr[0]):
66
+ return (family, sockaddr), seen
67
+ return None, seen
68
+
69
+
70
+ def _pump(src: socket.socket, dst: socket.socket) -> None:
71
+ try:
72
+ while True:
73
+ data = src.recv(65536)
74
+ if not data:
75
+ break
76
+ dst.sendall(data)
77
+ except OSError:
78
+ pass
79
+ finally:
80
+ for s in (dst, src):
81
+ try:
82
+ s.shutdown(socket.SHUT_RDWR)
83
+ except OSError:
84
+ pass
85
+
86
+
87
+ def _say(event: str, **kw) -> None:
88
+ print(json.dumps(dict(event=event, **kw)), flush=True)
89
+
90
+
91
+ def _reply(conn: socket.socket, status: str, why: str = "") -> None:
92
+ body = (why + "\n").encode()
93
+ try:
94
+ conn.sendall(f"HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {len(body)}\r\n"
95
+ f"Connection: close\r\n\r\n".encode() + body)
96
+ except OSError:
97
+ pass
98
+
99
+
100
+ def _read_head(conn: socket.socket) -> Tuple[bytes, bytes]:
101
+ """(the request's head, up to its blank line; what came after it)."""
102
+ buf = b""
103
+ while b"\r\n\r\n" not in buf:
104
+ data = conn.recv(65536)
105
+ if not data:
106
+ break
107
+ buf += data
108
+ if len(buf) > HEAD_LIMIT:
109
+ break
110
+ head, _, rest = buf.partition(b"\r\n\r\n")
111
+ return head, rest
112
+
113
+
114
+ def target(head: bytes) -> Tuple[str, str, int, bytes]:
115
+ """(method, host, port, the head as the origin is sent it) from a request
116
+ head; ValueError when it is not a proxy request."""
117
+ lines = head.decode("iso-8859-1").split("\r\n")
118
+ method, uri, version = lines[0].split(" ", 2)
119
+ if method.upper() == "CONNECT":
120
+ host, _, port = uri.rpartition(":")
121
+ return "CONNECT", host.strip("[]"), int(port), b""
122
+ u = urlsplit(uri)
123
+ if u.scheme != "http" or not u.hostname:
124
+ raise ValueError("an absolute http:// URL, or CONNECT host:port")
125
+ path = (u.path or "/") + ("?" + u.query if u.query else "")
126
+ kept = [l for l in lines[1:] if l and l.split(":", 1)[0].strip().lower() not in HOP]
127
+ out = "\r\n".join([f"{method} {path} {version}"] + kept + ["Connection: close", "", ""])
128
+ return method, u.hostname, u.port or 80, out.encode("iso-8859-1")
129
+
130
+
131
+ def _serve(conn: socket.socket) -> None:
132
+ try:
133
+ conn.settimeout(60)
134
+ head, rest = _read_head(conn)
135
+ try:
136
+ method, host, port, origin_head = target(head)
137
+ if not 0 < port < 65536:
138
+ raise ValueError("a port")
139
+ except (ValueError, IndexError):
140
+ _reply(conn, "400 Bad Request", "not a proxy request")
141
+ return
142
+ found, seen = resolve(host, port)
143
+ if found is None:
144
+ _say("refused", host=host, port=port, addresses=seen[:4])
145
+ _reply(conn, "403 Forbidden", f"{host} is not a public address")
146
+ return
147
+ family, sockaddr = found
148
+ try:
149
+ up = socket.socket(family, socket.SOCK_STREAM)
150
+ up.settimeout(30)
151
+ up.connect(sockaddr)
152
+ up.settimeout(None)
153
+ except OSError as e:
154
+ _say("unreachable", host=host, port=port, error=str(e)[:200])
155
+ _reply(conn, "502 Bad Gateway", f"{host}:{port} did not answer")
156
+ return
157
+ _say("allowed", host=host, port=port, method=method)
158
+ conn.settimeout(None)
159
+ if method == "CONNECT":
160
+ conn.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n")
161
+ if rest:
162
+ up.sendall(rest)
163
+ else:
164
+ up.sendall(origin_head + rest)
165
+ threading.Thread(target=_pump, args=(up, conn), daemon=True).start()
166
+ _pump(conn, up)
167
+ except OSError:
168
+ pass
169
+ finally:
170
+ try:
171
+ conn.close()
172
+ except OSError:
173
+ pass
174
+
175
+
176
+ def main(argv=None) -> int:
177
+ ap = argparse.ArgumentParser(prog="egress", description=__doc__.split("\n\n")[0])
178
+ ap.add_argument("--listen", required=True, help="address:port to listen on")
179
+ a = ap.parse_args(argv)
180
+ host, _, port = a.listen.rpartition(":")
181
+ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
182
+ srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
183
+ srv.bind((host, int(port)))
184
+ srv.listen(128)
185
+ _say("listening", port=int(port))
186
+ while True:
187
+ conn, _ = srv.accept()
188
+ threading.Thread(target=_serve, args=(conn,), daemon=True).start()
189
+
190
+
191
+ if __name__ == "__main__":
192
+ sys.exit(main())
@@ -0,0 +1,256 @@
1
+ """Fidelity: driving an agent over ACP sends the model what its own CLI sends.
2
+
3
+ An agent reached through an adapter is not obviously the agent as its users
4
+ run it: the adapter may choose a setting the CLI leaves to the provider, or
5
+ leave out a tool. This puts the two side by side. The agent's definition names
6
+ its own non-interactive command (``fidelity.command``, a template carrying
7
+ ``{prompt}``); the same units - the workspace kit's - are run both ways, each
8
+ unit at the same paths every time, its calls on a route of its own on a relay
9
+ that records each request's fields (``relay.request_fields``: everything but
10
+ the prompt, which is kept as digests). Each way runs twice, so a field that
11
+ differs between two runs of the same way is shown to be noise - an id, a time -
12
+ rather than assumed to be.
13
+
14
+ The comparison (:func:`compare`) is per field: the configuration fields over
15
+ every model request (``reasoning.effort``, ``tools[...]``, ``instructions``,
16
+ ``include``, ...), and the items of each unit's first request (``input[i]``:
17
+ the context the agent opens with). A few fields are left out by name
18
+ (``IGNORED``): the vendor's request metadata and cache keys, which label a
19
+ request and are not read by the model. The verdict is ``same`` when no other
20
+ field differs, ``differs`` otherwise, and it is written into the version's
21
+ ``proven`` as ``fidelity``, beside the kits' records.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import os
27
+ import shutil
28
+ import subprocess
29
+ import time
30
+ from typing import Callable, Dict, List, Optional
31
+
32
+ from nullius import prices as price_list
33
+ from nullius.standard import definitions, install, jsonl, relay, run
34
+
35
+ SIDES = ("acp", "cli")
36
+ # Fields that label a request for the vendor - ids, cache keys, analytics -
37
+ # and that the model does not read.
38
+ IGNORED = ("client_metadata", "metadata", "prompt_cache_key", "user", "safety_identifier", "previous_response_id",
39
+ "conversation")
40
+ MODEL_PATHS = ("/responses", "/chat/completions")
41
+ ABSENT = "—"
42
+ KIT_PORT = "agent.workspace/v1"
43
+
44
+
45
+ def _value(v) -> str:
46
+ if isinstance(v, dict) and set(v) <= {"sha256", "chars"} and "sha256" in v:
47
+ return f"sha256:{v['sha256']}" + (f" ({v['chars']} chars)" if "chars" in v else "")
48
+ return json.dumps(v, sort_keys=True)
49
+
50
+
51
+ def flatten(fields: dict) -> Dict[str, str]:
52
+ """A request's fields (as the relay recorded them) as one value per key,
53
+ without its ``input``/``messages``, which only a first request is read for."""
54
+ out = {}
55
+
56
+ def walk(prefix, v):
57
+ if isinstance(v, dict) and v and not ("sha256" in v and set(v) <= {"sha256", "chars"}):
58
+ for k, x in v.items():
59
+ walk(f"{prefix}.{k}", x)
60
+ else:
61
+ out[prefix] = _value(v)
62
+
63
+ for k, v in fields.items():
64
+ if k in IGNORED or k in ("input", "messages"):
65
+ continue
66
+ if k == "tools" and isinstance(v, list):
67
+ out["tools"] = json.dumps([t.get("name") for t in v])
68
+ for t in v:
69
+ out[f"tools[{t.get('name')}]"] = f"sha256:{t.get('sha256')}"
70
+ else:
71
+ walk(k, v)
72
+ return out
73
+
74
+
75
+ def opening(fields: dict) -> Dict[str, str]:
76
+ """A unit's first request: what the agent opened the conversation with,
77
+ item by item."""
78
+ items = fields.get("input") if "input" in fields else fields.get("messages")
79
+ items = items if isinstance(items, list) else []
80
+ out = {"input.items": str(len(items))}
81
+ for i, it in enumerate(items):
82
+ out[f"input[{i}]"] = f"{it.get('type') or 'message'}/{it.get('role') or '-'} sha256:{it.get('sha256')} ({it.get('chars')} chars)"
83
+ return out
84
+
85
+
86
+ def observe(routes: Dict[str, List[dict]]) -> Dict[str, Dict[str, set]]:
87
+ """One pass of one way (route -> the relay's records): every key and the
88
+ values it took, in two families - ``every`` over each model request, and
89
+ ``first`` over each unit's first request."""
90
+ every, first = [], []
91
+ for recs in routes.values():
92
+ asked = [r for r in recs if r.get("fields") and str(r.get("path", "")).endswith(MODEL_PATHS)]
93
+ every += [flatten(r["fields"]) for r in asked]
94
+ if asked:
95
+ first.append(opening(asked[0]["fields"]))
96
+ return {"every": every, "first": first}
97
+
98
+
99
+ def _values(obs: dict, fam: str, key: str) -> frozenset:
100
+ return frozenset(f.get(key, ABSENT) for f in obs[fam])
101
+
102
+
103
+ def compare(passes: Dict[str, List[Dict[str, List[dict]]]]) -> dict:
104
+ """``passes``: {"acp": [pass, ...], "cli": [pass, ...]}, a pass being
105
+ route -> the relay's records. A key is **noise** when one way's passes
106
+ disagree on it; of the rest, it is **equal** when both ways took the same
107
+ values, else **differing**. {verdict, equal, differing, noise, ignored,
108
+ requests}."""
109
+ obs = {s: [observe(p) for p in passes.get(s) or []] for s in SIDES}
110
+ keys = {fam: sorted({k for s in SIDES for o in obs[s] for f in o[fam] for k in f}) for fam in ("every", "first")}
111
+ equal, differing, noise = [], [], []
112
+ for fam in ("every", "first"):
113
+ for k in keys[fam]:
114
+ per = {s: [_values(o, fam, k) for o in obs[s]] for s in SIDES}
115
+ varies = [s for s in SIDES if len(set(per[s])) > 1]
116
+ if varies:
117
+ noise.append({"field": k, "varies": varies})
118
+ continue
119
+ got = {s: (per[s][0] if per[s] else frozenset()) for s in SIDES}
120
+ if got["acp"] == got["cli"]:
121
+ equal.append(k)
122
+ else:
123
+ differing.append({"field": k, "acp": sorted(got["acp"]), "cli": sorted(got["cli"])})
124
+ seen = {k for s in SIDES for p in passes.get(s) or [] for recs in p.values() for r in recs for k in (r.get("fields") or {})}
125
+ requests = {s: [len(o["every"]) for o in obs[s]] for s in SIDES}
126
+ if not all(requests[s] and all(requests[s]) for s in SIDES):
127
+ verdict = "no requests"
128
+ else:
129
+ verdict = "same" if not differing else "differs"
130
+ return {"verdict": verdict, "equal": equal, "differing": differing, "noise": noise,
131
+ "ignored": [k for k in IGNORED if k in seen], "requests": requests}
132
+
133
+
134
+ def table(result: dict) -> str:
135
+ """The comparison, short, for a shell."""
136
+ lines = [f"requests: acp {result['requests']['acp']}, cli {result['requests']['cli']} (per pass)",
137
+ f"equal: {len(result['equal'])} fields"]
138
+ for d in result["differing"]:
139
+ lines.append(f"DIFFERS {d['field']}:\n acp: {' | '.join(d['acp']) or ABSENT}\n cli: {' | '.join(d['cli']) or ABSENT}")
140
+ if result["noise"]:
141
+ lines.append("noise (varies between two runs of the same way): " + ", ".join(n["field"] for n in result["noise"]))
142
+ if result["ignored"]:
143
+ lines.append("left out by name (request metadata, cache keys): " + ", ".join(result["ignored"]))
144
+ lines.append(f"verdict: {result['verdict']}")
145
+ return "\n".join(lines)
146
+
147
+
148
+ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", passes: int = 2, write: bool = True,
149
+ say: Callable[[str], None] = print, env=None, unsandboxed: bool = False, options: Optional[dict] = None,
150
+ stamp: Optional[str] = None) -> dict:
151
+ """Run the workspace kit's units over ACP and through ``fidelity.command``,
152
+ ``passes`` times each (alternating), compare, and with ``write`` record the
153
+ result in the version's ``proven`` as ``fidelity``. Returns the comparison
154
+ with ``job``, the folder holding ``fidelity.json`` and each pass's records."""
155
+ if agent.get("_kind") != "agent" or "acp" not in (agent.get("start") or {}):
156
+ raise run.Refused(f"{agent.get('label')}: fidelity compares an agent driven over ACP with its own command")
157
+ cmd = (agent.get("fidelity") or {}).get("command")
158
+ if not cmd:
159
+ raise run.Refused(f"{definitions.title(agent)} names no command of its own (fidelity.command) to compare with")
160
+ kit = definitions.kit(KIT_PORT)
161
+ name = f"{stamp or time.strftime('%Y-%m-%d__%H-%M-%S')}__fidelity__{definitions.job_word(agent)}"
162
+ r = run.Run(kit, agent, model=model, provider=provider, jobs_root=jobs_root, name=name, concurrency=1,
163
+ say=say, env=env if env is not None else os.environ, proving=True, unsandboxed=unsandboxed, options=options)
164
+ os.makedirs(os.path.join(r.job_dir, ".work"), exist_ok=True)
165
+ say(f"fidelity {r.job_dir}")
166
+ try:
167
+ r.up = install.upstream(kit)
168
+ r.data = install.dataset(kit, r.ds, r.up)
169
+ r.inst = install.packages(agent)
170
+ except install.InstallFailed as e:
171
+ raise run.Refused(f"could not install what the run needs: {e}")
172
+ units = [str(u) for u in r.driver().request("units", dataset=r.ds["id"], timeout=300)["units"]]
173
+ # One relay for the whole comparison, on this machine: every route's
174
+ # records side by side, with the fields fidelity reads.
175
+ r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model,
176
+ side_models=(agent.get("model") or {}).get("side_models") or (),
177
+ key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start()
178
+ order = [(s, n) for n in range(1, passes + 1) for s in SIDES]
179
+ got: Dict[str, List[dict]] = {s: [] for s in SIDES}
180
+ solved: Dict[str, List[str]] = {}
181
+ try:
182
+ for side, n in order:
183
+ routes = {}
184
+ for i, unit in enumerate(units):
185
+ tag = f"{side}{n}-u{i:02d}"
186
+ ok = _one(r, side, unit, i, tag, cmd, os.path.join(r.job_dir, f"{side}{n}", run._slug(unit)))
187
+ routes[tag] = r.relay.calls(tag)
188
+ solved.setdefault(f"{side}{n}", []).append(f"{unit} {'passed' if ok else 'failed'}")
189
+ got[side].append(routes)
190
+ say(f" {side} pass {n}: " + ", ".join(solved[f'{side}{n}']))
191
+ finally:
192
+ for d in r.drivers:
193
+ d.close()
194
+ r.relay.stop()
195
+ result = dict(compare(got), job=r.job_dir, units=units, solved=solved, options=r.options, model=model, provider=provider,
196
+ version=str((agent.get("upstream") or {}).get("version") or ""))
197
+ with open(os.path.join(r.job_dir, "fidelity.json"), "w") as f:
198
+ json.dump(result, f, indent=2)
199
+ say(table(result))
200
+ if write:
201
+ rec = {"verdict": result["verdict"], "on": time.strftime("%Y-%m-%d"), "model": model, "provider": provider,
202
+ "version": result["version"], "job": os.path.basename(r.job_dir), "passes": passes,
203
+ "differs": result["differing"]}
204
+ changed = {k: v for k, v in r.options.items() if v != ((agent.get("options") or {}).get(k) or {}).get("default")}
205
+ if changed:
206
+ rec["options"] = changed
207
+ definitions.save(agent, lambda cur: cur.update(proven=dict(cur.get("proven") or {}, fidelity=rec)))
208
+ return result
209
+
210
+
211
+ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], keep: str) -> bool:
212
+ """One unit one way, in the unit's own fixed workspace and home - the same
213
+ paths every pass, so a path in the context is no difference - and its
214
+ agent folder kept under ``keep``. True when the kit scored it a pass."""
215
+ base = os.path.join(r.job_dir, "unit", f"u{i:02d}")
216
+ ws, home = os.path.join(base, "workspace"), os.path.join(base, ".home")
217
+ shutil.rmtree(base, ignore_errors=True)
218
+ os.makedirs(ws)
219
+ os.makedirs(home)
220
+ subprocess.run(["git", "init", "-q", ws], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
221
+ setup = r.driver().request("setup", unit=unit, dir=ws, timeout=600)
222
+ transcript: List[dict] = []
223
+ answer = ""
224
+ try:
225
+ if side == "acp":
226
+ r.relay.unit_url = r.relay.url(tag)
227
+ answer = r.drive_agent(unit, ws, home, tag, setup, transcript, r.relay)[0]
228
+ else:
229
+ answer = _cli(r, cmd, ws, home, tag, setup["prompt"], os.path.join(base, "agent"))
230
+ except (run._Ended, jsonl.Failed) as e:
231
+ r.say(f" {tag} {unit}: {e}")
232
+ try:
233
+ score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600)
234
+ except jsonl.Failed:
235
+ score = {}
236
+ if os.path.isdir(os.path.join(base, "agent")):
237
+ os.makedirs(os.path.dirname(keep), exist_ok=True)
238
+ shutil.move(os.path.join(base, "agent"), keep)
239
+ return score.get("reward") == 1.0
240
+
241
+
242
+ def _cli(r: "run.Run", cmd: List[str], ws: str, home: str, tag: str, prompt: str, agent_dir: str) -> str:
243
+ a = r.other
244
+ env = run.agent_env(a, r.inst, home, r.relay.url(tag), r.model, r.options)
245
+ words = install.words(cmd, {"here": a["_here"], "install": r.inst, "prompt": prompt})
246
+ words[0] = shutil.which(words[0], path=env["PATH"]) or words[0]
247
+ os.makedirs(agent_dir, exist_ok=True)
248
+ with open(os.path.join(agent_dir, "stdout.txt"), "w") as out, open(os.path.join(agent_dir, "agent.log"), "w") as err:
249
+ try:
250
+ p = run.run_group(words, r.turn_seconds, cwd=ws, env=env, stdout=out, stderr=err, text=True)
251
+ except subprocess.TimeoutExpired:
252
+ raise run._Ended("AgentTimeoutError", f"its command did not end in {r.turn_seconds} s")
253
+ if p.returncode != 0:
254
+ raise run._Ended("NonZeroAgentExitCodeError", f"its command: exit {p.returncode}")
255
+ with open(os.path.join(agent_dir, "stdout.txt")) as f:
256
+ return f.read().strip()
@@ -0,0 +1,139 @@
1
+ """A one-address door: a TCP forwarder to a single ``host:port``, or to a
2
+ single Unix socket (``unix:/path``).
3
+
4
+ On the container backend the agent's container sits on an internal network with
5
+ no route out; its relay is its way to the model. A licence of kind
6
+ ``server`` needs one more address: this forwarder, a container of its own
7
+ beside the relay (``container.Unit``), on the default bridge and on the
8
+ unit's network under the server's own name, listening on the server's port
9
+ and passing every connection to that one address - and to nothing else,
10
+ whatever the agent sends. It reads nothing of what it passes. It resolves the
11
+ server's name once, before it listens: on the unit's network, that name is
12
+ the forwarder's own. A name that does not resolve ends it with an
13
+ ``unresolved`` event.
14
+
15
+ A benchmark's MCP server run on the machine (``mcp_machine``) is reached the
16
+ same way: a forwarder on the unit's internal network only, the server's
17
+ socket mounted into it alone, ``--to unix:<socket>``.
18
+
19
+ On macOS, Docker Desktop cannot reach that socket through a mount: the
20
+ forwarder, on the default bridge too, goes to a loopback TCP bridge on the
21
+ machine (``container.Unit._door``), which serves a connection only after its
22
+ nonce. ``--nonce-file`` holds the nonce, mounted read-only and read once,
23
+ before it listens: each connection's first line.
24
+
25
+ Run as a file on its own (``python3 forward.py --listen 0.0.0.0:27000``,
26
+ the server in ``NULLIUS_FORWARD_TO``), mounted alone into the base image: it imports
27
+ nothing of this application, so its container is given nothing of it.
28
+ """
29
+ from __future__ import annotations
30
+
31
+ import argparse
32
+ import json
33
+ import os
34
+ import socket
35
+ import sys
36
+ import threading
37
+
38
+
39
+ def _pump(src: socket.socket, dst: socket.socket) -> None:
40
+ try:
41
+ while True:
42
+ data = src.recv(65536)
43
+ if not data:
44
+ break
45
+ dst.sendall(data)
46
+ except OSError:
47
+ pass
48
+ finally:
49
+ for s in (dst, src):
50
+ try:
51
+ s.shutdown(socket.SHUT_RDWR)
52
+ except OSError:
53
+ pass
54
+ s.close()
55
+
56
+
57
+ def _connect(targets) -> socket.socket:
58
+ """The first of ``targets`` (family, address) that accepts, or the last error."""
59
+ error = OSError("no address to connect to")
60
+ for family, addr in targets:
61
+ up = None
62
+ try:
63
+ up = socket.socket(family, socket.SOCK_STREAM) # in the try: a family this kernel lacks is skipped
64
+ up.settimeout(30)
65
+ up.connect(addr)
66
+ return up
67
+ except OSError as e:
68
+ error = e
69
+ if up is not None:
70
+ up.close()
71
+ raise error
72
+
73
+
74
+ def _serve(conn: socket.socket, targets, nonce: str = "") -> None:
75
+ up = None
76
+ try:
77
+ up = _connect(targets)
78
+ if nonce:
79
+ up.sendall((nonce + "\n").encode())
80
+ up.settimeout(None)
81
+ except OSError as e:
82
+ if up is not None:
83
+ up.close()
84
+ print(json.dumps({"event": "unreachable", "error": str(e)[:200]}), flush=True)
85
+ conn.close()
86
+ return
87
+ threading.Thread(target=_pump, args=(up, conn), daemon=True).start()
88
+ _pump(conn, up)
89
+
90
+
91
+ def _split(addr: str):
92
+ if addr.startswith("unix:"):
93
+ return addr[len("unix:"):], None
94
+ host, _, port = addr.rpartition(":")
95
+ return host, int(port)
96
+
97
+
98
+ def main(argv=None) -> int:
99
+ ap = argparse.ArgumentParser(prog="forward", description=__doc__.split("\n\n")[0])
100
+ ap.add_argument("--listen", required=True, help="address:port to listen on")
101
+ ap.add_argument("--to", default=os.environ.get("NULLIUS_FORWARD_TO", ""),
102
+ help="the one host:port, or unix:/path, every connection goes to (default: NULLIUS_FORWARD_TO)")
103
+ ap.add_argument("--nonce-file", help="a file whose text is sent as the first line of every connection")
104
+ a = ap.parse_args(argv)
105
+ if not a.to:
106
+ ap.error("--to, or NULLIUS_FORWARD_TO: where to forward")
107
+ nonce = ""
108
+ if a.nonce_file:
109
+ try:
110
+ with open(a.nonce_file) as f:
111
+ nonce = f.read().strip()
112
+ except OSError as e:
113
+ print(json.dumps({"event": "no-nonce", "error": str(e)[:200]}), flush=True)
114
+ return 1
115
+ if not nonce:
116
+ print(json.dumps({"event": "no-nonce", "error": "the nonce file is empty"}), flush=True)
117
+ return 1
118
+ lhost, lport = _split(a.listen)
119
+ host, port = _split(a.to)
120
+ if port is None:
121
+ targets = [(socket.AF_UNIX, host)]
122
+ else:
123
+ try:
124
+ targets = [(f, addr) for f, _, _, _, addr in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)]
125
+ except OSError as e:
126
+ print(json.dumps({"event": "unresolved", "host": host, "error": str(e)[:200]}), flush=True)
127
+ return 1
128
+ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
129
+ srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
130
+ srv.bind((lhost, lport))
131
+ srv.listen(64)
132
+ print(json.dumps({"event": "listening", "port": lport}), flush=True)
133
+ while True:
134
+ conn, _ = srv.accept()
135
+ threading.Thread(target=_serve, args=(conn, targets, nonce), daemon=True).start()
136
+
137
+
138
+ if __name__ == "__main__":
139
+ sys.exit(main())