nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,62 @@
1
+ """A stdio MCP server that is a benchmark's, run on the machine: this joins
2
+ its stdin and stdout to one TCP address, the unit's door to that server
3
+ (``NULLIUS_MCP_AT``, ``host:port``), and passes the bytes both ways
4
+ unchanged, reading none of them.
5
+
6
+ Mounted alone into the agent's container on the container backend and started
7
+ by the agent as any stdio MCP server (``mcp_machine``); it imports nothing
8
+ of this application, only the standard library, so the container needs
9
+ nothing but ``python3``.
10
+ """
11
+ import os
12
+ import socket
13
+ import sys
14
+ import threading
15
+
16
+
17
+ def main() -> int:
18
+ at = os.environ.get("NULLIUS_MCP_AT", "")
19
+ host, _, port = at.rpartition(":")
20
+ if not host or not port.isdigit():
21
+ sys.stderr.write("mcp_bridge: NULLIUS_MCP_AT, host:port of the server's door\n")
22
+ return 2
23
+ try:
24
+ s = socket.create_connection((host, int(port)), timeout=30)
25
+ except OSError as e:
26
+ sys.stderr.write(f"mcp_bridge: the server's door {at}: {e}\n")
27
+ return 1
28
+ s.settimeout(None)
29
+ out = sys.stdout.buffer
30
+
31
+ def down(): # the server's replies, to the agent
32
+ try:
33
+ while True:
34
+ data = s.recv(65536)
35
+ if not data:
36
+ break
37
+ out.write(data)
38
+ out.flush()
39
+ except OSError:
40
+ pass
41
+ os._exit(0)
42
+
43
+ threading.Thread(target=down, daemon=True).start()
44
+ src = sys.stdin.buffer
45
+ try:
46
+ while True:
47
+ data = src.read1(65536) if hasattr(src, "read1") else src.read(1)
48
+ if not data:
49
+ break
50
+ s.sendall(data)
51
+ except OSError:
52
+ pass
53
+ try:
54
+ s.shutdown(socket.SHUT_WR) # the agent is done: the server sees its stdin end
55
+ except OSError:
56
+ pass
57
+ threading.Event().wait(5) # what the server still says, then go
58
+ return 0
59
+
60
+
61
+ if __name__ == "__main__":
62
+ sys.exit(main())
@@ -0,0 +1,193 @@
1
+ """A benchmark's MCP server run on the machine, in the driver's environment.
2
+
3
+ A benchmark whose tools are its upstream's own code - with the upstream's
4
+ dependencies - cannot have that server started inside the agent's
5
+ container: the container carries the agent and nothing of the benchmark's.
6
+ So a benchmark may declare ``mcp_servers`` (``definitions._mcp_servers``),
7
+ each a command of its own (``{here}``, ``{install}``: the driver's
8
+ environment, ``driver.install``), and a unit's ``setup`` ask for one in its
9
+ ``mcp`` entry by name:
10
+
11
+ {"name": "airline", "server": "domain", "args": [...], "env": {...}}
12
+
13
+ The agent's session is still given a stdio MCP server - every agent that
14
+ proved ``agent.tools/v1`` takes one - and that server is the benchmark's:
15
+
16
+ - **local backend**: the entry is the server's own command, rendered
17
+ (:func:`local_entry`): the agent starts it, on this machine, from the
18
+ driver's environment, as it starts any stdio server;
19
+ - **container backend**: the run starts a :class:`Listener` per server per unit
20
+ on this machine - a Unix socket in a private folder; each connection to it
21
+ is one start of the server, its stdin and stdout the connection - and the
22
+ unit gets one door to it (``container.Unit``'s ``doors``: ``forward.py``
23
+ on the unit's internal network only, the socket alone mounted into it,
24
+ passing every connection to that socket and nowhere else). The agent's
25
+ entry is ``mcp_bridge.py``, a file of the standard library mounted alone
26
+ into its container, which joins its stdin and stdout to the door
27
+ (:func:`bridge_entry`). Nothing of the server, its environment or its
28
+ data enters the agent's container, and the door reaches nothing else.
29
+
30
+ Either way a server started twice in a unit is two processes, as a stdio
31
+ server is; what it logs, it logs on this machine (the trial's
32
+ ``verifier/``), out of the agent's reach in a container.
33
+ """
34
+ from __future__ import annotations
35
+
36
+ import os
37
+ import re
38
+ import shutil
39
+ import socket
40
+ import subprocess
41
+ import tempfile
42
+ import threading
43
+ from typing import List, Optional, Tuple
44
+
45
+ from nullius.standard import install
46
+
47
+ BRIDGE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp_bridge.py")
48
+ DOOR_PORT = 9000 # where a door listens, on the unit's network
49
+ _ALIAS = re.compile(r"[^a-z0-9-]+")
50
+
51
+
52
+ class Unknown(ValueError):
53
+ """A unit's setup named a server the benchmark does not declare."""
54
+
55
+
56
+ def wanted(setup: dict) -> List[dict]:
57
+ """The setup's ``mcp`` entries that name a server of the benchmark's."""
58
+ return [s for s in setup.get("mcp") or [] if isinstance(s, dict) and s.get("server") is not None]
59
+
60
+
61
+ def argv(bench: dict, entry: dict, inst: Optional[str]) -> List[str]:
62
+ """The server's command, its slots filled, and the entry's ``args``."""
63
+ servers = bench.get("mcp_servers") or {}
64
+ sv = servers.get(entry.get("server"))
65
+ if not isinstance(sv, dict):
66
+ raise Unknown(f"mcp: {entry.get('server')!r} is not one of the benchmark's mcp_servers "
67
+ f"({', '.join(servers) or 'none declared'})")
68
+ args = entry.get("args") or []
69
+ if not isinstance(args, list) or not all(isinstance(a, str) for a in args):
70
+ raise Unknown(f"mcp {entry.get('name')}: args, a list of words")
71
+ return install.words(sv["command"], {"here": bench["_here"], "install": inst or ""}) + list(args)
72
+
73
+
74
+ def server_env(base: dict, entry: dict) -> dict:
75
+ """The server's environment: the driver's (``base``: no secret, the
76
+ upstream and dataset named) and the entry's own ``env``."""
77
+ own = entry.get("env") or {}
78
+ if not isinstance(own, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in own.items()):
79
+ raise Unknown(f"mcp {entry.get('name')}: env, names to strings")
80
+ return dict(base, **own)
81
+
82
+
83
+ def local_entry(bench: dict, entry: dict, inst: Optional[str], base: dict) -> dict:
84
+ """The agent's stdio entry on the local backend: the server's own command."""
85
+ cmd = argv(bench, entry, inst)
86
+ return {"name": str(entry.get("name") or entry["server"]), "command": cmd[0], "args": cmd[1:],
87
+ "env": server_env(base, entry)}
88
+
89
+
90
+ def alias(name: str, i: int) -> str:
91
+ """The name a door answers by on the unit's network."""
92
+ return f"mcp-{i}-" + (_ALIAS.sub("-", name.lower()).strip("-")[:30] or "server")
93
+
94
+
95
+ def bridge_entry(name: str, door: str) -> dict:
96
+ """The agent's stdio entry in a container: the bridge, told its door."""
97
+ return {"name": name, "command": "python3", "args": [BRIDGE], "env": {"NULLIUS_MCP_AT": door}}
98
+
99
+
100
+ class Listener:
101
+ """A server, started once per connection to a Unix socket: the
102
+ connection is its stdin and stdout, its stderr the log. The socket sits
103
+ in a folder of its own (0700), short enough for a socket's path."""
104
+
105
+ def __init__(self, cmd: List[str], env: dict, cwd: str, log: str):
106
+ self.cmd, self.env, self.cwd, self.log = cmd, env, cwd, log
107
+ self.dir = tempfile.mkdtemp(prefix="nullius-mcp-")
108
+ self.path = os.path.join(self.dir, "s")
109
+ self.procs: List[subprocess.Popen] = []
110
+ self.sock: Optional[socket.socket] = None
111
+ self.lock = threading.Lock()
112
+ self.closed = False
113
+
114
+ def start(self) -> "Listener":
115
+ os.chmod(self.dir, 0o700)
116
+ self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
117
+ self.sock.bind(self.path)
118
+ self.sock.listen(8)
119
+ threading.Thread(target=self._accept, daemon=True).start()
120
+ return self
121
+
122
+ def _accept(self):
123
+ while True:
124
+ try:
125
+ conn, _ = self.sock.accept()
126
+ except OSError:
127
+ return
128
+ try:
129
+ with self.lock:
130
+ if self.closed:
131
+ conn.close()
132
+ return
133
+ with open(self.log, "a") as err:
134
+ self.procs.append(subprocess.Popen(self.cmd, cwd=self.cwd, env=self.env, stdin=conn.fileno(),
135
+ stdout=conn.fileno(), stderr=err, start_new_session=True))
136
+ except OSError as e:
137
+ with open(self.log, "a") as err:
138
+ err.write(f"[nullius] could not start the server: {e}\n")
139
+ finally:
140
+ conn.close() # the server holds its own copy
141
+
142
+ def stop(self):
143
+ with self.lock:
144
+ self.closed = True
145
+ if self.sock is not None:
146
+ try:
147
+ self.sock.close()
148
+ except OSError:
149
+ pass
150
+ for p in self.procs:
151
+ if p.poll() is None:
152
+ p.terminate()
153
+ for p in self.procs:
154
+ try:
155
+ p.wait(timeout=5)
156
+ except subprocess.TimeoutExpired:
157
+ p.kill()
158
+ shutil.rmtree(self.dir, ignore_errors=True)
159
+
160
+
161
+ def doors(bench: dict, setup: dict, inst: Optional[str], base: dict, cwd: str, log_dir: str
162
+ ) -> Tuple[List[dict], List[Tuple[str, int, str]], List[Listener]]:
163
+ """For the container backend: the setup's ``mcp`` with each machine server's
164
+ entry replaced by the bridge's, the doors the unit opens (alias, port,
165
+ socket) and the listeners started - stopped by the caller, also on an
166
+ error. Raises :class:`Unknown` for a server not declared."""
167
+ out, opened, started = [], [], []
168
+ os.makedirs(log_dir, exist_ok=True)
169
+ try:
170
+ for i, s in enumerate(setup.get("mcp") or [], 1):
171
+ if not (isinstance(s, dict) and s.get("server") is not None):
172
+ out.append(s)
173
+ continue
174
+ cmd, env = argv(bench, s, inst), server_env(base, s)
175
+ name = str(s.get("name") or s["server"])
176
+ ln = Listener(cmd, env, cwd, os.path.join(log_dir, f"mcp-{i}.log")).start()
177
+ started.append(ln)
178
+ a = alias(name, i)
179
+ opened.append((a, DOOR_PORT, ln.path))
180
+ out.append(bridge_entry(name, f"{a}:{DOOR_PORT}"))
181
+ except Exception:
182
+ for ln in started:
183
+ ln.stop()
184
+ raise
185
+ return out, opened, started
186
+
187
+
188
+ def entries(bench: dict, setup: dict, inst: Optional[str], base: dict) -> List[dict]:
189
+ """For the local backend: the setup's ``mcp``, each machine server's entry
190
+ its own command. Raises :class:`Unknown` for a server not declared."""
191
+ return [local_entry(bench, s, inst, base) if isinstance(s, dict) and s.get("server") is not None else s
192
+ for s in setup.get("mcp") or []]
193
+
@@ -0,0 +1,239 @@
1
+ """Shard jobs of one launch put back together as one job.
2
+
3
+ A launch run over several machines (``shard``) leaves one job per shard. The
4
+ pages would read each as a run of its own, so ``cli merge`` makes the one
5
+ job a reader expects: every shard's trials in it, under the names they had
6
+ (a trial's tag is its unit's place in the whole dataset, so shards never
7
+ share one), its ``result.json`` recomputed from them - the trials, the
8
+ verdicts, the earliest start and the latest end - and its ``config.json``
9
+ the shards' with ``shard`` taken out and ``merged_from`` put in.
10
+
11
+ What is merged must be one launch: the same launch id, benchmark and
12
+ version, dataset, agent or capability and model, and the same number of shards
13
+ - else it is refused, naming what differs, unless ``force``. Two trials of
14
+ one unit are always refused, with the units named: one of them would be
15
+ counted twice. Units no shard ran are not a refusal - a shard may be
16
+ missing, or stopped early - and are said: the shards absent, and the units a
17
+ shard planned and has no trial for.
18
+
19
+ Files are hard-linked into the merged job where the file system allows (a
20
+ copy where it does not), so a merge of a thousand trials costs no second
21
+ copy of their workspaces; each trial's ``result.json`` is a copy of its own,
22
+ rewritten to name the merged job. The shards are left as they were.
23
+ """
24
+ from __future__ import annotations
25
+
26
+ import json
27
+ import os
28
+ import shutil
29
+ from typing import Dict, List, Optional
30
+
31
+ from nullius.standard import job
32
+ from nullius.standard.names import backend as backend_name
33
+
34
+
35
+ # What every shard of one launch shares, as (what a refusal calls it, where it is in config.json).
36
+ SAME = (("launch", ("standard", "launch")), ("benchmark", ("standard", "benchmark")),
37
+ ("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")),
38
+ ("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")),
39
+ ("backend", ("standard", "backend")), ("number of shards", ("standard", "shard", "of")))
40
+
41
+
42
+ class Refused(Exception):
43
+ """The jobs cannot be merged as asked; the message says why."""
44
+
45
+
46
+ def _read(path: str) -> dict:
47
+ try:
48
+ with open(path, encoding="utf-8") as f:
49
+ got = json.load(f)
50
+ return got if isinstance(got, dict) else {}
51
+ except (OSError, ValueError):
52
+ return {}
53
+
54
+
55
+ def _at(d, path):
56
+ for k in path:
57
+ if isinstance(d, dict):
58
+ d = d.get(k)
59
+ elif isinstance(d, list) and isinstance(k, int) and k < len(d):
60
+ d = d[k]
61
+ else:
62
+ return None
63
+ return d
64
+
65
+
66
+ def _trials(job_dir: str) -> List[str]:
67
+ return sorted(n for n in os.listdir(job_dir)
68
+ if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json")))
69
+
70
+
71
+ def _link_tree(src: str, dest: str) -> None:
72
+ """``src`` into ``dest``, each file hard-linked (copied across devices)."""
73
+ for d, dirs, files in os.walk(src):
74
+ rel = os.path.relpath(d, src)
75
+ os.makedirs(os.path.join(dest, rel), exist_ok=True)
76
+ for f in files:
77
+ a, b = os.path.join(d, f), os.path.join(dest, rel, f)
78
+ if os.path.islink(a):
79
+ os.symlink(os.readlink(a), b)
80
+ continue
81
+ try:
82
+ os.link(a, b)
83
+ except OSError:
84
+ shutil.copy2(a, b)
85
+
86
+
87
+ def check(job_dirs: List[str], force: bool = False) -> dict:
88
+ """What merging ``job_dirs`` would make, without writing: the shards read,
89
+ the trials by unit, the notes; ``Refused`` when they cannot be merged."""
90
+ if not job_dirs:
91
+ raise Refused("no jobs to merge")
92
+ shards = []
93
+ for d in job_dirs:
94
+ d = os.path.abspath(d.rstrip("/"))
95
+ cfg, res = _read(os.path.join(d, "config.json")), _read(os.path.join(d, "result.json"))
96
+ if isinstance(cfg.get("standard"), dict) and "backend" in cfg["standard"]:
97
+ cfg["standard"]["backend"] = backend_name(cfg["standard"]["backend"])
98
+ if cfg.get("runner") != "standard":
99
+ raise Refused(f"{d}: not a job of the Nullius standard (no config.json naming it)")
100
+ sh = _at(cfg, ("standard", "shard"))
101
+ if not isinstance(sh, dict) and not force:
102
+ raise Refused(f"{d}: not a shard (its config has no standard.shard); --force merges it anyway")
103
+ shards.append({"dir": d, "name": os.path.basename(d), "config": cfg, "result": res, "shard": sh or {}})
104
+ if len({s["dir"] for s in shards}) != len(shards):
105
+ raise Refused("a job is named twice")
106
+ differ = []
107
+ for label, path in SAME:
108
+ seen = {json.dumps(_at(s["config"], path)) for s in shards}
109
+ if len(seen) > 1:
110
+ differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards))
111
+ if differ and not force:
112
+ raise Refused("the shards are not of one launch - " + "; ".join(differ) + " (--force merges them anyway)")
113
+ # every trial by its unit: two of one unit are never merged, force or not
114
+ by_unit: Dict[str, List[tuple]] = {}
115
+ names: Dict[str, List[str]] = {}
116
+ for s in shards:
117
+ s["trials"] = _trials(s["dir"])
118
+ for t in s["trials"]:
119
+ unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t
120
+ by_unit.setdefault(str(unit), []).append((s["name"], t))
121
+ names.setdefault(t, []).append(s["name"])
122
+ twice = {u: w for u, w in by_unit.items() if len(w) > 1}
123
+ if twice:
124
+ shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10])
125
+ raise Refused(f"{len(twice)} unit(s) have a trial in more than one shard: {shown}"
126
+ + (" and more" if len(twice) > 10 else ""))
127
+ clash = {t: w for t, w in names.items() if len(w) > 1}
128
+ if clash:
129
+ raise Refused("trials of the same name in several shards: " + ", ".join(sorted(clash)[:10]))
130
+ notes, missing = [], []
131
+ of = {s["shard"].get("of") for s in shards if s["shard"]}
132
+ totals = {s["shard"].get("units_total") for s in shards if s["shard"]}
133
+ if len(of) == 1:
134
+ n = of.pop()
135
+ have = sorted(s["shard"].get("index") for s in shards if s["shard"])
136
+ absent = [i for i in range(1, (n or 0) + 1) if i not in have]
137
+ if absent:
138
+ notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge")
139
+ dup = sorted({i for i in have if have.count(i) > 1})
140
+ if dup:
141
+ notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)")
142
+ for s in shards:
143
+ planned = s["shard"].get("planned")
144
+ if isinstance(planned, list):
145
+ ran = {u for u, w in by_unit.items() if any(j == s["name"] for j, _ in w)}
146
+ left = [u for u in planned if u not in ran]
147
+ if left:
148
+ missing += left
149
+ notes.append(f"{s['name']}: {len(left)} planned unit(s) with no trial: {', '.join(left[:5])}" + (" ..." if len(left) > 5 else ""))
150
+ if s["shard"].get("limit"):
151
+ notes.append(f"{s['name']}: limited to its first {s['shard']['limit']} units")
152
+ total = totals.pop() if len(totals) == 1 else None
153
+ if isinstance(total, int):
154
+ short = total - len(by_unit)
155
+ if short > 0:
156
+ notes.append(f"{short} of the dataset's {total} units are not in the merge")
157
+ return {"shards": shards, "by_unit": by_unit, "notes": notes, "missing": missing, "units_total": total, "differ": differ}
158
+
159
+
160
+ def _budget(budgets: list) -> Optional[dict]:
161
+ """The shards' spending caps as one (``run.Run.budget``): the caps and what was spent added up, stopped
162
+ when any shard was, their units cut off and not run together; a cap per unit, the same on every shard, is
163
+ kept with the units that reached it. None unless every shard had a cap: a cap over part of a launch is
164
+ not the launch's."""
165
+ number = lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)
166
+ if not budgets or not all(isinstance(x, dict) for x in budgets):
167
+ return None
168
+ whole = all(number(x.get("max_usd")) for x in budgets)
169
+ each = {x.get("unit_max_usd") for x in budgets}
170
+ each = each.pop() if len(each) == 1 else None
171
+ if not whole and not number(each):
172
+ return None
173
+ spent = [x.get("spent_usd") for x in budgets]
174
+ unsettled = round(sum(x.get("unsettled_usd") or 0 for x in budgets), 6)
175
+ return {"max_usd": round(sum(x["max_usd"] for x in budgets), 6) if whole else None,
176
+ "spent_usd": None if any(not isinstance(v, (int, float)) for v in spent) else round(sum(spent), 6),
177
+ **({"unsettled_usd": unsettled} if unsettled else {}), "stopped": whole and any(x.get("stopped") for x in budgets),
178
+ # how many shards there were and how many stopped: each at its own share, not the launch at the sum
179
+ **({"shards": len(budgets), "shards_stopped": sum(1 for x in budgets if x.get("stopped"))} if whole else {}),
180
+ "cut_off": [u for x in budgets for u in x.get("cut_off") or []],
181
+ "not_run": [u for x in budgets for u in x.get("not_run") or []],
182
+ **({"unit_max_usd": each, "over_unit": [u for x in budgets for u in x.get("over_unit") or []]} if number(each) else {})}
183
+
184
+
185
+ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict:
186
+ """Merge the shard jobs ``job_dirs`` into the new job ``out``.
187
+ {out, trials, merged_from, notes, missing}."""
188
+ got = check(job_dirs, force)
189
+ out = os.path.abspath(out.rstrip("/"))
190
+ if os.path.exists(out) and os.listdir(out):
191
+ raise Refused(f"{out} exists already and is not empty")
192
+ shards = got["shards"]
193
+ name = os.path.basename(out)
194
+ os.makedirs(out, exist_ok=True)
195
+ results = []
196
+ for s in shards:
197
+ for t in s["trials"]:
198
+ src, dest = os.path.join(s["dir"], t), os.path.join(out, t)
199
+ _link_tree(src, dest)
200
+ # the trial's record is its own copy, naming the merged job and the shard it came from
201
+ res = _read(os.path.join(src, "result.json"))
202
+ std = dict(res.get("standard") or {}, job=name, shard_job=s["name"])
203
+ res["standard"] = std
204
+ os.unlink(os.path.join(dest, "result.json"))
205
+ with open(os.path.join(dest, "result.json"), "w", encoding="utf-8") as f:
206
+ json.dump(res, f, indent=1)
207
+ results.append(res)
208
+ planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards)
209
+ starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")]
210
+ ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")]
211
+ errored = sum(1 for r in results if r.get("exception_info"))
212
+ rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results]
213
+ scores = [w["score"] for w in rewards if "score" in w]
214
+ budget = _budget([s["result"].get("budget") for s in shards])
215
+ with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f:
216
+ # ISO stamps in UTC sort as their instants
217
+ json.dump({"id": name, "started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None,
218
+ "n_total_trials": planned,
219
+ "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored,
220
+ "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0},
221
+ "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0),
222
+ "scored": len(scores), "mean_score": job.mean_score(results)},
223
+ **({"budget": budget} if budget else {})}, f, indent=1)
224
+ cfg = json.loads(json.dumps(shards[0]["config"]))
225
+ cfg["job_name"] = name
226
+ std = cfg.setdefault("standard", {})
227
+ std.pop("shard", None)
228
+ std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]),
229
+ **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards]
230
+ if got["units_total"] is not None:
231
+ std["units_total"] = got["units_total"]
232
+ if got["differ"]:
233
+ std["merged_despite"] = got["differ"]
234
+ if cfg.get("datasets"):
235
+ cfg["datasets"][0]["n_tasks"] = planned
236
+ with open(os.path.join(out, "config.json"), "w", encoding="utf-8") as f:
237
+ json.dump(cfg, f, indent=2)
238
+ return {"out": out, "trials": len(results), "merged_from": [s["name"] for s in shards], "notes": got["notes"],
239
+ "missing": got["missing"]}
@@ -0,0 +1,37 @@
1
+ """Canonical names, with read compatibility for earlier catalogues, jobs and recipes."""
2
+ from __future__ import annotations
3
+
4
+ BACKENDS = ("local", "container")
5
+
6
+
7
+ def backend(value):
8
+ """``docker``, the container backend's earlier name, reads as ``container``."""
9
+ return "container" if value == "docker" else value
10
+
11
+
12
+ def kind(value: str) -> str:
13
+ return "capability" if value == "feature" else value
14
+
15
+
16
+ def port(value: str) -> str:
17
+ return "cap." + value[len("feature."):] if isinstance(value, str) and value.startswith("feature.") else value
18
+
19
+
20
+ def definition(value):
21
+ """Normalize only schema vocabulary; preserve commands, paths and user options."""
22
+ if not isinstance(value, dict):
23
+ return value
24
+ result = dict(value)
25
+ if result.get("schema") == "nullius.feature/v1":
26
+ result["schema"] = "nullius.capability/v1"
27
+ if result.get("family") == "feature":
28
+ result["family"] = "capability"
29
+ for field in ("needs", "offers"):
30
+ if isinstance(result.get(field), list):
31
+ result[field] = [port(p) for p in result[field]]
32
+ for field in ("proven", "needs_skills"):
33
+ if isinstance(result.get(field), dict):
34
+ result[field] = {port(p): record for p, record in result[field].items()}
35
+ if isinstance(result.get("versions"), dict):
36
+ result["versions"] = {v: definition(block) for v, block in result["versions"].items()}
37
+ return result
@@ -0,0 +1,44 @@
1
+ """Fail closed on declared public-web requirements for evaluated agents.
2
+
3
+ Original setups and controls are separate protocols. A reachable HTTPS probe
4
+ proves connectivity, not that an agent can search, browse or interpret media.
5
+ """
6
+ from urllib.parse import urlsplit
7
+ from urllib.request import Request, urlopen
8
+
9
+
10
+ def problems(value):
11
+ if not isinstance(value, dict) or set(value) != {"access", "probe"}:
12
+ return ["network: {access: public, probe: an HTTPS URL}"]
13
+ try:
14
+ u = urlsplit(value["probe"])
15
+ valid = (value["access"] == "public" and u.scheme == "https" and u.hostname
16
+ and not u.username and not u.password and not u.query and not u.fragment
17
+ and u.port in (None, 443))
18
+ except (TypeError, ValueError, AttributeError):
19
+ valid = False
20
+ return [] if valid else ["network: public access with an HTTPS probe URL without credentials, query or fragment"]
21
+
22
+
23
+ def required(bench, other):
24
+ if other.get("_kind") != "agent":
25
+ return []
26
+ # Environment selection may also be made per unit by the driver. Require
27
+ # the declared connectivity before launching any evaluated agent.
28
+ return sorted({e["network"]["probe"] for e in (bench.get("environments") or {}).values()
29
+ if e.get("network")})
30
+
31
+
32
+ def check(bench, other, opener=urlopen):
33
+ # The same check on every backend: once, before any task. In a container the
34
+ # run's first unit also checks the route through its public proxy.
35
+ for url in required(bench, other):
36
+ try:
37
+ # Do not send provider credentials or benchmark access tokens.
38
+ with opener(Request(url, headers={"User-Agent": "Nullius-connectivity-check"}), timeout=10) as response:
39
+ if not 200 <= response.status < 400:
40
+ raise OSError(f"HTTP {response.status}")
41
+ except Exception as exc:
42
+ return [f"{bench['label']} requires public HTTPS access; connectivity check failed "
43
+ f"for {url}: {type(exc).__name__}. No tasks were started."]
44
+ return []
@@ -0,0 +1,53 @@
1
+ """How fast a run starts its units when the provider says it is going too fast.
2
+
3
+ The relay passes a 429 to the caller unchanged and never answers for the
4
+ provider; a unit whose last model call was refused that way ends as
5
+ ``RateLimitError``. Starting the next unit at once only sends it into the same
6
+ limit, so after such an end no new unit starts for a while: a pause that
7
+ doubles while units keep ending rate-limited (``FIRST`` seconds, up to
8
+ ``LONGEST``) and resets on a unit that ends any other way. Units already
9
+ running are left alone; the concurrency asked for is unchanged, only how soon
10
+ the next unit starts.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import threading
15
+ import time
16
+ from typing import Optional
17
+
18
+ FIRST = 5.0
19
+ LONGEST = 300.0
20
+ RATE_LIMITED = "RateLimitError"
21
+
22
+
23
+ class Pace:
24
+ def __init__(self, *stops: threading.Event, first: Optional[float] = None, longest: Optional[float] = None):
25
+ self.stops = [s for s in stops if s is not None]
26
+ self.first = FIRST if first is None else first
27
+ self.longest = LONGEST if longest is None else longest
28
+ self.pause = 0.0
29
+ self.until = 0.0
30
+ self._lock = threading.Lock()
31
+
32
+ def ended(self, exception_type: Optional[str]) -> Optional[float]:
33
+ """A unit's end: the pause it starts, in seconds, when it ended
34
+ rate-limited; None otherwise (and the pause is reset)."""
35
+ with self._lock:
36
+ if exception_type != RATE_LIMITED:
37
+ self.pause = 0.0
38
+ return None
39
+ self.pause = min(self.longest, self.pause * 2 if self.pause else self.first)
40
+ self.until = max(self.until, time.time() + self.pause)
41
+ return self.pause
42
+
43
+ def wait(self) -> None:
44
+ """Until the pause is over, or the run is stopped."""
45
+ while not any(s.is_set() for s in self.stops):
46
+ with self._lock:
47
+ left = self.until - time.time()
48
+ if left <= 0:
49
+ return
50
+ if self.stops:
51
+ self.stops[0].wait(min(left, 1.0))
52
+ else:
53
+ time.sleep(min(left, 1.0))