nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""A stdio MCP server that is a benchmark's, run on the machine: this joins
|
|
2
|
+
its stdin and stdout to one TCP address, the unit's door to that server
|
|
3
|
+
(``NULLIUS_MCP_AT``, ``host:port``), and passes the bytes both ways
|
|
4
|
+
unchanged, reading none of them.
|
|
5
|
+
|
|
6
|
+
Mounted alone into the agent's container on the container backend and started
|
|
7
|
+
by the agent as any stdio MCP server (``mcp_machine``); it imports nothing
|
|
8
|
+
of this application, only the standard library, so the container needs
|
|
9
|
+
nothing but ``python3``.
|
|
10
|
+
"""
|
|
11
|
+
import os
|
|
12
|
+
import socket
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def main() -> int:
|
|
18
|
+
at = os.environ.get("NULLIUS_MCP_AT", "")
|
|
19
|
+
host, _, port = at.rpartition(":")
|
|
20
|
+
if not host or not port.isdigit():
|
|
21
|
+
sys.stderr.write("mcp_bridge: NULLIUS_MCP_AT, host:port of the server's door\n")
|
|
22
|
+
return 2
|
|
23
|
+
try:
|
|
24
|
+
s = socket.create_connection((host, int(port)), timeout=30)
|
|
25
|
+
except OSError as e:
|
|
26
|
+
sys.stderr.write(f"mcp_bridge: the server's door {at}: {e}\n")
|
|
27
|
+
return 1
|
|
28
|
+
s.settimeout(None)
|
|
29
|
+
out = sys.stdout.buffer
|
|
30
|
+
|
|
31
|
+
def down(): # the server's replies, to the agent
|
|
32
|
+
try:
|
|
33
|
+
while True:
|
|
34
|
+
data = s.recv(65536)
|
|
35
|
+
if not data:
|
|
36
|
+
break
|
|
37
|
+
out.write(data)
|
|
38
|
+
out.flush()
|
|
39
|
+
except OSError:
|
|
40
|
+
pass
|
|
41
|
+
os._exit(0)
|
|
42
|
+
|
|
43
|
+
threading.Thread(target=down, daemon=True).start()
|
|
44
|
+
src = sys.stdin.buffer
|
|
45
|
+
try:
|
|
46
|
+
while True:
|
|
47
|
+
data = src.read1(65536) if hasattr(src, "read1") else src.read(1)
|
|
48
|
+
if not data:
|
|
49
|
+
break
|
|
50
|
+
s.sendall(data)
|
|
51
|
+
except OSError:
|
|
52
|
+
pass
|
|
53
|
+
try:
|
|
54
|
+
s.shutdown(socket.SHUT_WR) # the agent is done: the server sees its stdin end
|
|
55
|
+
except OSError:
|
|
56
|
+
pass
|
|
57
|
+
threading.Event().wait(5) # what the server still says, then go
|
|
58
|
+
return 0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
if __name__ == "__main__":
|
|
62
|
+
sys.exit(main())
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""A benchmark's MCP server run on the machine, in the driver's environment.
|
|
2
|
+
|
|
3
|
+
A benchmark whose tools are its upstream's own code - with the upstream's
|
|
4
|
+
dependencies - cannot have that server started inside the agent's
|
|
5
|
+
container: the container carries the agent and nothing of the benchmark's.
|
|
6
|
+
So a benchmark may declare ``mcp_servers`` (``definitions._mcp_servers``),
|
|
7
|
+
each a command of its own (``{here}``, ``{install}``: the driver's
|
|
8
|
+
environment, ``driver.install``), and a unit's ``setup`` ask for one in its
|
|
9
|
+
``mcp`` entry by name:
|
|
10
|
+
|
|
11
|
+
{"name": "airline", "server": "domain", "args": [...], "env": {...}}
|
|
12
|
+
|
|
13
|
+
The agent's session is still given a stdio MCP server - every agent that
|
|
14
|
+
proved ``agent.tools/v1`` takes one - and that server is the benchmark's:
|
|
15
|
+
|
|
16
|
+
- **local backend**: the entry is the server's own command, rendered
|
|
17
|
+
(:func:`local_entry`): the agent starts it, on this machine, from the
|
|
18
|
+
driver's environment, as it starts any stdio server;
|
|
19
|
+
- **container backend**: the run starts a :class:`Listener` per server per unit
|
|
20
|
+
on this machine - a Unix socket in a private folder; each connection to it
|
|
21
|
+
is one start of the server, its stdin and stdout the connection - and the
|
|
22
|
+
unit gets one door to it (``container.Unit``'s ``doors``: ``forward.py``
|
|
23
|
+
on the unit's internal network only, the socket alone mounted into it,
|
|
24
|
+
passing every connection to that socket and nowhere else). The agent's
|
|
25
|
+
entry is ``mcp_bridge.py``, a file of the standard library mounted alone
|
|
26
|
+
into its container, which joins its stdin and stdout to the door
|
|
27
|
+
(:func:`bridge_entry`). Nothing of the server, its environment or its
|
|
28
|
+
data enters the agent's container, and the door reaches nothing else.
|
|
29
|
+
|
|
30
|
+
Either way a server started twice in a unit is two processes, as a stdio
|
|
31
|
+
server is; what it logs, it logs on this machine (the trial's
|
|
32
|
+
``verifier/``), out of the agent's reach in a container.
|
|
33
|
+
"""
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import os
|
|
37
|
+
import re
|
|
38
|
+
import shutil
|
|
39
|
+
import socket
|
|
40
|
+
import subprocess
|
|
41
|
+
import tempfile
|
|
42
|
+
import threading
|
|
43
|
+
from typing import List, Optional, Tuple
|
|
44
|
+
|
|
45
|
+
from nullius.standard import install
|
|
46
|
+
|
|
47
|
+
BRIDGE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp_bridge.py")
|
|
48
|
+
DOOR_PORT = 9000 # where a door listens, on the unit's network
|
|
49
|
+
_ALIAS = re.compile(r"[^a-z0-9-]+")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Unknown(ValueError):
|
|
53
|
+
"""A unit's setup named a server the benchmark does not declare."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def wanted(setup: dict) -> List[dict]:
|
|
57
|
+
"""The setup's ``mcp`` entries that name a server of the benchmark's."""
|
|
58
|
+
return [s for s in setup.get("mcp") or [] if isinstance(s, dict) and s.get("server") is not None]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def argv(bench: dict, entry: dict, inst: Optional[str]) -> List[str]:
|
|
62
|
+
"""The server's command, its slots filled, and the entry's ``args``."""
|
|
63
|
+
servers = bench.get("mcp_servers") or {}
|
|
64
|
+
sv = servers.get(entry.get("server"))
|
|
65
|
+
if not isinstance(sv, dict):
|
|
66
|
+
raise Unknown(f"mcp: {entry.get('server')!r} is not one of the benchmark's mcp_servers "
|
|
67
|
+
f"({', '.join(servers) or 'none declared'})")
|
|
68
|
+
args = entry.get("args") or []
|
|
69
|
+
if not isinstance(args, list) or not all(isinstance(a, str) for a in args):
|
|
70
|
+
raise Unknown(f"mcp {entry.get('name')}: args, a list of words")
|
|
71
|
+
return install.words(sv["command"], {"here": bench["_here"], "install": inst or ""}) + list(args)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def server_env(base: dict, entry: dict) -> dict:
|
|
75
|
+
"""The server's environment: the driver's (``base``: no secret, the
|
|
76
|
+
upstream and dataset named) and the entry's own ``env``."""
|
|
77
|
+
own = entry.get("env") or {}
|
|
78
|
+
if not isinstance(own, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in own.items()):
|
|
79
|
+
raise Unknown(f"mcp {entry.get('name')}: env, names to strings")
|
|
80
|
+
return dict(base, **own)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def local_entry(bench: dict, entry: dict, inst: Optional[str], base: dict) -> dict:
|
|
84
|
+
"""The agent's stdio entry on the local backend: the server's own command."""
|
|
85
|
+
cmd = argv(bench, entry, inst)
|
|
86
|
+
return {"name": str(entry.get("name") or entry["server"]), "command": cmd[0], "args": cmd[1:],
|
|
87
|
+
"env": server_env(base, entry)}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def alias(name: str, i: int) -> str:
|
|
91
|
+
"""The name a door answers by on the unit's network."""
|
|
92
|
+
return f"mcp-{i}-" + (_ALIAS.sub("-", name.lower()).strip("-")[:30] or "server")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def bridge_entry(name: str, door: str) -> dict:
|
|
96
|
+
"""The agent's stdio entry in a container: the bridge, told its door."""
|
|
97
|
+
return {"name": name, "command": "python3", "args": [BRIDGE], "env": {"NULLIUS_MCP_AT": door}}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Listener:
|
|
101
|
+
"""A server, started once per connection to a Unix socket: the
|
|
102
|
+
connection is its stdin and stdout, its stderr the log. The socket sits
|
|
103
|
+
in a folder of its own (0700), short enough for a socket's path."""
|
|
104
|
+
|
|
105
|
+
def __init__(self, cmd: List[str], env: dict, cwd: str, log: str):
|
|
106
|
+
self.cmd, self.env, self.cwd, self.log = cmd, env, cwd, log
|
|
107
|
+
self.dir = tempfile.mkdtemp(prefix="nullius-mcp-")
|
|
108
|
+
self.path = os.path.join(self.dir, "s")
|
|
109
|
+
self.procs: List[subprocess.Popen] = []
|
|
110
|
+
self.sock: Optional[socket.socket] = None
|
|
111
|
+
self.lock = threading.Lock()
|
|
112
|
+
self.closed = False
|
|
113
|
+
|
|
114
|
+
def start(self) -> "Listener":
|
|
115
|
+
os.chmod(self.dir, 0o700)
|
|
116
|
+
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
117
|
+
self.sock.bind(self.path)
|
|
118
|
+
self.sock.listen(8)
|
|
119
|
+
threading.Thread(target=self._accept, daemon=True).start()
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def _accept(self):
|
|
123
|
+
while True:
|
|
124
|
+
try:
|
|
125
|
+
conn, _ = self.sock.accept()
|
|
126
|
+
except OSError:
|
|
127
|
+
return
|
|
128
|
+
try:
|
|
129
|
+
with self.lock:
|
|
130
|
+
if self.closed:
|
|
131
|
+
conn.close()
|
|
132
|
+
return
|
|
133
|
+
with open(self.log, "a") as err:
|
|
134
|
+
self.procs.append(subprocess.Popen(self.cmd, cwd=self.cwd, env=self.env, stdin=conn.fileno(),
|
|
135
|
+
stdout=conn.fileno(), stderr=err, start_new_session=True))
|
|
136
|
+
except OSError as e:
|
|
137
|
+
with open(self.log, "a") as err:
|
|
138
|
+
err.write(f"[nullius] could not start the server: {e}\n")
|
|
139
|
+
finally:
|
|
140
|
+
conn.close() # the server holds its own copy
|
|
141
|
+
|
|
142
|
+
def stop(self):
|
|
143
|
+
with self.lock:
|
|
144
|
+
self.closed = True
|
|
145
|
+
if self.sock is not None:
|
|
146
|
+
try:
|
|
147
|
+
self.sock.close()
|
|
148
|
+
except OSError:
|
|
149
|
+
pass
|
|
150
|
+
for p in self.procs:
|
|
151
|
+
if p.poll() is None:
|
|
152
|
+
p.terminate()
|
|
153
|
+
for p in self.procs:
|
|
154
|
+
try:
|
|
155
|
+
p.wait(timeout=5)
|
|
156
|
+
except subprocess.TimeoutExpired:
|
|
157
|
+
p.kill()
|
|
158
|
+
shutil.rmtree(self.dir, ignore_errors=True)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def doors(bench: dict, setup: dict, inst: Optional[str], base: dict, cwd: str, log_dir: str
|
|
162
|
+
) -> Tuple[List[dict], List[Tuple[str, int, str]], List[Listener]]:
|
|
163
|
+
"""For the container backend: the setup's ``mcp`` with each machine server's
|
|
164
|
+
entry replaced by the bridge's, the doors the unit opens (alias, port,
|
|
165
|
+
socket) and the listeners started - stopped by the caller, also on an
|
|
166
|
+
error. Raises :class:`Unknown` for a server not declared."""
|
|
167
|
+
out, opened, started = [], [], []
|
|
168
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
169
|
+
try:
|
|
170
|
+
for i, s in enumerate(setup.get("mcp") or [], 1):
|
|
171
|
+
if not (isinstance(s, dict) and s.get("server") is not None):
|
|
172
|
+
out.append(s)
|
|
173
|
+
continue
|
|
174
|
+
cmd, env = argv(bench, s, inst), server_env(base, s)
|
|
175
|
+
name = str(s.get("name") or s["server"])
|
|
176
|
+
ln = Listener(cmd, env, cwd, os.path.join(log_dir, f"mcp-{i}.log")).start()
|
|
177
|
+
started.append(ln)
|
|
178
|
+
a = alias(name, i)
|
|
179
|
+
opened.append((a, DOOR_PORT, ln.path))
|
|
180
|
+
out.append(bridge_entry(name, f"{a}:{DOOR_PORT}"))
|
|
181
|
+
except Exception:
|
|
182
|
+
for ln in started:
|
|
183
|
+
ln.stop()
|
|
184
|
+
raise
|
|
185
|
+
return out, opened, started
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def entries(bench: dict, setup: dict, inst: Optional[str], base: dict) -> List[dict]:
|
|
189
|
+
"""For the local backend: the setup's ``mcp``, each machine server's entry
|
|
190
|
+
its own command. Raises :class:`Unknown` for a server not declared."""
|
|
191
|
+
return [local_entry(bench, s, inst, base) if isinstance(s, dict) and s.get("server") is not None else s
|
|
192
|
+
for s in setup.get("mcp") or []]
|
|
193
|
+
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""Shard jobs of one launch put back together as one job.
|
|
2
|
+
|
|
3
|
+
A launch run over several machines (``shard``) leaves one job per shard. The
|
|
4
|
+
pages would read each as a run of its own, so ``cli merge`` makes the one
|
|
5
|
+
job a reader expects: every shard's trials in it, under the names they had
|
|
6
|
+
(a trial's tag is its unit's place in the whole dataset, so shards never
|
|
7
|
+
share one), its ``result.json`` recomputed from them - the trials, the
|
|
8
|
+
verdicts, the earliest start and the latest end - and its ``config.json``
|
|
9
|
+
the shards' with ``shard`` taken out and ``merged_from`` put in.
|
|
10
|
+
|
|
11
|
+
What is merged must be one launch: the same launch id, benchmark and
|
|
12
|
+
version, dataset, agent or capability and model, and the same number of shards
|
|
13
|
+
- else it is refused, naming what differs, unless ``force``. Two trials of
|
|
14
|
+
one unit are always refused, with the units named: one of them would be
|
|
15
|
+
counted twice. Units no shard ran are not a refusal - a shard may be
|
|
16
|
+
missing, or stopped early - and are said: the shards absent, and the units a
|
|
17
|
+
shard planned and has no trial for.
|
|
18
|
+
|
|
19
|
+
Files are hard-linked into the merged job where the file system allows (a
|
|
20
|
+
copy where it does not), so a merge of a thousand trials costs no second
|
|
21
|
+
copy of their workspaces; each trial's ``result.json`` is a copy of its own,
|
|
22
|
+
rewritten to name the merged job. The shards are left as they were.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import shutil
|
|
29
|
+
from typing import Dict, List, Optional
|
|
30
|
+
|
|
31
|
+
from nullius.standard import job
|
|
32
|
+
from nullius.standard.names import backend as backend_name
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# What every shard of one launch shares, as (what a refusal calls it, where it is in config.json).
|
|
36
|
+
SAME = (("launch", ("standard", "launch")), ("benchmark", ("standard", "benchmark")),
|
|
37
|
+
("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")),
|
|
38
|
+
("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")),
|
|
39
|
+
("backend", ("standard", "backend")), ("number of shards", ("standard", "shard", "of")))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Refused(Exception):
|
|
43
|
+
"""The jobs cannot be merged as asked; the message says why."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _read(path: str) -> dict:
|
|
47
|
+
try:
|
|
48
|
+
with open(path, encoding="utf-8") as f:
|
|
49
|
+
got = json.load(f)
|
|
50
|
+
return got if isinstance(got, dict) else {}
|
|
51
|
+
except (OSError, ValueError):
|
|
52
|
+
return {}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _at(d, path):
|
|
56
|
+
for k in path:
|
|
57
|
+
if isinstance(d, dict):
|
|
58
|
+
d = d.get(k)
|
|
59
|
+
elif isinstance(d, list) and isinstance(k, int) and k < len(d):
|
|
60
|
+
d = d[k]
|
|
61
|
+
else:
|
|
62
|
+
return None
|
|
63
|
+
return d
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _trials(job_dir: str) -> List[str]:
|
|
67
|
+
return sorted(n for n in os.listdir(job_dir)
|
|
68
|
+
if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json")))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _link_tree(src: str, dest: str) -> None:
|
|
72
|
+
"""``src`` into ``dest``, each file hard-linked (copied across devices)."""
|
|
73
|
+
for d, dirs, files in os.walk(src):
|
|
74
|
+
rel = os.path.relpath(d, src)
|
|
75
|
+
os.makedirs(os.path.join(dest, rel), exist_ok=True)
|
|
76
|
+
for f in files:
|
|
77
|
+
a, b = os.path.join(d, f), os.path.join(dest, rel, f)
|
|
78
|
+
if os.path.islink(a):
|
|
79
|
+
os.symlink(os.readlink(a), b)
|
|
80
|
+
continue
|
|
81
|
+
try:
|
|
82
|
+
os.link(a, b)
|
|
83
|
+
except OSError:
|
|
84
|
+
shutil.copy2(a, b)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def check(job_dirs: List[str], force: bool = False) -> dict:
|
|
88
|
+
"""What merging ``job_dirs`` would make, without writing: the shards read,
|
|
89
|
+
the trials by unit, the notes; ``Refused`` when they cannot be merged."""
|
|
90
|
+
if not job_dirs:
|
|
91
|
+
raise Refused("no jobs to merge")
|
|
92
|
+
shards = []
|
|
93
|
+
for d in job_dirs:
|
|
94
|
+
d = os.path.abspath(d.rstrip("/"))
|
|
95
|
+
cfg, res = _read(os.path.join(d, "config.json")), _read(os.path.join(d, "result.json"))
|
|
96
|
+
if isinstance(cfg.get("standard"), dict) and "backend" in cfg["standard"]:
|
|
97
|
+
cfg["standard"]["backend"] = backend_name(cfg["standard"]["backend"])
|
|
98
|
+
if cfg.get("runner") != "standard":
|
|
99
|
+
raise Refused(f"{d}: not a job of the Nullius standard (no config.json naming it)")
|
|
100
|
+
sh = _at(cfg, ("standard", "shard"))
|
|
101
|
+
if not isinstance(sh, dict) and not force:
|
|
102
|
+
raise Refused(f"{d}: not a shard (its config has no standard.shard); --force merges it anyway")
|
|
103
|
+
shards.append({"dir": d, "name": os.path.basename(d), "config": cfg, "result": res, "shard": sh or {}})
|
|
104
|
+
if len({s["dir"] for s in shards}) != len(shards):
|
|
105
|
+
raise Refused("a job is named twice")
|
|
106
|
+
differ = []
|
|
107
|
+
for label, path in SAME:
|
|
108
|
+
seen = {json.dumps(_at(s["config"], path)) for s in shards}
|
|
109
|
+
if len(seen) > 1:
|
|
110
|
+
differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards))
|
|
111
|
+
if differ and not force:
|
|
112
|
+
raise Refused("the shards are not of one launch - " + "; ".join(differ) + " (--force merges them anyway)")
|
|
113
|
+
# every trial by its unit: two of one unit are never merged, force or not
|
|
114
|
+
by_unit: Dict[str, List[tuple]] = {}
|
|
115
|
+
names: Dict[str, List[str]] = {}
|
|
116
|
+
for s in shards:
|
|
117
|
+
s["trials"] = _trials(s["dir"])
|
|
118
|
+
for t in s["trials"]:
|
|
119
|
+
unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t
|
|
120
|
+
by_unit.setdefault(str(unit), []).append((s["name"], t))
|
|
121
|
+
names.setdefault(t, []).append(s["name"])
|
|
122
|
+
twice = {u: w for u, w in by_unit.items() if len(w) > 1}
|
|
123
|
+
if twice:
|
|
124
|
+
shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10])
|
|
125
|
+
raise Refused(f"{len(twice)} unit(s) have a trial in more than one shard: {shown}"
|
|
126
|
+
+ (" and more" if len(twice) > 10 else ""))
|
|
127
|
+
clash = {t: w for t, w in names.items() if len(w) > 1}
|
|
128
|
+
if clash:
|
|
129
|
+
raise Refused("trials of the same name in several shards: " + ", ".join(sorted(clash)[:10]))
|
|
130
|
+
notes, missing = [], []
|
|
131
|
+
of = {s["shard"].get("of") for s in shards if s["shard"]}
|
|
132
|
+
totals = {s["shard"].get("units_total") for s in shards if s["shard"]}
|
|
133
|
+
if len(of) == 1:
|
|
134
|
+
n = of.pop()
|
|
135
|
+
have = sorted(s["shard"].get("index") for s in shards if s["shard"])
|
|
136
|
+
absent = [i for i in range(1, (n or 0) + 1) if i not in have]
|
|
137
|
+
if absent:
|
|
138
|
+
notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge")
|
|
139
|
+
dup = sorted({i for i in have if have.count(i) > 1})
|
|
140
|
+
if dup:
|
|
141
|
+
notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)")
|
|
142
|
+
for s in shards:
|
|
143
|
+
planned = s["shard"].get("planned")
|
|
144
|
+
if isinstance(planned, list):
|
|
145
|
+
ran = {u for u, w in by_unit.items() if any(j == s["name"] for j, _ in w)}
|
|
146
|
+
left = [u for u in planned if u not in ran]
|
|
147
|
+
if left:
|
|
148
|
+
missing += left
|
|
149
|
+
notes.append(f"{s['name']}: {len(left)} planned unit(s) with no trial: {', '.join(left[:5])}" + (" ..." if len(left) > 5 else ""))
|
|
150
|
+
if s["shard"].get("limit"):
|
|
151
|
+
notes.append(f"{s['name']}: limited to its first {s['shard']['limit']} units")
|
|
152
|
+
total = totals.pop() if len(totals) == 1 else None
|
|
153
|
+
if isinstance(total, int):
|
|
154
|
+
short = total - len(by_unit)
|
|
155
|
+
if short > 0:
|
|
156
|
+
notes.append(f"{short} of the dataset's {total} units are not in the merge")
|
|
157
|
+
return {"shards": shards, "by_unit": by_unit, "notes": notes, "missing": missing, "units_total": total, "differ": differ}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _budget(budgets: list) -> Optional[dict]:
|
|
161
|
+
"""The shards' spending caps as one (``run.Run.budget``): the caps and what was spent added up, stopped
|
|
162
|
+
when any shard was, their units cut off and not run together; a cap per unit, the same on every shard, is
|
|
163
|
+
kept with the units that reached it. None unless every shard had a cap: a cap over part of a launch is
|
|
164
|
+
not the launch's."""
|
|
165
|
+
number = lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)
|
|
166
|
+
if not budgets or not all(isinstance(x, dict) for x in budgets):
|
|
167
|
+
return None
|
|
168
|
+
whole = all(number(x.get("max_usd")) for x in budgets)
|
|
169
|
+
each = {x.get("unit_max_usd") for x in budgets}
|
|
170
|
+
each = each.pop() if len(each) == 1 else None
|
|
171
|
+
if not whole and not number(each):
|
|
172
|
+
return None
|
|
173
|
+
spent = [x.get("spent_usd") for x in budgets]
|
|
174
|
+
unsettled = round(sum(x.get("unsettled_usd") or 0 for x in budgets), 6)
|
|
175
|
+
return {"max_usd": round(sum(x["max_usd"] for x in budgets), 6) if whole else None,
|
|
176
|
+
"spent_usd": None if any(not isinstance(v, (int, float)) for v in spent) else round(sum(spent), 6),
|
|
177
|
+
**({"unsettled_usd": unsettled} if unsettled else {}), "stopped": whole and any(x.get("stopped") for x in budgets),
|
|
178
|
+
# how many shards there were and how many stopped: each at its own share, not the launch at the sum
|
|
179
|
+
**({"shards": len(budgets), "shards_stopped": sum(1 for x in budgets if x.get("stopped"))} if whole else {}),
|
|
180
|
+
"cut_off": [u for x in budgets for u in x.get("cut_off") or []],
|
|
181
|
+
"not_run": [u for x in budgets for u in x.get("not_run") or []],
|
|
182
|
+
**({"unit_max_usd": each, "over_unit": [u for x in budgets for u in x.get("over_unit") or []]} if number(each) else {})}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def merge(job_dirs: List[str], out: str, force: bool = False) -> dict:
|
|
186
|
+
"""Merge the shard jobs ``job_dirs`` into the new job ``out``.
|
|
187
|
+
{out, trials, merged_from, notes, missing}."""
|
|
188
|
+
got = check(job_dirs, force)
|
|
189
|
+
out = os.path.abspath(out.rstrip("/"))
|
|
190
|
+
if os.path.exists(out) and os.listdir(out):
|
|
191
|
+
raise Refused(f"{out} exists already and is not empty")
|
|
192
|
+
shards = got["shards"]
|
|
193
|
+
name = os.path.basename(out)
|
|
194
|
+
os.makedirs(out, exist_ok=True)
|
|
195
|
+
results = []
|
|
196
|
+
for s in shards:
|
|
197
|
+
for t in s["trials"]:
|
|
198
|
+
src, dest = os.path.join(s["dir"], t), os.path.join(out, t)
|
|
199
|
+
_link_tree(src, dest)
|
|
200
|
+
# the trial's record is its own copy, naming the merged job and the shard it came from
|
|
201
|
+
res = _read(os.path.join(src, "result.json"))
|
|
202
|
+
std = dict(res.get("standard") or {}, job=name, shard_job=s["name"])
|
|
203
|
+
res["standard"] = std
|
|
204
|
+
os.unlink(os.path.join(dest, "result.json"))
|
|
205
|
+
with open(os.path.join(dest, "result.json"), "w", encoding="utf-8") as f:
|
|
206
|
+
json.dump(res, f, indent=1)
|
|
207
|
+
results.append(res)
|
|
208
|
+
planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards)
|
|
209
|
+
starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")]
|
|
210
|
+
ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")]
|
|
211
|
+
errored = sum(1 for r in results if r.get("exception_info"))
|
|
212
|
+
rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results]
|
|
213
|
+
scores = [w["score"] for w in rewards if "score" in w]
|
|
214
|
+
budget = _budget([s["result"].get("budget") for s in shards])
|
|
215
|
+
with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f:
|
|
216
|
+
# ISO stamps in UTC sort as their instants
|
|
217
|
+
json.dump({"id": name, "started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None,
|
|
218
|
+
"n_total_trials": planned,
|
|
219
|
+
"stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored,
|
|
220
|
+
"n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0},
|
|
221
|
+
"standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0),
|
|
222
|
+
"scored": len(scores), "mean_score": job.mean_score(results)},
|
|
223
|
+
**({"budget": budget} if budget else {})}, f, indent=1)
|
|
224
|
+
cfg = json.loads(json.dumps(shards[0]["config"]))
|
|
225
|
+
cfg["job_name"] = name
|
|
226
|
+
std = cfg.setdefault("standard", {})
|
|
227
|
+
std.pop("shard", None)
|
|
228
|
+
std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]),
|
|
229
|
+
**({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards]
|
|
230
|
+
if got["units_total"] is not None:
|
|
231
|
+
std["units_total"] = got["units_total"]
|
|
232
|
+
if got["differ"]:
|
|
233
|
+
std["merged_despite"] = got["differ"]
|
|
234
|
+
if cfg.get("datasets"):
|
|
235
|
+
cfg["datasets"][0]["n_tasks"] = planned
|
|
236
|
+
with open(os.path.join(out, "config.json"), "w", encoding="utf-8") as f:
|
|
237
|
+
json.dump(cfg, f, indent=2)
|
|
238
|
+
return {"out": out, "trials": len(results), "merged_from": [s["name"] for s in shards], "notes": got["notes"],
|
|
239
|
+
"missing": got["missing"]}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Canonical names, with read compatibility for earlier catalogues, jobs and recipes."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
BACKENDS = ("local", "container")
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def backend(value):
|
|
8
|
+
"""``docker``, the container backend's earlier name, reads as ``container``."""
|
|
9
|
+
return "container" if value == "docker" else value
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def kind(value: str) -> str:
|
|
13
|
+
return "capability" if value == "feature" else value
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def port(value: str) -> str:
|
|
17
|
+
return "cap." + value[len("feature."):] if isinstance(value, str) and value.startswith("feature.") else value
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def definition(value):
|
|
21
|
+
"""Normalize only schema vocabulary; preserve commands, paths and user options."""
|
|
22
|
+
if not isinstance(value, dict):
|
|
23
|
+
return value
|
|
24
|
+
result = dict(value)
|
|
25
|
+
if result.get("schema") == "nullius.feature/v1":
|
|
26
|
+
result["schema"] = "nullius.capability/v1"
|
|
27
|
+
if result.get("family") == "feature":
|
|
28
|
+
result["family"] = "capability"
|
|
29
|
+
for field in ("needs", "offers"):
|
|
30
|
+
if isinstance(result.get(field), list):
|
|
31
|
+
result[field] = [port(p) for p in result[field]]
|
|
32
|
+
for field in ("proven", "needs_skills"):
|
|
33
|
+
if isinstance(result.get(field), dict):
|
|
34
|
+
result[field] = {port(p): record for p, record in result[field].items()}
|
|
35
|
+
if isinstance(result.get("versions"), dict):
|
|
36
|
+
result["versions"] = {v: definition(block) for v, block in result["versions"].items()}
|
|
37
|
+
return result
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Fail closed on declared public-web requirements for evaluated agents.
|
|
2
|
+
|
|
3
|
+
Original setups and controls are separate protocols. A reachable HTTPS probe
|
|
4
|
+
proves connectivity, not that an agent can search, browse or interpret media.
|
|
5
|
+
"""
|
|
6
|
+
from urllib.parse import urlsplit
|
|
7
|
+
from urllib.request import Request, urlopen
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def problems(value):
|
|
11
|
+
if not isinstance(value, dict) or set(value) != {"access", "probe"}:
|
|
12
|
+
return ["network: {access: public, probe: an HTTPS URL}"]
|
|
13
|
+
try:
|
|
14
|
+
u = urlsplit(value["probe"])
|
|
15
|
+
valid = (value["access"] == "public" and u.scheme == "https" and u.hostname
|
|
16
|
+
and not u.username and not u.password and not u.query and not u.fragment
|
|
17
|
+
and u.port in (None, 443))
|
|
18
|
+
except (TypeError, ValueError, AttributeError):
|
|
19
|
+
valid = False
|
|
20
|
+
return [] if valid else ["network: public access with an HTTPS probe URL without credentials, query or fragment"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def required(bench, other):
|
|
24
|
+
if other.get("_kind") != "agent":
|
|
25
|
+
return []
|
|
26
|
+
# Environment selection may also be made per unit by the driver. Require
|
|
27
|
+
# the declared connectivity before launching any evaluated agent.
|
|
28
|
+
return sorted({e["network"]["probe"] for e in (bench.get("environments") or {}).values()
|
|
29
|
+
if e.get("network")})
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def check(bench, other, opener=urlopen):
|
|
33
|
+
# The same check on every backend: once, before any task. In a container the
|
|
34
|
+
# run's first unit also checks the route through its public proxy.
|
|
35
|
+
for url in required(bench, other):
|
|
36
|
+
try:
|
|
37
|
+
# Do not send provider credentials or benchmark access tokens.
|
|
38
|
+
with opener(Request(url, headers={"User-Agent": "Nullius-connectivity-check"}), timeout=10) as response:
|
|
39
|
+
if not 200 <= response.status < 400:
|
|
40
|
+
raise OSError(f"HTTP {response.status}")
|
|
41
|
+
except Exception as exc:
|
|
42
|
+
return [f"{bench['label']} requires public HTTPS access; connectivity check failed "
|
|
43
|
+
f"for {url}: {type(exc).__name__}. No tasks were started."]
|
|
44
|
+
return []
|
nullius/standard/pace.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""How fast a run starts its units when the provider says it is going too fast.
|
|
2
|
+
|
|
3
|
+
The relay passes a 429 to the caller unchanged and never answers for the
|
|
4
|
+
provider; a unit whose last model call was refused that way ends as
|
|
5
|
+
``RateLimitError``. Starting the next unit at once only sends it into the same
|
|
6
|
+
limit, so after such an end no new unit starts for a while: a pause that
|
|
7
|
+
doubles while units keep ending rate-limited (``FIRST`` seconds, up to
|
|
8
|
+
``LONGEST``) and resets on a unit that ends any other way. Units already
|
|
9
|
+
running are left alone; the concurrency asked for is unchanged, only how soon
|
|
10
|
+
the next unit starts.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
FIRST = 5.0
|
|
19
|
+
LONGEST = 300.0
|
|
20
|
+
RATE_LIMITED = "RateLimitError"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Pace:
|
|
24
|
+
def __init__(self, *stops: threading.Event, first: Optional[float] = None, longest: Optional[float] = None):
|
|
25
|
+
self.stops = [s for s in stops if s is not None]
|
|
26
|
+
self.first = FIRST if first is None else first
|
|
27
|
+
self.longest = LONGEST if longest is None else longest
|
|
28
|
+
self.pause = 0.0
|
|
29
|
+
self.until = 0.0
|
|
30
|
+
self._lock = threading.Lock()
|
|
31
|
+
|
|
32
|
+
def ended(self, exception_type: Optional[str]) -> Optional[float]:
|
|
33
|
+
"""A unit's end: the pause it starts, in seconds, when it ended
|
|
34
|
+
rate-limited; None otherwise (and the pause is reset)."""
|
|
35
|
+
with self._lock:
|
|
36
|
+
if exception_type != RATE_LIMITED:
|
|
37
|
+
self.pause = 0.0
|
|
38
|
+
return None
|
|
39
|
+
self.pause = min(self.longest, self.pause * 2 if self.pause else self.first)
|
|
40
|
+
self.until = max(self.until, time.time() + self.pause)
|
|
41
|
+
return self.pause
|
|
42
|
+
|
|
43
|
+
def wait(self) -> None:
|
|
44
|
+
"""Until the pause is over, or the run is stopped."""
|
|
45
|
+
while not any(s.is_set() for s in self.stops):
|
|
46
|
+
with self._lock:
|
|
47
|
+
left = self.until - time.time()
|
|
48
|
+
if left <= 0:
|
|
49
|
+
return
|
|
50
|
+
if self.stops:
|
|
51
|
+
self.stops[0].wait(min(left, 1.0))
|
|
52
|
+
else:
|
|
53
|
+
time.sleep(min(left, 1.0))
|