nullius-eval 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +54 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +16 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +744 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +994 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +453 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +25 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +34 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +354 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +87 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +387 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +145 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/providers.js +78 -0
- _nullius_app/static/settings/sandbox.js +83 -0
- _nullius_app/static/settings/settings.css +229 -0
- _nullius_app/static/style.css +1751 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +477 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +301 -0
- _nullius_app/templates/catalogue/benchmarks.html +92 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +446 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/settings.html +508 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +392 -0
- _nullius_app/web/catalogue.py +374 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +205 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +224 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +461 -0
- _nullius_app/web/settings.py +210 -0
- _nullius_app/web/shared.py +468 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +202 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +120 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +115 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +316 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +124 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +37 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +208 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +114 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +42 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +64 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +323 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +551 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +74 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +105 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +605 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +111 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +495 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +70 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +189 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +160 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +170 -0
- nullius/catalogue/catalog.py +413 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +85 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +83 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1624 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +339 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +416 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +172 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +112 -0
- nullius/paths.py +84 -0
- nullius/prerequisites.py +111 -0
- nullius/progress.py +294 -0
- nullius/providers.py +115 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +189 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +139 -0
- nullius/runs/live.py +705 -0
- nullius/runs/parse.py +991 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +146 -0
- nullius/secrets.py +192 -0
- nullius/settings.py +456 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +301 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +650 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1381 -0
- nullius/standard/definitions.py +1577 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +255 -0
- nullius/standard/forward.py +92 -0
- nullius/standard/hostreq.py +65 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +397 -0
- nullius/standard/job.py +541 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +209 -0
- nullius/standard/names.py +30 -0
- nullius/standard/payload.py +255 -0
- nullius/standard/ports.py +64 -0
- nullius/standard/prove.py +357 -0
- nullius/standard/recipe.py +663 -0
- nullius/standard/relay.py +629 -0
- nullius/standard/run.py +1367 -0
- nullius/standard/shard.py +71 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +409 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0rc1.dist-info/METADATA +93 -0
- nullius_eval-0.1.0rc1.dist-info/RECORD +268 -0
- nullius_eval-0.1.0rc1.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0rc1.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""How to add an agent or a capability, as the Agents and Capabilities tabs show it:
|
|
2
|
+
the commands to type and the files to write, in order.
|
|
3
|
+
|
|
4
|
+
Each step is ``{title, text, code, lang, file}``: ``code`` is what is typed or
|
|
5
|
+
written, ``file`` where it goes. The definitions shown are real: the suite
|
|
6
|
+
checks every ``agent.json`` and ``capability.json`` here with
|
|
7
|
+
``definitions.problems``, so the page never shows one that would be refused.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import shlex
|
|
14
|
+
|
|
15
|
+
from nullius import paths
|
|
16
|
+
|
|
17
|
+
CLI = "nullius standard"
|
|
18
|
+
|
|
19
|
+
AGENT_JSON = {
|
|
20
|
+
"schema": "nullius.agent/v1",
|
|
21
|
+
"key": "my-agent",
|
|
22
|
+
"label": "My agent",
|
|
23
|
+
"summary": "What the agent is, and how it is driven: its ACP adapter, its model through the relay.",
|
|
24
|
+
"upstream": {"package": "@acme/my-agent-acp", "version": "1.4.2", "license": "MIT"},
|
|
25
|
+
"install": {"npm": ["@acme/my-agent-acp@1.4.2"]},
|
|
26
|
+
"start": {"acp": ["{install}/node_modules/.bin/my-agent-acp"]},
|
|
27
|
+
"model": {
|
|
28
|
+
"apis": ["chat"],
|
|
29
|
+
"env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "OPENAI_MODEL": "{model}"},
|
|
30
|
+
},
|
|
31
|
+
"offers": [],
|
|
32
|
+
"proven": {},
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
# What an agent may add: an option a run can set (its default the vendor's own,
|
|
36
|
+
# filled into a template as {opt_<name>}), and its own non-interactive command,
|
|
37
|
+
# for cli fidelity to compare with ACP.
|
|
38
|
+
AGENT_OPTIONS = {
|
|
39
|
+
"model": {
|
|
40
|
+
"apis": ["chat"],
|
|
41
|
+
"env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "OPENAI_MODEL": "{model}",
|
|
42
|
+
"MY_AGENT_REASONING": "{opt_reasoning_effort}"},
|
|
43
|
+
},
|
|
44
|
+
"options": {"reasoning_effort": {
|
|
45
|
+
"summary": "How hard the model reasons. The default is the agent's own, as its documentation or its source says.",
|
|
46
|
+
"values": ["low", "medium", "high"], "default": "medium"}},
|
|
47
|
+
"fidelity": {"command": ["{install}/node_modules/.bin/my-agent", "run", "--yes", "--", "{prompt}"]},
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
FEATURE_JSON = {
|
|
51
|
+
"schema": "nullius.capability/v1",
|
|
52
|
+
"key": "my-memory",
|
|
53
|
+
"label": "My memory",
|
|
54
|
+
"summary": "What the memory keeps, and what it gives back for a question.",
|
|
55
|
+
"upstream": {"package": "my-memory", "version": "0.3.1", "license": "MIT"},
|
|
56
|
+
"kind": "memory",
|
|
57
|
+
"install": {"pip": ["my-memory==0.3.1"]},
|
|
58
|
+
"start": {"command": ["{install}/.venv/bin/python", "{here}/capability.py"]},
|
|
59
|
+
"offers": [],
|
|
60
|
+
"proven": {},
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
# The same memory installed from its own repository, calling a model: the parts that change.
|
|
64
|
+
FEATURE_FROM_REPO = {
|
|
65
|
+
"upstream": {"repository": "https://github.com/acme/my-memory", "tested": "0123456789abcdef0123456789abcdef01234567",
|
|
66
|
+
"license": "MIT"},
|
|
67
|
+
"install": {"python": "3.12", "project": "packages/memory", "extras": ["embeddings"],
|
|
68
|
+
"indexes": [{"url": "https://download.pytorch.org/whl/cpu", "packages": ["torch"]}],
|
|
69
|
+
"prefetch": ["BAAI/bge-small-en-v1.5"]},
|
|
70
|
+
"env": {"HF_HUB_OFFLINE": "1"},
|
|
71
|
+
"model": {"apis": ["chat"], "env": {"API_BASE": "{base_url}", "API_KEY": "{key}", "MODEL_NAME": "{model}"}},
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
FEATURE_PY = '''"""My memory behind cap.memory/v1: one JSON line in, one JSON line out."""
|
|
75
|
+
import json
|
|
76
|
+
import sys
|
|
77
|
+
|
|
78
|
+
from my_memory import Store # your library
|
|
79
|
+
|
|
80
|
+
OUT, sys.stdout = sys.stdout, sys.stderr # stdout: the replies only
|
|
81
|
+
store = Store() # one process per question
|
|
82
|
+
|
|
83
|
+
for line in sys.stdin:
|
|
84
|
+
req = json.loads(line)
|
|
85
|
+
try:
|
|
86
|
+
if req["op"] == "add": # session: {id, date, turns, text}
|
|
87
|
+
s = req["session"]
|
|
88
|
+
store.add(s["text"], id=s["id"], date=s["date"])
|
|
89
|
+
out = {}
|
|
90
|
+
elif req["op"] == "recall": # question: the text
|
|
91
|
+
hits = store.search(req["question"], k=5)
|
|
92
|
+
out = {"memory": "\\n\\n".join(h.text for h in hits)}
|
|
93
|
+
else:
|
|
94
|
+
out = {"error": "unknown op " + req["op"]}
|
|
95
|
+
except Exception as e:
|
|
96
|
+
out = {"error": f"{type(e).__name__}: {e}"}
|
|
97
|
+
print(json.dumps(out), file=OUT, flush=True)
|
|
98
|
+
'''
|
|
99
|
+
|
|
100
|
+
TRY_FEATURE = """printf '%s\\n' \\
|
|
101
|
+
'{"op":"add","session":{"id":"s1","date":"2026-01-01","turns":[{"role":"user","text":"My sister moved to Lisbon."}],"text":"user: My sister moved to Lisbon."}}' \\
|
|
102
|
+
'{"op":"recall","question":"Where did my sister move?"}' \\
|
|
103
|
+
| python3 {wrapper}
|
|
104
|
+
# {}
|
|
105
|
+
# {"memory": "user: My sister moved to Lisbon."}"""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _json(d) -> str:
|
|
109
|
+
return json.dumps(d, indent=2)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def steps(kind: str) -> list:
|
|
113
|
+
agent = os.path.join(paths.catalogue_dir(), "agents", "my-agent", "agent.json")
|
|
114
|
+
capability = os.path.join(paths.catalogue_dir(), "capabilities", "my-memory")
|
|
115
|
+
definition = os.path.join(capability, "capability.json")
|
|
116
|
+
wrapper = os.path.join(capability, "capability.py")
|
|
117
|
+
if kind == "agent":
|
|
118
|
+
return [
|
|
119
|
+
{"title": "Create its folder and definition", "lang": "shell",
|
|
120
|
+
"text": "The key is lower case letters, digits and dashes; it names the folder in your catalogue.",
|
|
121
|
+
"code": f"{CLI} new agent my-agent\n# wrote {agent}"},
|
|
122
|
+
{"title": "Fill in agent.json", "lang": "json", "file": agent,
|
|
123
|
+
"text": "<b>install</b> pins the package (npm <code>name@version</code>, or pip <code>name==version</code>). "
|
|
124
|
+
"<b>start.acp</b> is the program that speaks ACP on stdin and stdout; <code>{install}</code> is where it was installed. "
|
|
125
|
+
"<b>model.env</b> tells it where its model is: the relay's address, a placeholder key and the run's model - "
|
|
126
|
+
"use the variable names the agent itself reads. Leave <b>offers</b> and <b>proven</b> empty: step 4 writes them.",
|
|
127
|
+
"code": _json(AGENT_JSON)},
|
|
128
|
+
{"title": "Check the definition", "lang": "shell", "text": "Every field is checked; a problem is named with its field.",
|
|
129
|
+
"code": f"{CLI} check\n# ok"},
|
|
130
|
+
{"title": "Prove its ports", "lang": "shell",
|
|
131
|
+
"text": "Runs the kit of every port it can take and writes what passed into agent.json. "
|
|
132
|
+
"<code>--unsandboxed</code> is your yes to running the agent on this machine as you. Or use <i>Prove its ports</i> on its page.",
|
|
133
|
+
"code": f"{CLI} conform my-agent --model gpt-5-mini --unsandboxed\n"
|
|
134
|
+
"# agent.task/v1: passed 3/3\n# agent.workspace/v1: passed 2/2\n# agent.tools/v1: passed 2/2\n# agent.conversation/v1: passed 2/2"},
|
|
135
|
+
{"title": "Make sure its calls went through the relay", "lang": "shell",
|
|
136
|
+
"text": "Each kit trial records the agent's model calls. An empty file means the agent called its vendor directly: "
|
|
137
|
+
"it ignored <code>model.env</code>, and needs the relay declared as a provider in its own config file "
|
|
138
|
+
"(<code>model.files</code>).",
|
|
139
|
+
"code": "cat jobs_standard/*__kit__agent-task-v1__my-agent/*/agent/calls.jsonl | head -3"},
|
|
140
|
+
{"title": "Its options, and its own command", "lang": "json", "file": agent,
|
|
141
|
+
"text": "<b>options</b> are what a run may set of the agent's behaviour (<code>--option name=value</code>), each with its "
|
|
142
|
+
"values and a <b>default</b> that is the vendor's own, written into its variables or config file as "
|
|
143
|
+
"<code>{opt_name}</code>; the job records every value. Declare the settings other agents declare "
|
|
144
|
+
"(<code>reasoning_effort</code>) when it has them. Write a setting explicitly when the adapter and the "
|
|
145
|
+
"agent's own command would each fill it in their own way. <b>fidelity.command</b> is the agent's own "
|
|
146
|
+
"non-interactive command, carrying <code>{prompt}</code>.",
|
|
147
|
+
"code": _json(AGENT_OPTIONS)},
|
|
148
|
+
{"title": "Show ACP sends what its own command sends", "lang": "shell",
|
|
149
|
+
"text": "Runs the workspace kit's two units over ACP and through <b>fidelity.command</b>, twice each, and compares "
|
|
150
|
+
"every field of the requests its model was sent; what differs between two runs of one way is noise. "
|
|
151
|
+
"The verdict is written into agent.json beside the kits' proofs.",
|
|
152
|
+
"code": f"{CLI} fidelity my-agent --model gpt-5-mini --unsandboxed\n# equal: 20 fields\n# verdict: same"},
|
|
153
|
+
{"title": "Run a benchmark with it", "lang": "shell",
|
|
154
|
+
"text": "Or from the Run builder: runner <i>Nullius</i>, a benchmark, then your agent on the Agent step.",
|
|
155
|
+
"code": f"{CLI} run BENCHMARK my-agent --model gpt-5-mini --limit 2 --unsandboxed"},
|
|
156
|
+
{"title": "A new release: add it as a version", "lang": "shell",
|
|
157
|
+
"text": "The older version stays, with its proofs; the new one is offered once its own kits pass. "
|
|
158
|
+
"Check the pins it was given (a package that moved changes its name too), then prove it. "
|
|
159
|
+
"Set <b>default</b> in agent.json to make it the one a bare <code>my-agent</code> names.",
|
|
160
|
+
"code": f"{CLI} new version my-agent 1.5.0\n# wrote {agent}: my-agent@1.5.0 beside 1.4.2\n"
|
|
161
|
+
f"{CLI} conform my-agent@1.5.0 --model gpt-5-mini --unsandboxed"},
|
|
162
|
+
]
|
|
163
|
+
return [
|
|
164
|
+
{"title": "Create its folder, definition and wrapper", "lang": "shell",
|
|
165
|
+
"text": "The key is lower case letters, digits and dashes; it names the folder in your catalogue.",
|
|
166
|
+
"code": f"{CLI} new capability my-memory\n# wrote {definition}\n# wrote {wrapper}"},
|
|
167
|
+
{"title": "Write capability.py around your memory", "lang": "python", "file": wrapper,
|
|
168
|
+
"text": "Nullius sends <code>add</code> once per session of the history, then <code>recall</code> with the question, "
|
|
169
|
+
"and gives what comes back to the benchmark's reader. Replace the three marked calls with your library's.",
|
|
170
|
+
"code": FEATURE_PY},
|
|
171
|
+
{"title": "Try it by hand", "lang": "shell", "text": "Two lines in, two lines out, before Nullius is involved (with your library installed).",
|
|
172
|
+
"code": TRY_FEATURE.replace("{wrapper}", shlex.quote(wrapper))},
|
|
173
|
+
{"title": "Fill in capability.json", "lang": "json", "file": definition,
|
|
174
|
+
"text": "<b>install.pip</b> pins its packages; <b>start.command</b> runs the wrapper in that environment. "
|
|
175
|
+
"Leave <b>offers</b> and <b>proven</b> empty: step 6 writes them.",
|
|
176
|
+
"code": _json(FEATURE_JSON)},
|
|
177
|
+
{"title": "Or: installed from its repository, calling a model", "lang": "json",
|
|
178
|
+
"text": "Replace <b>upstream</b> and <b>install</b> with these when the memory lives in a repository: it is cloned at that "
|
|
179
|
+
"commit and its Python project installed (CPU torch first, the embedder fetched once). A memory that calls a model "
|
|
180
|
+
"gets a <b>model</b> block with the variable names it reads: its calls then go through the relay, as an agent's.",
|
|
181
|
+
"code": _json(FEATURE_FROM_REPO)},
|
|
182
|
+
{"title": "Check, then prove its port", "lang": "shell", "text": "Or use <i>Prove its ports</i> on its page.",
|
|
183
|
+
"code": f"{CLI} check\n# ok\n{CLI} conform my-memory --model gpt-5-mini\n# cap.memory/v1: passed 2/2"},
|
|
184
|
+
{"title": "Run a capability benchmark with it", "lang": "shell",
|
|
185
|
+
"text": "Or from the Run builder: runner <i>Nullius</i>, a benchmark that takes its port, then your capability on "
|
|
186
|
+
"the step named after its kind (Memory for a memory).",
|
|
187
|
+
"code": f"{CLI} run BENCHMARK my-memory --model gpt-5-mini --limit 10"},
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# The prompt the Add dialog builds for a coding assistant - any of them, so it
|
|
192
|
+
# names a skill's file as well as its command. Three slots, filled on the page
|
|
193
|
+
# with what was typed: {name}, {source}, {remark} - what the assistant should
|
|
194
|
+
# know beyond what it reads from the source; the Remark line is left out when
|
|
195
|
+
# there is none (``static/catalogue/add.js``).
|
|
196
|
+
_KINDS = ("benchmark", "agent", "capability")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def prompt(kind: str) -> str:
|
|
200
|
+
if kind not in _KINDS:
|
|
201
|
+
raise ValueError(f"unknown catalogue kind: {kind}")
|
|
202
|
+
folder = "capabilities" if kind == "capability" else kind + "s"
|
|
203
|
+
target = os.path.join(paths.catalogue_dir(), folder, "<key>")
|
|
204
|
+
specific = {
|
|
205
|
+
"benchmark": "First ask me how it should run: Harbor or the Nullius standard (its Original setup). Recommend a route.\n",
|
|
206
|
+
"agent": "Ask before running unsandboxed. Verify native capabilities and supported extensions.\n",
|
|
207
|
+
"capability": "Verify its callable operations and proven ports.\n",
|
|
208
|
+
}[kind]
|
|
209
|
+
return (f'Add the {kind} "{{name}}" to Nullius.\nSource: {{source}}\nRemark: {{remark}}\n\n'
|
|
210
|
+
f"Follow /import-{kind}, or .claude/skills/import-{kind}/SKILL.md in the source checkout, "
|
|
211
|
+
"with docs/standard.md and AGENTS.md. If unavailable, ask for the checkout.\n"
|
|
212
|
+
f"Write to {target}, even when the skill names package paths. "
|
|
213
|
+
"For an existing bundled entry, copy its whole folder there first; never edit installed files. "
|
|
214
|
+
f"On another machine, use app/nullius/catalogue/{folder}/<key> in a checkout and tell me to update Nullius.\n"
|
|
215
|
+
"Use pinned upstream code; ask before implementing behavior of our own. "
|
|
216
|
+
f"{specific}Report what fits, what passed, and what is blocked or untested.")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def prompts() -> dict:
|
|
220
|
+
return {kind: prompt(kind) for kind in _KINDS}
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Where what a run needs is installed, and installing it once.
|
|
2
|
+
|
|
3
|
+
Everything goes under ``STANDARD_DIR`` (``standard/`` in the persistent user data directory):
|
|
4
|
+
|
|
5
|
+
- ``agents/<key>/<digest>/`` and ``capabilities/<key>/<digest>/``: the packages a
|
|
6
|
+
definition's ``install`` names (``npm`` into ``node_modules``, ``pip`` and
|
|
7
|
+
``wheels`` - a wheel by its URL, checked by its sha256 - into a ``uv`` environment), the digest taken over that list so a re-pin installs
|
|
8
|
+
beside the old one rather than over it;
|
|
9
|
+
- ``drivers/<key>/<digest>/``: a benchmark's ``driver.install`` the same way -
|
|
10
|
+
the environment its upstream's own code needs, which its driver imports
|
|
11
|
+
(``driver_argv``), never this application's and never an agent's. Its
|
|
12
|
+
``project`` is installed from the benchmark's upstream clone (below), the
|
|
13
|
+
same commit, rather than fetched a second time;
|
|
14
|
+
- ``upstreams/<key>/<commit>/``: a benchmark's upstream repository at its
|
|
15
|
+
tested commit;
|
|
16
|
+
- ``datasets/<sha256>/<file>``: a dataset served outside its upstream, under
|
|
17
|
+
``STANDARD_DATA_DIR`` instead when that is set - datasets can be gigabytes
|
|
18
|
+
where installs are not, so they may sit on a bigger disk than the installs.
|
|
19
|
+
|
|
20
|
+
A folder is complete when its ``.installed`` marker is; a failed install
|
|
21
|
+
leaves no marker and is tried again from scratch. Two runs installing the same
|
|
22
|
+
thing at once take turns (a lock file beside the folder): the second finds the
|
|
23
|
+
first one's marker rather than removing its half-made folder. A dataset file is checked
|
|
24
|
+
against its sha256 before a run reads it.
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import contextlib
|
|
29
|
+
import fcntl
|
|
30
|
+
import hashlib
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import re
|
|
34
|
+
import shutil
|
|
35
|
+
import subprocess
|
|
36
|
+
from typing import Callable, Dict, List, Optional
|
|
37
|
+
|
|
38
|
+
from nullius.standard import definitions
|
|
39
|
+
|
|
40
|
+
from nullius import paths
|
|
41
|
+
|
|
42
|
+
HOME = os.environ.get("STANDARD_DIR") or os.path.join(paths.data_dir(), "standard")
|
|
43
|
+
MARK = ".installed"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def data_dir(home: str = HOME, env=os.environ) -> str:
|
|
47
|
+
"""Where downloaded datasets are kept: ``STANDARD_DATA_DIR``, else
|
|
48
|
+
``datasets/`` in the standard's home. A dataset already downloaded
|
|
49
|
+
elsewhere is not moved: setting the variable downloads it once more."""
|
|
50
|
+
return env.get("STANDARD_DATA_DIR") or os.path.join(home, "datasets")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class InstallFailed(Exception):
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# A variable an install step is never given: a package's install script runs
|
|
58
|
+
# as published, and a run's process holds the model key (under --from-run as
|
|
59
|
+
# OPENAI_API_KEY, API_TOKEN, LLM_API_KEY) - the relay is the only holder meant.
|
|
60
|
+
SECRET = re.compile(r"(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?)$", re.I) # the last word: GIT_CONFIG_KEY_0 is git's own
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def install_env(env=None) -> dict:
|
|
64
|
+
"""This process's environment less every variable named like a secret."""
|
|
65
|
+
return {k: v for k, v in (os.environ if env is None else env).items() if not SECRET.search(k)}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def git_env() -> dict:
|
|
69
|
+
"""What a fetch of an upstream runs with: the installer's environment and
|
|
70
|
+
no prompt. A private repository - a fork of the user's - is fetched with
|
|
71
|
+
this machine's own git credentials (its credential helper, its ssh agent,
|
|
72
|
+
in its HOME); without them the fetch fails, said, rather than waiting on a
|
|
73
|
+
terminal for a password."""
|
|
74
|
+
return dict(install_env(), GIT_TERMINAL_PROMPT="0")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _run(cmd: List[str], cwd: str, log: str, env: Optional[dict] = None):
|
|
78
|
+
with open(log, "a") as f:
|
|
79
|
+
f.write("$ " + " ".join(cmd) + "\n")
|
|
80
|
+
f.flush()
|
|
81
|
+
try:
|
|
82
|
+
r = subprocess.run(cmd, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, env=install_env() if env is None else env)
|
|
83
|
+
except FileNotFoundError:
|
|
84
|
+
raise InstallFailed(f"{cmd[0]} is not on this machine")
|
|
85
|
+
if r.returncode != 0:
|
|
86
|
+
with open(log, errors="replace") as f:
|
|
87
|
+
tail = f.read().strip().splitlines()[-3:]
|
|
88
|
+
raise InstallFailed(f"{' '.join(cmd[:3])}...: exit {r.returncode}: {' | '.join(tail)[-300:]}")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@contextlib.contextmanager
|
|
92
|
+
def _locked(where: str):
|
|
93
|
+
os.makedirs(os.path.dirname(where), exist_ok=True)
|
|
94
|
+
with open(where + ".lock", "w") as f:
|
|
95
|
+
fcntl.flock(f, fcntl.LOCK_EX)
|
|
96
|
+
try:
|
|
97
|
+
yield
|
|
98
|
+
finally:
|
|
99
|
+
fcntl.flock(f, fcntl.LOCK_UN)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _digest(obj) -> str:
|
|
103
|
+
return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest()[:12]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def packages(d: dict, home: str = HOME, npm_in: Optional[Callable[[List[str], str], List[str]]] = None,
|
|
107
|
+
src: Optional[str] = None) -> str:
|
|
108
|
+
"""The folder an agent's or a capability's packages are installed in,
|
|
109
|
+
installing them first when they are not. A definition installing nothing
|
|
110
|
+
gets an empty folder all the same, so ``{install}`` always names one.
|
|
111
|
+
|
|
112
|
+
``npm_in``: how to run ``npm`` where this machine has none - the docker
|
|
113
|
+
backend's base image, which carries Node (``container.install_argv``):
|
|
114
|
+
given the command and the folder, the argument list that runs it there.
|
|
115
|
+
A host is not asked for Node to run an agent it will only start in a
|
|
116
|
+
container. Without it, a machine with no ``npm`` is refused, said.
|
|
117
|
+
|
|
118
|
+
``src``: a checkout of the definition's upstream at ``upstream.tested``
|
|
119
|
+
already on this machine, which ``project`` is installed from instead of
|
|
120
|
+
a clone of its own (a benchmark's driver: its upstream clone)."""
|
|
121
|
+
inst = d.get("install") or {}
|
|
122
|
+
up = (d.get("upstream") or {}) if inst.get("project") else {}
|
|
123
|
+
where = packages_dir(d, home)
|
|
124
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
125
|
+
return where
|
|
126
|
+
with _locked(where):
|
|
127
|
+
return _packages(inst, where, up, npm_in, src)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def packages_dir(d: dict, home: str = HOME) -> str:
|
|
131
|
+
"""Where ``packages`` installs a definition's packages, installed or not."""
|
|
132
|
+
inst = d.get("install") or {}
|
|
133
|
+
kind = {"agent": "agents", "driver": "drivers"}.get(d.get("_kind"), "capabilities")
|
|
134
|
+
up = (d.get("upstream") or {}) if inst.get("project") else {}
|
|
135
|
+
return os.path.join(home, kind, d["key"], _digest(dict(inst, _at=up.get("tested")) if up else inst))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def ready(where: Optional[str]) -> bool:
|
|
139
|
+
"""Whether an install folder is complete (its marker is there): a run says
|
|
140
|
+
the stage was there already rather than that it took no time."""
|
|
141
|
+
return bool(where) and os.path.exists(os.path.join(where, MARK))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
HF = "hf" # inside an install: the models prefetched from the hub (HF_HUB_CACHE at run time)
|
|
145
|
+
WHEELS = "wheels" # inside an install: the wheels it names by URL, as downloaded
|
|
146
|
+
SRC = "src" # inside an install: the definition's upstream, at its tested commit, for install.project
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _packages(inst: dict, where: str, up: Optional[dict] = None, npm_in: Optional[Callable] = None,
|
|
150
|
+
src: Optional[str] = None) -> str:
|
|
151
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
152
|
+
return where # another run finished it while this one waited
|
|
153
|
+
if os.path.isdir(where):
|
|
154
|
+
shutil.rmtree(where)
|
|
155
|
+
os.makedirs(where)
|
|
156
|
+
log = os.path.join(where, "install.log")
|
|
157
|
+
if inst.get("npm"):
|
|
158
|
+
cmd = ["npm", "install", "--no-audit", "--no-fund", "--prefix", where] + list(inst["npm"])
|
|
159
|
+
if shutil.which("npm"):
|
|
160
|
+
_run(cmd, where, log)
|
|
161
|
+
elif npm_in is None:
|
|
162
|
+
raise InstallFailed("npm is not on this machine: install Node.js 22 here, or run on the docker backend, "
|
|
163
|
+
"which installs with the Node in its base image")
|
|
164
|
+
else:
|
|
165
|
+
_run(npm_in(cmd, where), where, log)
|
|
166
|
+
for scratch in (".npm-home", ".npm-cache"): # what npm kept of its own, in the container
|
|
167
|
+
shutil.rmtree(os.path.join(where, scratch), ignore_errors=True)
|
|
168
|
+
if inst.get("pip") or inst.get("project") or inst.get("indexes") or inst.get("wheels"):
|
|
169
|
+
py = os.path.join(where, ".venv", "bin", "python")
|
|
170
|
+
# uv's own interpreter, never the system's: an own image mounts the one the
|
|
171
|
+
# environment links to (container.python_homes), and /usr/bin is the image's
|
|
172
|
+
_run(["uv", "venv", "--quiet", "--python-preference", "only-managed", os.path.join(where, ".venv")]
|
|
173
|
+
+ (["--python", inst["python"]] if inst.get("python") else []), where, log)
|
|
174
|
+
for x in inst.get("indexes") or []: # first: the project's resolution then keeps them
|
|
175
|
+
_run(["uv", "pip", "install", "--quiet", "--python", py, "--index-url", x["url"]] + list(x["packages"]), where, log)
|
|
176
|
+
wheels = [] # each checked by its sha256 before it is installed
|
|
177
|
+
for x in inst.get("wheels") or []:
|
|
178
|
+
os.makedirs(os.path.join(where, WHEELS), exist_ok=True)
|
|
179
|
+
wheels.append(os.path.join(where, WHEELS, x["url"].rsplit("/", 1)[1]))
|
|
180
|
+
_fetch(x["url"], x["sha256"], wheels[-1], "wheel")
|
|
181
|
+
locked = bool(inst.get("project") and inst.get("locked"))
|
|
182
|
+
if (inst.get("pip") or wheels) and not locked:
|
|
183
|
+
# resolved together: a wheel's dependencies are pip's pins
|
|
184
|
+
_run(["uv", "pip", "install", "--quiet", "--python", py] + list(inst.get("pip") or []) + wheels, where, log)
|
|
185
|
+
if inst.get("project"):
|
|
186
|
+
if src is None:
|
|
187
|
+
src = os.path.join(where, SRC)
|
|
188
|
+
os.makedirs(src)
|
|
189
|
+
for cmd in (["git", "init", "--quiet"], ["git", "fetch", "--quiet", "--depth", "1", up["repository"], up["tested"]],
|
|
190
|
+
["git", "checkout", "--quiet", "FETCH_HEAD"]):
|
|
191
|
+
_run(cmd, src, log, env=git_env())
|
|
192
|
+
proj = os.path.normpath(os.path.join(src, inst["project"]))
|
|
193
|
+
extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else ""
|
|
194
|
+
if locked:
|
|
195
|
+
# the project's own lock file, its pins as published, and pip's packages
|
|
196
|
+
# resolved beside them in one go: a conflict fails the install, said
|
|
197
|
+
lock = os.path.join(where, "requirements.lock")
|
|
198
|
+
_run(["uv", "export", "--frozen", "--no-dev", "--no-emit-project", "--no-hashes", "--quiet",
|
|
199
|
+
"--project", proj, "-o", lock] + [w for x in inst.get("extras") or [] for w in ("--extra", x)], where, log)
|
|
200
|
+
_run(["uv", "pip", "install", "--quiet", "--python", py, "-r", lock] + list(inst.get("pip") or []) + wheels, where, log)
|
|
201
|
+
_run(["uv", "pip", "install", "--quiet", "--python", py, "--no-deps", "-e", proj + extras], where, log)
|
|
202
|
+
else:
|
|
203
|
+
_run(["uv", "pip", "install", "--quiet", "--python", py, "-e", proj + extras], where, log)
|
|
204
|
+
if inst.get("prefetch"):
|
|
205
|
+
env = dict(install_env(), HF_HUB_CACHE=os.path.join(where, HF))
|
|
206
|
+
_run([py, "-c", "import sys\nfrom huggingface_hub import snapshot_download\nfor m in sys.argv[1:]: snapshot_download(m)"]
|
|
207
|
+
+ list(inst["prefetch"]), where, log, env=env)
|
|
208
|
+
with open(os.path.join(where, MARK), "w") as f:
|
|
209
|
+
json.dump(inst, f)
|
|
210
|
+
return where
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def process_env(d: dict, where: str, home: str, options: Optional[dict] = None) -> dict:
|
|
214
|
+
"""What an install adds to its process's environment: the definition's own
|
|
215
|
+
``env`` rendered (``{opt_<name>}`` from ``options``, the run's values), and
|
|
216
|
+
``HF_HUB_CACHE`` when it prefetched models."""
|
|
217
|
+
slots = {"install": where, "home": home, "here": d.get("_here", ""),
|
|
218
|
+
**{"opt_" + k: v for k, v in (options or {}).items()}}
|
|
219
|
+
out = {}
|
|
220
|
+
if (d.get("install") or {}).get("prefetch"):
|
|
221
|
+
out["HF_HUB_CACHE"] = os.path.join(where, HF)
|
|
222
|
+
for k, v in (d.get("env") or {}).items():
|
|
223
|
+
for s, val in slots.items():
|
|
224
|
+
v = v.replace("{" + s + "}", val)
|
|
225
|
+
out[k] = v
|
|
226
|
+
return out
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def prune(d: dict, home: str) -> List[str]:
|
|
230
|
+
"""Remove what the definition declares as ``caches`` from a unit's home,
|
|
231
|
+
once the unit has ended and its trial is written: the paths it removed.
|
|
232
|
+
A path is resolved under ``home`` and refused when it would leave it
|
|
233
|
+
(a symlink the agent made included); the definition's check already
|
|
234
|
+
holds them to ``{home}/...``."""
|
|
235
|
+
root, gone = os.path.realpath(home), []
|
|
236
|
+
for c in d.get("caches") or []:
|
|
237
|
+
if not isinstance(c, str) or not c.startswith("{home}/"):
|
|
238
|
+
continue
|
|
239
|
+
path = os.path.realpath(os.path.join(home, c[len("{home}/"):]))
|
|
240
|
+
if not path.startswith(root + os.sep) or not os.path.lexists(path):
|
|
241
|
+
continue
|
|
242
|
+
if os.path.isdir(path):
|
|
243
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
244
|
+
else:
|
|
245
|
+
with contextlib.suppress(OSError):
|
|
246
|
+
os.remove(path)
|
|
247
|
+
gone.append(path)
|
|
248
|
+
return gone
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def bin_dirs(where: str) -> List[str]:
|
|
252
|
+
"""The folders of an install whose programs a start command may name."""
|
|
253
|
+
return [p for p in (os.path.join(where, "node_modules", ".bin"), os.path.join(where, ".venv", "bin")) if os.path.isdir(p)]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def driver_packages(bench: dict, home: str = HOME) -> Optional[str]:
|
|
257
|
+
"""The folder a benchmark's ``driver.install`` is installed in, installing
|
|
258
|
+
it first when it is not (once per digest, as an agent's packages); None
|
|
259
|
+
for a driver that declares none and runs on what the machine has. Its
|
|
260
|
+
``project`` comes from the benchmark's upstream clone, cloned first when
|
|
261
|
+
it is not there: the same commit, fetched once."""
|
|
262
|
+
d = definitions.driver_install(bench)
|
|
263
|
+
if not d["install"]:
|
|
264
|
+
return None
|
|
265
|
+
src = upstream(bench, home) if d["install"].get("project") else None
|
|
266
|
+
return packages(d, home, src=src)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def driver_dir(bench: dict, home: str = HOME) -> Optional[str]:
|
|
270
|
+
"""Where ``driver_packages`` installs, installed or not; None when it declares nothing."""
|
|
271
|
+
d = definitions.driver_install(bench)
|
|
272
|
+
return packages_dir(d, home) if d["install"] else None
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def driver_argv(bench: dict, home: str = HOME) -> tuple:
|
|
276
|
+
"""How a benchmark's driver is started: its ``driver.command`` with
|
|
277
|
+
``{here}`` (the definition's folder) and ``{install}`` (its
|
|
278
|
+
``driver.install`` folder, installed first when it is not) filled, and
|
|
279
|
+
what that install adds to the driver's environment (``HF_HUB_CACHE``
|
|
280
|
+
when it prefetched models). Every place that starts a driver - a run, a
|
|
281
|
+
shard, a proof's checks - starts it here."""
|
|
282
|
+
where = driver_packages(bench, home)
|
|
283
|
+
extra = process_env(definitions.driver_install(bench), where, home) if where else {}
|
|
284
|
+
return words(bench["driver"]["command"], {"here": bench["_here"], "install": where or ""}), extra
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def upstream(bench: dict, home: str = HOME) -> Optional[str]:
|
|
288
|
+
"""A benchmark's upstream at its tested commit, cloned once; None when the
|
|
289
|
+
definition has no upstream repository (its data sits beside it)."""
|
|
290
|
+
u = bench.get("upstream") or {}
|
|
291
|
+
where = upstream_dir(bench, home)
|
|
292
|
+
if where is None:
|
|
293
|
+
return None
|
|
294
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
295
|
+
return where
|
|
296
|
+
with _locked(where):
|
|
297
|
+
return _upstream(u, where)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def upstream_dir(bench: dict, home: str = HOME) -> Optional[str]:
|
|
301
|
+
"""Where ``upstream`` clones a benchmark's upstream; None when it has none."""
|
|
302
|
+
u = bench.get("upstream") or {}
|
|
303
|
+
return os.path.join(home, "upstreams", bench["key"], u["tested"]) if u.get("repository") else None
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _upstream(u: dict, where: str) -> str:
|
|
307
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
308
|
+
return where
|
|
309
|
+
if os.path.isdir(where):
|
|
310
|
+
shutil.rmtree(where)
|
|
311
|
+
os.makedirs(where)
|
|
312
|
+
log = os.path.join(os.path.dirname(where), u["tested"] + ".log")
|
|
313
|
+
_run(["git", "init", "--quiet"], where, log)
|
|
314
|
+
_run(["git", "fetch", "--quiet", "--depth", "1", u["repository"], u["tested"]], where, log, env=git_env())
|
|
315
|
+
_run(["git", "checkout", "--quiet", "FETCH_HEAD"], where, log)
|
|
316
|
+
open(os.path.join(where, MARK), "w").close()
|
|
317
|
+
return where
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def sha256_of(path: str) -> str:
|
|
321
|
+
h = hashlib.sha256()
|
|
322
|
+
with open(path, "rb") as f:
|
|
323
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
324
|
+
h.update(chunk)
|
|
325
|
+
return h.hexdigest()
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def dataset(bench: dict, ds: dict, up: Optional[str], progress: Optional[Callable] = None) -> Optional[str]:
|
|
329
|
+
"""The dataset's file, checked by its sha256: in the upstream when there
|
|
330
|
+
is one, else beside the definition. None for a dataset with no file (its
|
|
331
|
+
driver makes its units). ``progress``: told a download's bytes as they
|
|
332
|
+
come (``download``)."""
|
|
333
|
+
if ds.get("url"):
|
|
334
|
+
return download(ds["url"], ds["sha256"], progress=progress)
|
|
335
|
+
if not ds.get("path"):
|
|
336
|
+
return None
|
|
337
|
+
path = os.path.join(up or bench["_here"], ds["path"])
|
|
338
|
+
if not os.path.isfile(path):
|
|
339
|
+
raise InstallFailed(f"dataset {ds['id']}: {ds['path']} is not in {'the upstream' if up else 'the definition folder'}")
|
|
340
|
+
got = sha256_of(path)
|
|
341
|
+
if got != ds["sha256"]:
|
|
342
|
+
raise InstallFailed(f"dataset {ds['id']}: sha256 {got[:12]}..., the definition says {ds['sha256'][:12]}...")
|
|
343
|
+
return path
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def download_path(url: str, sha256: str, home: str = HOME) -> str:
|
|
347
|
+
"""Where ``download`` keeps a dataset, downloaded or not."""
|
|
348
|
+
return os.path.join(data_dir(home), sha256, os.path.basename(url.split("?", 1)[0]) or "data")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def download(url: str, sha256: str, home: str = HOME, progress: Optional[Callable] = None) -> str:
|
|
352
|
+
"""A dataset served outside its upstream, downloaded once under its
|
|
353
|
+
digest (``data_dir``), and checked by it. ``progress(done, total)`` is
|
|
354
|
+
told the bytes so far after each chunk (``total`` from the server's
|
|
355
|
+
length, None when it gives none), and once more with ``final=True``."""
|
|
356
|
+
where = download_path(url, sha256, home)
|
|
357
|
+
if os.path.isfile(where):
|
|
358
|
+
return where
|
|
359
|
+
with _locked(where):
|
|
360
|
+
if not os.path.isfile(where):
|
|
361
|
+
_fetch(url, sha256, where, "dataset", progress)
|
|
362
|
+
return where
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _fetch(url: str, sha256: str, where: str, what: str, progress: Optional[Callable] = None) -> None:
|
|
366
|
+
"""``url`` downloaded to ``where``, kept only when it is the file ``sha256`` names."""
|
|
367
|
+
import urllib.request
|
|
368
|
+
part = where + ".part"
|
|
369
|
+
try:
|
|
370
|
+
with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "nullius"}), timeout=60) as r, open(part, "wb") as f:
|
|
371
|
+
total = int(r.headers.get("Content-Length") or 0) or None
|
|
372
|
+
done = 0
|
|
373
|
+
for chunk in iter(lambda: r.read(1 << 20), b""):
|
|
374
|
+
f.write(chunk)
|
|
375
|
+
done += len(chunk)
|
|
376
|
+
if progress:
|
|
377
|
+
progress(done, total)
|
|
378
|
+
if progress:
|
|
379
|
+
progress(done, total, final=True)
|
|
380
|
+
except OSError as e:
|
|
381
|
+
raise InstallFailed(f"{what} {url}: {e}")
|
|
382
|
+
got = sha256_of(part)
|
|
383
|
+
if got != sha256:
|
|
384
|
+
os.remove(part)
|
|
385
|
+
raise InstallFailed(f"{what} {url}: sha256 {got[:12]}..., the definition says {sha256[:12]}...")
|
|
386
|
+
os.replace(part, where)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def words(cmd: List[str], slots: Dict[str, str]) -> List[str]:
|
|
390
|
+
"""A definition's command with its slots filled: each slot is replaced
|
|
391
|
+
inside its word, and the word stays one argument - no shell reads it."""
|
|
392
|
+
out = []
|
|
393
|
+
for w in cmd:
|
|
394
|
+
for k, v in slots.items():
|
|
395
|
+
w = w.replace("{" + k + "}", v)
|
|
396
|
+
out.append(w)
|
|
397
|
+
return out
|