nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""A benchmark's host requirements (``host`` in its ``standard.json``): what
|
|
2
|
+
the machine its driver runs on must have for some of its options - programs on
|
|
3
|
+
PATH, installed by ``apt`` and ``npm``. Drivers for agent/capability runs stay
|
|
4
|
+
on the machine; Docker Original setups install those with an ``install`` in
|
|
5
|
+
their image and are refused the rest (a GPU).
|
|
6
|
+
|
|
7
|
+
Each requirement is checked by its programs and optional GPU memory minimum
|
|
8
|
+
(:func:`problems`) and installed by the commands
|
|
9
|
+
:func:`install_lines` gives: shown to a person to run, or run for them on an
|
|
10
|
+
explicit choice (``web/execute.py``), with ``sudo -n`` so a password is never
|
|
11
|
+
asked for - a machine where sudo needs one is told the commands instead.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import shlex
|
|
16
|
+
import shutil
|
|
17
|
+
import subprocess
|
|
18
|
+
from typing import Dict, List, Optional
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def requirements(bench: dict, names: List[str]) -> Dict[str, dict]:
|
|
22
|
+
"""The named requirements, as the benchmark declares them."""
|
|
23
|
+
h = bench.get("host") or {}
|
|
24
|
+
return {n: h[n] for n in names if n in h}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def in_image(bench: dict, names: List[str]) -> List[str]:
|
|
28
|
+
"""The named requirements a Docker Original setup's image installs (its
|
|
29
|
+
``install``); the rest, a GPU above all, no image can provide."""
|
|
30
|
+
return [n for n, r in requirements(bench, names).items() if r.get("install")]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def missing(bench: dict, names: List[str], which=shutil.which) -> Dict[str, List[str]]:
|
|
34
|
+
"""Each named requirement this machine lacks, with the programs it lacks."""
|
|
35
|
+
out = {}
|
|
36
|
+
for n, r in requirements(bench, names).items():
|
|
37
|
+
gone = [p for p in r.get("check") or [] if not which(p)]
|
|
38
|
+
if gone:
|
|
39
|
+
out[n] = gone
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def gpu_memory_mb() -> Optional[int]:
|
|
44
|
+
"""The largest GPU's total memory in MiB, or None when it cannot be read.
|
|
45
|
+
Never add cards together or treat an unavailable query as zero memory."""
|
|
46
|
+
try:
|
|
47
|
+
result = subprocess.run(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
|
|
48
|
+
capture_output=True, text=True, timeout=5)
|
|
49
|
+
values = [line.strip() for line in result.stdout.splitlines()]
|
|
50
|
+
if result.returncode == 0 and values and all(v.isascii() and v.isdecimal() and int(v) > 0 for v in values):
|
|
51
|
+
return max(map(int, values))
|
|
52
|
+
except (OSError, subprocess.TimeoutExpired, UnicodeError):
|
|
53
|
+
pass
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def problems(bench: dict, names: List[str], which=None) -> Dict[str, str]:
|
|
58
|
+
"""Why each selected requirement is missing on this machine, shared by
|
|
59
|
+
the builder and run startup. Query GPU memory only when needed, once."""
|
|
60
|
+
which = which or shutil.which
|
|
61
|
+
out = {n: ", ".join(ps) + " not on PATH" for n, ps in missing(bench, names, which).items()}
|
|
62
|
+
gpu = {n: r["gpu_memory_mb"] for n, r in requirements(bench, names).items()
|
|
63
|
+
if n not in out and "gpu_memory_mb" in r}
|
|
64
|
+
if gpu:
|
|
65
|
+
available = gpu_memory_mb() if which("nvidia-smi") else None
|
|
66
|
+
for n, needed in gpu.items():
|
|
67
|
+
if available is None:
|
|
68
|
+
out[n] = "nvidia-smi could not report GPU memory" if which("nvidia-smi") else "nvidia-smi not on PATH"
|
|
69
|
+
elif available < needed:
|
|
70
|
+
out[n] = f"needs at least {needed} MiB of GPU memory; largest GPU has {available} MiB"
|
|
71
|
+
return out
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def install_lines(bench: dict, names: List[str], sudo: str = "sudo") -> List[str]:
|
|
75
|
+
"""The commands that install the named requirements: apt first (with ``sudo``), then npm."""
|
|
76
|
+
apt, npm = [], []
|
|
77
|
+
for r in requirements(bench, names).values():
|
|
78
|
+
apt += [p for p in (r.get("install") or {}).get("apt") or [] if p not in apt]
|
|
79
|
+
npm += [p for p in (r.get("install") or {}).get("npm") or [] if p not in npm]
|
|
80
|
+
lines = []
|
|
81
|
+
if apt:
|
|
82
|
+
lines.append(f"{sudo} apt-get install -y " + " ".join(shlex.quote(p) for p in apt))
|
|
83
|
+
if npm:
|
|
84
|
+
lines.append("npm install -g " + " ".join(shlex.quote(p) for p in npm))
|
|
85
|
+
return lines
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def install_script(bench: dict, names: List[str]) -> Optional[str]:
|
|
89
|
+
"""The install as one script for a person's explicit yes: ``sudo -n`` (never a password prompt),
|
|
90
|
+
stopping at the first command that fails; None when there is nothing to install."""
|
|
91
|
+
lines = install_lines(bench, names, sudo="sudo -n")
|
|
92
|
+
if not lines:
|
|
93
|
+
return None
|
|
94
|
+
return "set -e\n" + "\n".join(lines) + "\n"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def all_of(bench: dict) -> Dict[str, dict]:
|
|
98
|
+
"""Every host requirement of the benchmark, each with the option values that need it."""
|
|
99
|
+
out = {n: dict(r, needed_by=[]) for n, r in (bench.get("host") or {}).items()}
|
|
100
|
+
for name, o in (bench.get("options") or {}).items():
|
|
101
|
+
for v, need in (o.get("needs") or {}).items():
|
|
102
|
+
for h in need.get("host") or []:
|
|
103
|
+
if h in out:
|
|
104
|
+
out[h]["needed_by"].append(f"{name}={v}")
|
|
105
|
+
return out
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""How to add an agent or a capability, as the Agents and Capabilities tabs show it:
|
|
2
|
+
the commands to type and the files to write, in order.
|
|
3
|
+
|
|
4
|
+
Each step is ``{title, text, code, lang, file}``: ``code`` is what is typed or
|
|
5
|
+
written, ``file`` where it goes. The definitions shown are real: the suite
|
|
6
|
+
checks every ``agent.json`` and ``capability.json`` here with
|
|
7
|
+
``definitions.problems``, so the page never shows one that would be refused.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import shlex
|
|
14
|
+
|
|
15
|
+
from nullius import paths
|
|
16
|
+
|
|
17
|
+
CLI = "nullius standard"
|
|
18
|
+
|
|
19
|
+
AGENT_JSON = {
|
|
20
|
+
"schema": "nullius.agent/v1",
|
|
21
|
+
"key": "my-agent",
|
|
22
|
+
"label": "My agent",
|
|
23
|
+
"summary": "What the agent is, and how it is driven: its ACP adapter, its model through the relay.",
|
|
24
|
+
"upstream": {"package": "@acme/my-agent-acp", "version": "1.4.2", "license": "MIT"},
|
|
25
|
+
"install": {"npm": ["@acme/my-agent-acp@1.4.2"]},
|
|
26
|
+
"start": {"acp": ["{install}/node_modules/.bin/my-agent-acp"]},
|
|
27
|
+
"model": {
|
|
28
|
+
"apis": ["chat"],
|
|
29
|
+
"env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "OPENAI_MODEL": "{model}"},
|
|
30
|
+
},
|
|
31
|
+
"offers": [],
|
|
32
|
+
"proven": {},
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
# What an agent may add: an option a run can set (its default the vendor's own,
|
|
36
|
+
# filled into a template as {opt_<name>}), and its own non-interactive command,
|
|
37
|
+
# for cli fidelity to compare with ACP.
|
|
38
|
+
AGENT_OPTIONS = {
|
|
39
|
+
"model": {
|
|
40
|
+
"apis": ["chat"],
|
|
41
|
+
"env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "OPENAI_MODEL": "{model}",
|
|
42
|
+
"MY_AGENT_REASONING": "{opt_reasoning_effort}"},
|
|
43
|
+
},
|
|
44
|
+
"options": {"reasoning_effort": {
|
|
45
|
+
"summary": "How hard the model reasons. The default is the agent's own, as its documentation or its source says.",
|
|
46
|
+
"values": ["low", "medium", "high"], "default": "medium"}},
|
|
47
|
+
"fidelity": {"command": ["{install}/node_modules/.bin/my-agent", "run", "--yes", "--", "{prompt}"]},
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
FEATURE_JSON = {
|
|
51
|
+
"schema": "nullius.capability/v1",
|
|
52
|
+
"key": "my-memory",
|
|
53
|
+
"label": "My memory",
|
|
54
|
+
"summary": "What the memory keeps, and what it gives back for a question.",
|
|
55
|
+
"upstream": {"package": "my-memory", "version": "0.3.1", "license": "MIT"},
|
|
56
|
+
"kind": "memory",
|
|
57
|
+
"install": {"pip": ["my-memory==0.3.1"]},
|
|
58
|
+
"start": {"command": ["{install}/.venv/bin/python", "{here}/capability.py"]},
|
|
59
|
+
"offers": [],
|
|
60
|
+
"proven": {},
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
# The same memory installed from its own repository, calling a model: the parts that change.
|
|
64
|
+
FEATURE_FROM_REPO = {
|
|
65
|
+
"upstream": {"repository": "https://github.com/acme/my-memory", "tested": "0123456789abcdef0123456789abcdef01234567",
|
|
66
|
+
"license": "MIT"},
|
|
67
|
+
"install": {"python": "3.12", "project": "packages/memory", "extras": ["embeddings"],
|
|
68
|
+
"indexes": [{"url": "https://download.pytorch.org/whl/cpu", "packages": ["torch"]}],
|
|
69
|
+
"prefetch": ["BAAI/bge-small-en-v1.5"]},
|
|
70
|
+
"env": {"HF_HUB_OFFLINE": "1"},
|
|
71
|
+
"model": {"apis": ["chat"], "env": {"API_BASE": "{base_url}", "API_KEY": "{key}", "MODEL_NAME": "{model}"}},
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
FEATURE_PY = '''"""My memory behind cap.memory/v1: one JSON line in, one JSON line out."""
|
|
75
|
+
import json
|
|
76
|
+
import sys
|
|
77
|
+
|
|
78
|
+
from my_memory import Store # your library
|
|
79
|
+
|
|
80
|
+
OUT, sys.stdout = sys.stdout, sys.stderr # stdout: the replies only
|
|
81
|
+
store = Store() # one process per question
|
|
82
|
+
|
|
83
|
+
for line in sys.stdin:
|
|
84
|
+
req = json.loads(line)
|
|
85
|
+
try:
|
|
86
|
+
if req["op"] == "add": # session: {id, date, turns, text}
|
|
87
|
+
s = req["session"]
|
|
88
|
+
store.add(s["text"], id=s["id"], date=s["date"])
|
|
89
|
+
out = {}
|
|
90
|
+
elif req["op"] == "recall": # question: the text
|
|
91
|
+
hits = store.search(req["question"], k=5)
|
|
92
|
+
out = {"memory": "\\n\\n".join(h.text for h in hits)}
|
|
93
|
+
else:
|
|
94
|
+
out = {"error": "unknown op " + req["op"]}
|
|
95
|
+
except Exception as e:
|
|
96
|
+
out = {"error": f"{type(e).__name__}: {e}"}
|
|
97
|
+
print(json.dumps(out), file=OUT, flush=True)
|
|
98
|
+
'''
|
|
99
|
+
|
|
100
|
+
TRY_FEATURE = """printf '%s\\n' \\
|
|
101
|
+
'{"op":"add","session":{"id":"s1","date":"2026-01-01","turns":[{"role":"user","text":"My sister moved to Lisbon."}],"text":"user: My sister moved to Lisbon."}}' \\
|
|
102
|
+
'{"op":"recall","question":"Where did my sister move?"}' \\
|
|
103
|
+
| python3 {wrapper}
|
|
104
|
+
# {}
|
|
105
|
+
# {"memory": "user: My sister moved to Lisbon."}"""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _json(d) -> str:
|
|
109
|
+
return json.dumps(d, indent=2)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def steps(kind: str) -> list:
|
|
113
|
+
agent = os.path.join(paths.catalogue_dir(), "agents", "my-agent", "agent.json")
|
|
114
|
+
capability = os.path.join(paths.catalogue_dir(), "capabilities", "my-memory")
|
|
115
|
+
definition = os.path.join(capability, "capability.json")
|
|
116
|
+
wrapper = os.path.join(capability, "capability.py")
|
|
117
|
+
if kind == "agent":
|
|
118
|
+
return [
|
|
119
|
+
{"title": "Create its folder and definition", "lang": "shell",
|
|
120
|
+
"text": "The key is lower case letters, digits and dashes; it names the folder in your catalogue.",
|
|
121
|
+
"code": f"{CLI} new agent my-agent\n# wrote {agent}"},
|
|
122
|
+
{"title": "Fill in agent.json", "lang": "json", "file": agent,
|
|
123
|
+
"text": "<b>install</b> pins the package (npm <code>name@version</code>, or pip <code>name==version</code>). "
|
|
124
|
+
"<b>start.acp</b> is the program that speaks ACP on stdin and stdout; <code>{install}</code> is where it was installed. "
|
|
125
|
+
"<b>model.env</b> tells it where its model is: the relay's address, a placeholder key and the run's model - "
|
|
126
|
+
"use the variable names the agent itself reads. Leave <b>offers</b> and <b>proven</b> empty: step 4 writes them.",
|
|
127
|
+
"code": _json(AGENT_JSON)},
|
|
128
|
+
{"title": "Check the definition", "lang": "shell", "text": "Every field is checked; a problem is named with its field.",
|
|
129
|
+
"code": f"{CLI} check\n# ok"},
|
|
130
|
+
{"title": "Prove its ports", "lang": "shell",
|
|
131
|
+
"text": "Runs the kit of every port it can take and writes what passed into agent.json. "
|
|
132
|
+
"<code>--unsandboxed</code> is your yes to running the agent on this machine as you. Or use <i>Prove its ports</i> on its page.",
|
|
133
|
+
"code": f"{CLI} conform my-agent --model gpt-5-mini --unsandboxed\n"
|
|
134
|
+
"# agent.task/v1: passed 3/3\n# agent.workspace/v1: passed 2/2\n# agent.tools/v1: passed 2/2\n# agent.conversation/v1: passed 2/2"},
|
|
135
|
+
{"title": "Make sure its calls went through the relay", "lang": "shell",
|
|
136
|
+
"text": "Each kit trial records the agent's model calls. An empty file means the agent called its vendor directly: "
|
|
137
|
+
"it ignored <code>model.env</code>, and needs the relay declared as a provider in its own config file "
|
|
138
|
+
"(<code>model.files</code>).",
|
|
139
|
+
"code": "cat jobs_standard/*__kit__agent-task-v1__my-agent/*/agent/calls.jsonl | head -3"},
|
|
140
|
+
{"title": "Its options, and its own command", "lang": "json", "file": agent,
|
|
141
|
+
"text": "<b>options</b> are what a run may set of the agent's behaviour (<code>--option name=value</code>), each with its "
|
|
142
|
+
"values and a <b>default</b> that is the vendor's own, written into its variables or config file as "
|
|
143
|
+
"<code>{opt_name}</code>; the job records every value. Declare the settings other agents declare "
|
|
144
|
+
"(<code>reasoning_effort</code>) when it has them. Write a setting explicitly when the adapter and the "
|
|
145
|
+
"agent's own command would each fill it in their own way. <b>fidelity.command</b> is the agent's own "
|
|
146
|
+
"non-interactive command, carrying <code>{prompt}</code>.",
|
|
147
|
+
"code": _json(AGENT_OPTIONS)},
|
|
148
|
+
{"title": "Show ACP sends what its own command sends", "lang": "shell",
|
|
149
|
+
"text": "Runs the workspace kit's two units over ACP and through <b>fidelity.command</b>, twice each, and compares "
|
|
150
|
+
"every field of the requests its model was sent; what differs between two runs of one way is noise. "
|
|
151
|
+
"The verdict is written into agent.json beside the kits' proofs.",
|
|
152
|
+
"code": f"{CLI} fidelity my-agent --model gpt-5-mini --unsandboxed\n# equal: 20 fields\n# verdict: same"},
|
|
153
|
+
{"title": "Run a benchmark with it", "lang": "shell",
|
|
154
|
+
"text": "Or from the Run builder: runner <i>Nullius</i>, a benchmark, then your agent on the Agent step.",
|
|
155
|
+
"code": f"{CLI} run BENCHMARK my-agent --model gpt-5-mini --limit 2 --unsandboxed"},
|
|
156
|
+
{"title": "A new release: add it as a version", "lang": "shell",
|
|
157
|
+
"text": "The older version stays, with its proofs; the new one is offered once its own kits pass. "
|
|
158
|
+
"Check the pins it was given (a package that moved changes its name too), then prove it. "
|
|
159
|
+
"Set <b>default</b> in agent.json to make it the one a bare <code>my-agent</code> names.",
|
|
160
|
+
"code": f"{CLI} new version my-agent 1.5.0\n# wrote {agent}: my-agent@1.5.0 beside 1.4.2\n"
|
|
161
|
+
f"{CLI} conform my-agent@1.5.0 --model gpt-5-mini --unsandboxed"},
|
|
162
|
+
]
|
|
163
|
+
return [
|
|
164
|
+
{"title": "Create its folder, definition and wrapper", "lang": "shell",
|
|
165
|
+
"text": "The key is lower case letters, digits and dashes; it names the folder in your catalogue.",
|
|
166
|
+
"code": f"{CLI} new capability my-memory\n# wrote {definition}\n# wrote {wrapper}"},
|
|
167
|
+
{"title": "Write capability.py around your memory", "lang": "python", "file": wrapper,
|
|
168
|
+
"text": "Nullius sends <code>add</code> once per session of the history, then <code>recall</code> with the question, "
|
|
169
|
+
"and gives what comes back to the benchmark's reader. Replace the three marked calls with your library's.",
|
|
170
|
+
"code": FEATURE_PY},
|
|
171
|
+
{"title": "Try it by hand", "lang": "shell", "text": "Two lines in, two lines out, before Nullius is involved (with your library installed).",
|
|
172
|
+
"code": TRY_FEATURE.replace("{wrapper}", shlex.quote(wrapper))},
|
|
173
|
+
{"title": "Fill in capability.json", "lang": "json", "file": definition,
|
|
174
|
+
"text": "<b>install.pip</b> pins its packages; <b>start.command</b> runs the wrapper in that environment. "
|
|
175
|
+
"Leave <b>offers</b> and <b>proven</b> empty: step 6 writes them.",
|
|
176
|
+
"code": _json(FEATURE_JSON)},
|
|
177
|
+
{"title": "Or: installed from its repository, calling a model", "lang": "json",
|
|
178
|
+
"text": "Replace <b>upstream</b> and <b>install</b> with these when the memory lives in a repository: it is cloned at that "
|
|
179
|
+
"commit and its Python project installed (CPU torch first, the embedder fetched once). A memory that calls a model "
|
|
180
|
+
"gets a <b>model</b> block with the variable names it reads: its calls then go through the relay, as an agent's.",
|
|
181
|
+
"code": _json(FEATURE_FROM_REPO)},
|
|
182
|
+
{"title": "Check, then prove its port", "lang": "shell", "text": "Or use <i>Prove its ports</i> on its page.",
|
|
183
|
+
"code": f"{CLI} check\n# ok\n{CLI} conform my-memory --model gpt-5-mini\n# cap.memory/v1: passed 2/2"},
|
|
184
|
+
{"title": "Run a capability benchmark with it", "lang": "shell",
|
|
185
|
+
"text": "Or from the Run builder: runner <i>Nullius</i>, a benchmark that takes its port, then your capability on "
|
|
186
|
+
"the step named after its kind (Memory for a memory).",
|
|
187
|
+
"code": f"{CLI} run BENCHMARK my-memory --model gpt-5-mini --limit 10"},
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# The prompt the Add dialog builds for a coding assistant - any of them, so it
|
|
192
|
+
# names a skill's file as well as its command. Three slots, filled on the page
|
|
193
|
+
# with what was typed: {name}, {source}, {remark} - what the assistant should
|
|
194
|
+
# know beyond what it reads from the source; the Remark line is left out when
|
|
195
|
+
# there is none (``static/catalogue/add.js``).
|
|
196
|
+
_KINDS = ("benchmark", "agent", "capability")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def prompt(kind: str) -> str:
|
|
200
|
+
if kind not in _KINDS:
|
|
201
|
+
raise ValueError(f"unknown catalogue kind: {kind}")
|
|
202
|
+
folder = "capabilities" if kind == "capability" else kind + "s"
|
|
203
|
+
target = os.path.join(paths.catalogue_dir(), folder, "<key>")
|
|
204
|
+
specific = {
|
|
205
|
+
"benchmark": "First ask me how it should run: Harbor or the Nullius standard (its Original setup). Recommend a route.\n",
|
|
206
|
+
"agent": "Ask before running unsandboxed. Verify native capabilities and supported extensions.\n",
|
|
207
|
+
"capability": "Verify its callable operations and proven ports.\n",
|
|
208
|
+
}[kind]
|
|
209
|
+
return (f'Add the {kind} "{{name}}" to Nullius.\nSource: {{source}}\nRemark: {{remark}}\n\n'
|
|
210
|
+
f"Follow /import-{kind}, or .claude/skills/import-{kind}/SKILL.md in the source checkout, "
|
|
211
|
+
"with docs/standard.md and AGENTS.md. If unavailable, ask for the checkout.\n"
|
|
212
|
+
f"Write to {target}, even when the skill names package paths. "
|
|
213
|
+
"For an existing bundled entry, copy its whole folder there first; never edit installed files. "
|
|
214
|
+
f"On another machine, use app/nullius/catalogue/{folder}/<key> in a checkout and tell me to update Nullius.\n"
|
|
215
|
+
"Use pinned upstream code; ask before implementing behavior of our own. "
|
|
216
|
+
f"{specific}Report what fits, what passed, and what is blocked or untested.")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def prompts() -> dict:
|
|
220
|
+
return {kind: prompt(kind) for kind in _KINDS}
|