nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Kit for agent.tools/v1, version 2: five checks, each a different use of the
|
|
2
|
+
tools of one MCP server (``mcp_kit.py``) given to the session.
|
|
3
|
+
|
|
4
|
+
- ``one-tool``: the server offers a single tool without arguments;
|
|
5
|
+
- ``arguments``: a tool taking a string and an integer, its answer theirs;
|
|
6
|
+
- ``decoys``: the right tool among twenty of alike names and answers;
|
|
7
|
+
- ``chain``: three tools, each one's output the next one's input;
|
|
8
|
+
- ``retry``: the first call is refused as a tool error saying what was wrong,
|
|
9
|
+
and the agent calls again, corrected.
|
|
10
|
+
|
|
11
|
+
Each unit draws a seed the prompt never carries; every answer is derived from
|
|
12
|
+
it, so none can be guessed. A unit passes when the final message carries the
|
|
13
|
+
answer *and* the server's own log shows the calls that produce it - written
|
|
14
|
+
beside the trial's verdict (``verifier/mcp-calls.jsonl``), outside the
|
|
15
|
+
workspace the agent works in. The prompts say what to find, never which tool.
|
|
16
|
+
"""
|
|
17
|
+
import hashlib
|
|
18
|
+
import importlib.util
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import secrets
|
|
23
|
+
import sys
|
|
24
|
+
|
|
25
|
+
from nullius.standard.container import CHECK_IMAGE
|
|
26
|
+
from nullius.standard.jsonl import serve
|
|
27
|
+
|
|
28
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
29
|
+
_spec = importlib.util.spec_from_file_location("nullius_kit_mcp", os.path.join(HERE, "mcp_kit.py"))
|
|
30
|
+
mcp = importlib.util.module_from_spec(_spec)
|
|
31
|
+
_spec.loader.exec_module(mcp)
|
|
32
|
+
|
|
33
|
+
UNITS = ("one-tool", "arguments", "decoys", "chain", "retry")
|
|
34
|
+
WAREHOUSES = ("Riverside", "Northgate", "Old Quay", "Millbrook")
|
|
35
|
+
SITES = ("Harbour Road", "Cedar Yard", "Mill Lane", "Station Hill")
|
|
36
|
+
FINISH = " Reply with it and nothing else."
|
|
37
|
+
SEEDS = {}
|
|
38
|
+
# The agent starts the server: in its container on docker, by the image's own python3 (this machine's
|
|
39
|
+
# interpreter, mounted over it, would not run there); here, by this one.
|
|
40
|
+
PYTHON = "python3" if os.environ.get(CHECK_IMAGE) else sys.executable
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def asked(seed):
|
|
44
|
+
"""What the prompts name, from the seed: never an answer, only the question."""
|
|
45
|
+
n = int(hashlib.sha256(seed.encode()).hexdigest(), 16)
|
|
46
|
+
return {"warehouse": WAREHOUSES[n % 4], "item": 100 + n // 4 % 900, "parcel": f"P-{1000 + n // 3600 % 9000}",
|
|
47
|
+
"order": 1000 + n // 7 ** 9 % 9000, "site": SITES[n // 11 ** 9 % 4], "year": 2019 + n // 13 ** 9 % 8}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def prompt(unit, seed):
|
|
51
|
+
a = asked(seed)
|
|
52
|
+
use = "Using the tools of the MCP server named kit, given to this session, find "
|
|
53
|
+
return {
|
|
54
|
+
"one-tool": "The MCP server named kit, given to this session, holds the code word of this check. Find it." + FINISH,
|
|
55
|
+
"arguments": use + f"the stock code of item {a['item']} in the {a['warehouse']} warehouse." + FINISH,
|
|
56
|
+
"decoys": use + f"the carrier's tracking number of parcel {a['parcel']}." + FINISH,
|
|
57
|
+
"chain": use + f"the settlement code of the account that pays for order {a['order']}." + FINISH,
|
|
58
|
+
"retry": use + f"the permit code of the site called {a['site']} for the year {a['year']}." + FINISH,
|
|
59
|
+
}[unit]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def expected(unit, seed):
|
|
63
|
+
a = asked(seed)
|
|
64
|
+
if unit == "one-tool":
|
|
65
|
+
return mcp.call(seed, "lookup_code", {})
|
|
66
|
+
if unit == "arguments":
|
|
67
|
+
return mcp.code(seed, "stock", mcp.norm(a["warehouse"]), a["item"])
|
|
68
|
+
if unit == "decoys":
|
|
69
|
+
return mcp.code(seed, "get_tracking_number", mcp.norm(a["parcel"]))
|
|
70
|
+
if unit == "chain":
|
|
71
|
+
return mcp.chain(seed, a["order"])[2]
|
|
72
|
+
return mcp.code(seed, "permit", mcp.site_id(seed, a["site"]), a["year"])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def log_path(dir):
|
|
76
|
+
"""The server's call log: beside the trial's verdict, not in the workspace."""
|
|
77
|
+
return os.path.join(os.path.dirname(os.path.abspath(dir)), "verifier", "mcp-calls.jsonl")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def read_log(path):
|
|
81
|
+
out = []
|
|
82
|
+
try:
|
|
83
|
+
with open(path) as f:
|
|
84
|
+
for line in f:
|
|
85
|
+
try:
|
|
86
|
+
out.append(json.loads(line))
|
|
87
|
+
except ValueError:
|
|
88
|
+
pass
|
|
89
|
+
except OSError:
|
|
90
|
+
pass
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _flat(s):
|
|
95
|
+
return re.sub(r"[^A-Za-z0-9]", "", str(s or "")).upper()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _ok(c, tool, **want):
|
|
99
|
+
"""A call of ``tool`` that succeeded with these arguments (compared as the server compares them)."""
|
|
100
|
+
if c.get("tool") != tool or c.get("isError"):
|
|
101
|
+
return False
|
|
102
|
+
args = c.get("arguments") or {}
|
|
103
|
+
for k, v in want.items():
|
|
104
|
+
got = args.get(k)
|
|
105
|
+
if isinstance(v, int):
|
|
106
|
+
if isinstance(got, bool) or not isinstance(got, (int, float)) or got != v:
|
|
107
|
+
return False
|
|
108
|
+
elif mcp.norm(got) != mcp.norm(v):
|
|
109
|
+
return False
|
|
110
|
+
return True
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def missing(unit, seed, calls):
|
|
114
|
+
"""What the log lacks for ``unit``: None when the calls are there."""
|
|
115
|
+
a = asked(seed)
|
|
116
|
+
if unit == "one-tool":
|
|
117
|
+
return None if any(_ok(c, "lookup_code") for c in calls) else "lookup_code was never called"
|
|
118
|
+
if unit == "arguments":
|
|
119
|
+
return None if any(_ok(c, "get_stock_code", warehouse=a["warehouse"], item=a["item"]) for c in calls) \
|
|
120
|
+
else f"no successful get_stock_code call for item {a['item']} in {a['warehouse']}"
|
|
121
|
+
if unit == "decoys":
|
|
122
|
+
return None if any(_ok(c, "get_tracking_number", parcel=a["parcel"]) for c in calls) \
|
|
123
|
+
else f"get_tracking_number was never called for {a['parcel']}"
|
|
124
|
+
if unit == "chain":
|
|
125
|
+
customer, account, _ = mcp.chain(seed, a["order"])
|
|
126
|
+
hops = [("get_order_customer", {"order": a["order"]}), ("get_customer_account", {"customer": customer}),
|
|
127
|
+
("get_account_settlement", {"account": account})]
|
|
128
|
+
i = 0
|
|
129
|
+
for c in calls: # the three hops, in order
|
|
130
|
+
if i < len(hops) and _ok(c, hops[i][0], **hops[i][1]):
|
|
131
|
+
i += 1
|
|
132
|
+
return None if i == len(hops) else f"the chain stopped before {hops[i][0]}"
|
|
133
|
+
sid = mcp.site_id(seed, a["site"])
|
|
134
|
+
refused = next((n for n, c in enumerate(calls) if c.get("tool") == "get_permit_code" and c.get("isError")), None)
|
|
135
|
+
if refused is None:
|
|
136
|
+
return "get_permit_code never answered with an error"
|
|
137
|
+
if not any(_ok(c, "get_permit_code", site=sid, year=a["year"]) for c in calls[refused + 1:]):
|
|
138
|
+
return "no corrected get_permit_code call after the error"
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def verdict(unit, seed, calls, answer):
|
|
143
|
+
"""(passed, detail) for one unit, from the server's log and the final message."""
|
|
144
|
+
want = expected(unit, seed)
|
|
145
|
+
said = answer or ""
|
|
146
|
+
why = None if _flat(want) in _flat(said) else "the final message does not carry the answer"
|
|
147
|
+
why = why or missing(unit, seed, calls)
|
|
148
|
+
codes = set(re.findall(r"[A-Za-z0-9]{4}-?[A-Za-z0-9]{4}", said))
|
|
149
|
+
came = [c["tool"] for c in calls if not c.get("isError") and any(_flat(x) in _flat(c.get("text")) for x in codes)]
|
|
150
|
+
detail = {"said": said[:200], "expected": want, "calls": [c.get("tool") for c in calls]}
|
|
151
|
+
if why:
|
|
152
|
+
detail["why"] = why
|
|
153
|
+
if came:
|
|
154
|
+
detail["answer_from"] = sorted(set(came))
|
|
155
|
+
return why is None, detail
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def units(dataset):
|
|
159
|
+
return {"units": list(UNITS)}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def setup(unit, dir):
|
|
163
|
+
if unit not in UNITS:
|
|
164
|
+
raise ValueError(f"no unit {unit!r}")
|
|
165
|
+
SEEDS[unit] = seed = secrets.token_hex(16)
|
|
166
|
+
log = log_path(dir)
|
|
167
|
+
os.makedirs(os.path.dirname(log), exist_ok=True)
|
|
168
|
+
if os.path.exists(log):
|
|
169
|
+
os.remove(log)
|
|
170
|
+
env = {"KIT_SEED": seed, "KIT_LOG": log}
|
|
171
|
+
if unit == "one-tool":
|
|
172
|
+
env["KIT_TOOLS"] = "lookup_code"
|
|
173
|
+
return {"prompt": prompt(unit, seed),
|
|
174
|
+
"mcp": [{"name": "kit", "command": PYTHON, "args": [os.path.join(HERE, "mcp_kit.py")], "env": env}]}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def score(unit, dir, transcript, answer):
|
|
178
|
+
if unit not in SEEDS:
|
|
179
|
+
raise ValueError(f"{unit} was not set up")
|
|
180
|
+
ok, detail = verdict(unit, SEEDS[unit], read_log(log_path(dir)), answer)
|
|
181
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": detail}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def oracle(unit, dir):
|
|
185
|
+
"""The reference: the calls a correct agent makes, sent to the kit's own
|
|
186
|
+
server code (``mcp.handle``, logged as the server logs them), and the
|
|
187
|
+
answer they lead to - so the reference passes through the same scorer."""
|
|
188
|
+
seed, a = SEEDS[unit], asked(SEEDS[unit])
|
|
189
|
+
if unit == "chain":
|
|
190
|
+
customer, account, _ = mcp.chain(seed, a["order"])
|
|
191
|
+
calls = [("get_order_customer", {"order": a["order"]}), ("get_customer_account", {"customer": customer}),
|
|
192
|
+
("get_account_settlement", {"account": account})]
|
|
193
|
+
else:
|
|
194
|
+
calls = {"one-tool": [("lookup_code", {})],
|
|
195
|
+
"arguments": [("get_stock_code", {"warehouse": a["warehouse"], "item": a["item"]})],
|
|
196
|
+
"decoys": [("get_tracking_number", {"parcel": a["parcel"]})],
|
|
197
|
+
"retry": [("get_permit_code", {"site": a["site"], "year": a["year"]}), # refused: a name, not an id
|
|
198
|
+
("get_permit_code", {"site": mcp.site_id(seed, a["site"]), "year": a["year"]})]}[unit]
|
|
199
|
+
for n, (tool, args) in enumerate(calls):
|
|
200
|
+
mcp.handle({"jsonrpc": "2.0", "id": n, "method": "tools/call", "params": {"name": tool, "arguments": args}},
|
|
201
|
+
seed, log=log_path(dir))
|
|
202
|
+
return {"answer": expected(unit, seed)}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
if __name__ == "__main__":
|
|
206
|
+
serve({"units": units, "setup": setup, "score": score, "oracle": oracle})
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""The tools kit's MCP server: JSON-RPC 2.0 over stdio, one message per line.
|
|
2
|
+
|
|
3
|
+
Twenty tools of a small shipping office - stock, parcels, orders, accounts,
|
|
4
|
+
site permits - most of them decoys for any one check: names alike, answers
|
|
5
|
+
of the same shape. Every value is derived from ``$KIT_SEED`` (one seed per
|
|
6
|
+
unit, drawn by the driver and never in a prompt), so an answer cannot be
|
|
7
|
+
guessed, only fetched with the right tool and the right arguments.
|
|
8
|
+
|
|
9
|
+
Every ``tools/call`` is appended to ``$KIT_LOG`` as one JSON line - the tool,
|
|
10
|
+
its arguments, whether it was an error, what it said - and the driver scores
|
|
11
|
+
from that log and the final answer. ``$KIT_TOOLS``, a comma-separated list,
|
|
12
|
+
narrows what ``tools/list`` offers (the one-tool check); every tool by default.
|
|
13
|
+
|
|
14
|
+
The driver imports this module for the same derivations (:func:`code`,
|
|
15
|
+
:func:`site_id`, :func:`chain`), so the server and the scoring cannot drift.
|
|
16
|
+
"""
|
|
17
|
+
import base64
|
|
18
|
+
import hashlib
|
|
19
|
+
import hmac
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import sys
|
|
23
|
+
import time
|
|
24
|
+
|
|
25
|
+
PROTOCOL = "2025-06-18"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def code(seed, *parts):
|
|
29
|
+
"""A code such as ``Q7KD-29XF``: eight characters of an HMAC of ``parts``
|
|
30
|
+
under the unit's seed. Every tool answers in this shape."""
|
|
31
|
+
d = hmac.new(str(seed).encode(), "|".join(str(p) for p in parts).encode(), hashlib.sha256).digest()
|
|
32
|
+
s = base64.b32encode(d).decode()[:8]
|
|
33
|
+
return f"{s[:4]}-{s[4:]}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def norm(s):
|
|
37
|
+
"""An identifier as the tools compare it: case and outer spaces ignored."""
|
|
38
|
+
return " ".join(str(s).split()).upper()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def site_id(seed, name):
|
|
42
|
+
return "SITE-" + code(seed, "site", norm(name)).replace("-", "")[:5]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def chain(seed, order):
|
|
46
|
+
"""Order -> customer -> account -> settlement code: the three hops."""
|
|
47
|
+
customer = "CUS-" + code(seed, "customer", int(order))
|
|
48
|
+
account = "ACC-" + code(seed, "account", customer)
|
|
49
|
+
return customer, account, code(seed, "settlement", account)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _s(desc):
|
|
53
|
+
return {"type": "string", "description": desc}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _i(desc):
|
|
57
|
+
return {"type": "integer", "description": desc}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
PARCEL = {"parcel": _s("The parcel's id, such as P-1234.")}
|
|
61
|
+
NUMBERS = { # one family of near-twins: the decoy check asks for one of them
|
|
62
|
+
"get_tracking_number": "The carrier's tracking number of a parcel, as the carrier knows it.",
|
|
63
|
+
"get_tracing_number": "The internal trace number of a parcel in the warehouse scanner log.",
|
|
64
|
+
"get_order_number": "The number of the customer order a parcel belongs to.",
|
|
65
|
+
"get_invoice_number": "The number of the invoice billed for a parcel.",
|
|
66
|
+
"get_return_number": "The return authorisation number of a parcel sent back.",
|
|
67
|
+
"get_customs_number": "The customs declaration number of a parcel.",
|
|
68
|
+
"get_batch_number": "The packing batch number a parcel was packed in.",
|
|
69
|
+
"get_route_number": "The delivery route number a parcel is on.",
|
|
70
|
+
"get_pallet_number": "The number of the pallet a parcel travels on.",
|
|
71
|
+
"get_seal_number": "The number of the tamper seal on a parcel.",
|
|
72
|
+
}
|
|
73
|
+
# "The site", not "the site's id": a schema asking for an id the agent has no
|
|
74
|
+
# way to learn sends a careful agent looking for one instead of calling - the
|
|
75
|
+
# error is where it learns the id, as from many a real API.
|
|
76
|
+
SITE_YEAR = {"site": _s("The site."),"year": _i("The year, such as 2025.")}
|
|
77
|
+
STOCK = {"warehouse": _s("The warehouse's name."), "item": _i("The item number.")}
|
|
78
|
+
|
|
79
|
+
TOOLS = [
|
|
80
|
+
("lookup_code", "Returns the code word of this check.", {}),
|
|
81
|
+
("get_stock_code", "The stock code of an item in a warehouse.", STOCK),
|
|
82
|
+
("get_stock_count", "How many of an item a warehouse holds.", STOCK),
|
|
83
|
+
*[(n, d, PARCEL) for n, d in NUMBERS.items()],
|
|
84
|
+
("get_order_status", "Where an order stands: received, packed, shipped or delivered.", {"order": _i("The order number.")}),
|
|
85
|
+
("get_order_customer", "The id of the customer who placed an order.", {"order": _i("The order number.")}),
|
|
86
|
+
("get_customer_account", "The id of the account a customer pays from.", {"customer": _s("The customer's id.")}),
|
|
87
|
+
("get_account_settlement", "The settlement code of an account.", {"account": _s("The account's id.")}),
|
|
88
|
+
("get_account_balance", "The balance of an account, in euros.", {"account": _s("The account's id.")}),
|
|
89
|
+
("get_permit_code", "The permit code of a site for a year.", SITE_YEAR),
|
|
90
|
+
("get_permit_fee", "The permit fee of a site for a year, in euros.", SITE_YEAR),
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ToolError(Exception):
|
|
95
|
+
"""What was wrong with a call, said back to the agent as ``isError``."""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _int(args, name):
|
|
99
|
+
v = args.get(name)
|
|
100
|
+
if isinstance(v, bool) or not (isinstance(v, int) or (isinstance(v, float) and v.is_integer())):
|
|
101
|
+
raise ToolError(f"{name} must be an integer, not {json.dumps(v)}")
|
|
102
|
+
return int(v)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _str(args, name):
|
|
106
|
+
v = args.get(name)
|
|
107
|
+
if not isinstance(v, str) or not v.strip():
|
|
108
|
+
raise ToolError(f"{name} must be a non-empty string, not {json.dumps(v)}")
|
|
109
|
+
return v
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _site(seed, args):
|
|
113
|
+
"""A site is named by its id; a name is refused with the id it has, the
|
|
114
|
+
one way to learn it - so the first call errs and the second can succeed."""
|
|
115
|
+
s = norm(_str(args, "site"))
|
|
116
|
+
if not s.startswith("SITE-"):
|
|
117
|
+
raise ToolError(f"{_str(args, 'site')!r} is not a site id: sites are named by id here, "
|
|
118
|
+
f"and the site called {_str(args, 'site').strip()!r} has the id {site_id(seed, s)}")
|
|
119
|
+
return s
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def call(seed, name, args):
|
|
123
|
+
"""The text a tool returns; :class:`ToolError` for a call it refuses."""
|
|
124
|
+
args = args if isinstance(args, dict) else {}
|
|
125
|
+
if name == "lookup_code":
|
|
126
|
+
return "kit-" + code(seed, "lookup").replace("-", "").lower()
|
|
127
|
+
if name in ("get_stock_code", "get_stock_count"):
|
|
128
|
+
w, i = norm(_str(args, "warehouse")), _int(args, "item")
|
|
129
|
+
if name == "get_stock_count":
|
|
130
|
+
return f"{int(code(seed, 'count', w, i).encode().hex(), 16) % 500} in stock"
|
|
131
|
+
return f"stock code of item {i} in {w.title()}: {code(seed, 'stock', w, i)}"
|
|
132
|
+
if name in NUMBERS:
|
|
133
|
+
p = norm(_str(args, "parcel"))
|
|
134
|
+
return f"{name[4:].replace('_', ' ')} of parcel {p}: {code(seed, name, p)}"
|
|
135
|
+
if name == "get_order_status":
|
|
136
|
+
o = _int(args, "order")
|
|
137
|
+
return f"order {o}: " + ("received", "packed", "shipped", "delivered")[ord(code(seed, "status", o)[0]) % 4]
|
|
138
|
+
if name == "get_order_customer":
|
|
139
|
+
return f"order {_int(args, 'order')} was placed by customer {chain(seed, _int(args, 'order'))[0]}"
|
|
140
|
+
if name == "get_customer_account":
|
|
141
|
+
c = norm(_str(args, "customer"))
|
|
142
|
+
return f"customer {c} pays from account ACC-{code(seed, 'account', c)}"
|
|
143
|
+
if name == "get_account_settlement":
|
|
144
|
+
a = norm(_str(args, "account"))
|
|
145
|
+
return f"settlement code of account {a}: {code(seed, 'settlement', a)}"
|
|
146
|
+
if name == "get_account_balance":
|
|
147
|
+
a = norm(_str(args, "account"))
|
|
148
|
+
return f"balance of account {a}: {int(code(seed, 'balance', a).encode().hex(), 16) % 90000 / 100:.2f} EUR"
|
|
149
|
+
if name in ("get_permit_code", "get_permit_fee"):
|
|
150
|
+
s, y = _site(seed, args), _int(args, "year")
|
|
151
|
+
if name == "get_permit_fee":
|
|
152
|
+
return f"permit fee of {s} for {y}: {100 + int(code(seed, 'fee', s, y).encode().hex(), 16) % 900} EUR"
|
|
153
|
+
return f"permit code of {s} for {y}: {code(seed, 'permit', s, y)}"
|
|
154
|
+
raise KeyError(name)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def listed(only=None):
|
|
158
|
+
keep = [t for t in TOOLS if not only or t[0] in only]
|
|
159
|
+
return [{"name": n, "description": d,
|
|
160
|
+
"inputSchema": dict({"type": "object", "properties": props}, **({"required": list(props)} if props else {}))}
|
|
161
|
+
for n, d, props in keep]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def handle(m, seed, only=None, log=None):
|
|
165
|
+
"""One JSON-RPC message in, its reply out (None for a notification)."""
|
|
166
|
+
if not isinstance(m, dict) or "id" not in m:
|
|
167
|
+
return None
|
|
168
|
+
method, rid, params = m.get("method"), m["id"], m.get("params") or {}
|
|
169
|
+
if method == "initialize":
|
|
170
|
+
result = {"protocolVersion": params.get("protocolVersion") or PROTOCOL,
|
|
171
|
+
"capabilities": {"tools": {}}, "serverInfo": {"name": "kit", "version": "2"}}
|
|
172
|
+
elif method == "tools/list":
|
|
173
|
+
result = {"tools": listed(only)}
|
|
174
|
+
elif method == "tools/call":
|
|
175
|
+
name, args = params.get("name"), params.get("arguments") or {}
|
|
176
|
+
if name not in [t["name"] for t in listed(only)]:
|
|
177
|
+
return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"unknown tool {name!r}"}}
|
|
178
|
+
try:
|
|
179
|
+
text, err = call(seed, name, args), False
|
|
180
|
+
except ToolError as e:
|
|
181
|
+
text, err = str(e), True
|
|
182
|
+
if log:
|
|
183
|
+
with open(log, "a") as f:
|
|
184
|
+
f.write(json.dumps({"t": time.time(), "tool": name, "arguments": args, "isError": err, "text": text}) + "\n")
|
|
185
|
+
result = {"content": [{"type": "text", "text": text}], "isError": err}
|
|
186
|
+
elif method == "ping":
|
|
187
|
+
result = {}
|
|
188
|
+
elif method in ("resources/list", "prompts/list"):
|
|
189
|
+
result = {method.split("/")[0]: []}
|
|
190
|
+
else:
|
|
191
|
+
return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": "no " + str(method)}}
|
|
192
|
+
return {"jsonrpc": "2.0", "id": rid, "result": result}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def main():
|
|
196
|
+
seed, log = os.environ.get("KIT_SEED", ""), os.environ.get("KIT_LOG") or None
|
|
197
|
+
only = [t for t in os.environ.get("KIT_TOOLS", "").split(",") if t] or None
|
|
198
|
+
for line in sys.stdin:
|
|
199
|
+
try:
|
|
200
|
+
m = json.loads(line)
|
|
201
|
+
except ValueError:
|
|
202
|
+
continue
|
|
203
|
+
reply = handle(m, seed, only, log)
|
|
204
|
+
if reply is not None:
|
|
205
|
+
print(json.dumps(reply), flush=True)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
if __name__ == "__main__":
|
|
209
|
+
main()
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "agent-tools-v1",
|
|
4
|
+
"kit_version": 2,
|
|
5
|
+
"label": "Kit: agent.tools/v1",
|
|
6
|
+
"summary": "Five checks against one MCP server of twenty tools: two basic - a tool without arguments, one with typed arguments - and three skills - the right tool among decoys, a chain of three, a call refused and corrected.",
|
|
7
|
+
"family": "agent",
|
|
8
|
+
"needs": [
|
|
9
|
+
"agent.tools/v1"
|
|
10
|
+
],
|
|
11
|
+
"basic": [
|
|
12
|
+
"one-tool",
|
|
13
|
+
"arguments"
|
|
14
|
+
],
|
|
15
|
+
"skills": {
|
|
16
|
+
"decoys": "chooses the right tool among similar ones",
|
|
17
|
+
"chain": "feeds one tool's answer into the next, three times over",
|
|
18
|
+
"retry": "reads a tool's error and calls it again, corrected"
|
|
19
|
+
},
|
|
20
|
+
"driver": {
|
|
21
|
+
"command": [
|
|
22
|
+
"python3",
|
|
23
|
+
"{here}/driver.py"
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
"datasets": [
|
|
27
|
+
{
|
|
28
|
+
"id": "kit",
|
|
29
|
+
"units": 5
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"units": {
|
|
33
|
+
"noun": "check",
|
|
34
|
+
"nouns": "checks"
|
|
35
|
+
},
|
|
36
|
+
"controls": {
|
|
37
|
+
"nop": {
|
|
38
|
+
"source": "nullius",
|
|
39
|
+
"from": "Nullius: no agent runs; the answer is empty and the workspace is left as setup made it."
|
|
40
|
+
},
|
|
41
|
+
"oracle": {
|
|
42
|
+
"source": "nullius",
|
|
43
|
+
"from": "The code word the kit's setup gave the unit's MCP server."
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"score": {
|
|
47
|
+
"kind": "pass",
|
|
48
|
+
"says": "the final message carries the answer, and the server's call log shows the calls that produce it"
|
|
49
|
+
},
|
|
50
|
+
"limits": {
|
|
51
|
+
"turn_seconds": 300
|
|
52
|
+
},
|
|
53
|
+
"proven": {
|
|
54
|
+
"kit": {
|
|
55
|
+
"units": {
|
|
56
|
+
"units": 5,
|
|
57
|
+
"passed": "5 ids, the same twice, unique",
|
|
58
|
+
"on": "2026-10-12"
|
|
59
|
+
},
|
|
60
|
+
"leak": {
|
|
61
|
+
"units": 5,
|
|
62
|
+
"looked_for": 1,
|
|
63
|
+
"short": 4,
|
|
64
|
+
"passed": "5/5",
|
|
65
|
+
"note": "4 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
|
|
66
|
+
"on": "2026-10-12"
|
|
67
|
+
},
|
|
68
|
+
"nop": {
|
|
69
|
+
"passed": "0/5",
|
|
70
|
+
"value": 0.0,
|
|
71
|
+
"expect": {
|
|
72
|
+
"at_most": 0.0
|
|
73
|
+
},
|
|
74
|
+
"units": 5,
|
|
75
|
+
"job": "2026-10-12__16-28-38__agent-tools-v1__kit__prove-nop",
|
|
76
|
+
"source": "nullius",
|
|
77
|
+
"on": "2026-10-12"
|
|
78
|
+
},
|
|
79
|
+
"oracle": {
|
|
80
|
+
"passed": "5/5",
|
|
81
|
+
"value": 1.0,
|
|
82
|
+
"expect": {
|
|
83
|
+
"at_least": 1.0
|
|
84
|
+
},
|
|
85
|
+
"units": 5,
|
|
86
|
+
"job": "2026-10-12__16-28-38__agent-tools-v1__kit__prove-oracle",
|
|
87
|
+
"source": "nullius",
|
|
88
|
+
"on": "2026-10-12"
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
"check_details": {
|
|
93
|
+
"one-tool": {
|
|
94
|
+
"label": "Call one tool",
|
|
95
|
+
"task": "Find the code word using the single available MCP tool.",
|
|
96
|
+
"passes": "The final answer contains the generated code and the server records a successful lookup_code call."
|
|
97
|
+
},
|
|
98
|
+
"arguments": {
|
|
99
|
+
"label": "Pass typed arguments",
|
|
100
|
+
"task": "Find an item’s stock code in a named warehouse.",
|
|
101
|
+
"passes": "The answer contains the code and the server records get_stock_code with the correct warehouse and numeric item."
|
|
102
|
+
},
|
|
103
|
+
"decoys": {
|
|
104
|
+
"label": "Choose among decoys",
|
|
105
|
+
"task": "Find a parcel’s tracking number among twenty similar tools.",
|
|
106
|
+
"passes": "The answer contains the tracking number and the server records get_tracking_number for that parcel."
|
|
107
|
+
},
|
|
108
|
+
"chain": {
|
|
109
|
+
"label": "Chain three calls",
|
|
110
|
+
"task": "Find the settlement code for the account paying for an order.",
|
|
111
|
+
"passes": "The answer contains the code; successful order → customer → account → settlement calls appear in that order."
|
|
112
|
+
},
|
|
113
|
+
"retry": {
|
|
114
|
+
"label": "Recover from a tool error",
|
|
115
|
+
"task": "Find a site’s permit code for a given year.",
|
|
116
|
+
"passes": "The answer contains the code; a failed get_permit_code call is followed by a successful call with the correct site ID and year."
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Kit for agent.workspace/v1: the agent reads and writes in its working directory."""
|
|
2
|
+
import os
|
|
3
|
+
import stat
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from nullius.standard.container import check_argv
|
|
8
|
+
from nullius.standard.jsonl import serve
|
|
9
|
+
|
|
10
|
+
CALC = "def add(a, b):\n return a - b\n"
|
|
11
|
+
TEST = "from calc import add\nassert add(2, 3) == 5, add(2, 3)\nprint('OK')\n"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def units(dataset):
|
|
15
|
+
return {"units": ["write", "fix"]}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def setup(unit, dir):
|
|
19
|
+
if unit == "write":
|
|
20
|
+
return {"prompt": "In the current working directory, create a file named result.txt whose whole content is: nullius-ok"}
|
|
21
|
+
with open(os.path.join(dir, "calc.py"), "w") as f:
|
|
22
|
+
f.write(CALC)
|
|
23
|
+
with open(os.path.join(dir, "test_calc.py"), "w") as f:
|
|
24
|
+
f.write(TEST)
|
|
25
|
+
return {"prompt": "In the current working directory, calc.py has a bug: `python3 test_calc.py` fails. Fix calc.py so the test prints OK. Do not change test_calc.py."}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def score(unit, dir, transcript, answer):
|
|
29
|
+
if unit == "write":
|
|
30
|
+
got = _read_plain(os.path.join(dir, "result.txt"))
|
|
31
|
+
ok = got == "nullius-ok"
|
|
32
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"result.txt": got}}
|
|
33
|
+
test = os.path.join(dir, "test_calc.py")
|
|
34
|
+
if os.path.lexists(test): # the test as given, whatever happened to it - a link included
|
|
35
|
+
os.unlink(test)
|
|
36
|
+
with open(test, "w") as f:
|
|
37
|
+
f.write(TEST)
|
|
38
|
+
argv = check_argv(dir, [sys.executable, "test_calc.py"], ["timeout", "60", "python3", "test_calc.py"])
|
|
39
|
+
r = subprocess.run(argv, cwd=dir, capture_output=True, text=True, timeout=120)
|
|
40
|
+
ok = r.returncode == 0 and "OK" in r.stdout
|
|
41
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"stdout": r.stdout[-500:], "stderr": r.stderr[-500:]}}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read_plain(path):
|
|
45
|
+
"""A regular file's text, or None: a link the agent planted is not followed
|
|
46
|
+
out of its workspace (on docker it could point at the user's settings)."""
|
|
47
|
+
try:
|
|
48
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
49
|
+
except OSError:
|
|
50
|
+
return None
|
|
51
|
+
with os.fdopen(fd) as f:
|
|
52
|
+
return f.read().strip() if stat.S_ISREG(os.fstat(f.fileno()).st_mode) else None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def oracle(unit, dir):
|
|
56
|
+
if unit == "write":
|
|
57
|
+
with open(os.path.join(dir, "result.txt"), "w") as f:
|
|
58
|
+
f.write("nullius-ok\n")
|
|
59
|
+
else:
|
|
60
|
+
with open(os.path.join(dir, "calc.py"), "w") as f:
|
|
61
|
+
f.write(CALC.replace("a - b", "a + b"))
|
|
62
|
+
return {"answer": ""}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
serve({"units": units, "setup": setup, "score": score, "oracle": oracle})
|