nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "agent-workspace-v1",
|
|
4
|
+
"kit_version": 1,
|
|
5
|
+
"label": "Kit: agent.workspace/v1",
|
|
6
|
+
"summary": "Write a file with given content; fix a one-line bug so a test passes - in the session's working directory.",
|
|
7
|
+
"family": "agent",
|
|
8
|
+
"needs": [
|
|
9
|
+
"agent.workspace/v1"
|
|
10
|
+
],
|
|
11
|
+
"driver": {
|
|
12
|
+
"command": [
|
|
13
|
+
"python3",
|
|
14
|
+
"{here}/driver.py"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
"datasets": [
|
|
18
|
+
{
|
|
19
|
+
"id": "kit",
|
|
20
|
+
"units": 2
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"units": {
|
|
24
|
+
"noun": "check",
|
|
25
|
+
"nouns": "checks"
|
|
26
|
+
},
|
|
27
|
+
"controls": {
|
|
28
|
+
"nop": {
|
|
29
|
+
"source": "nullius",
|
|
30
|
+
"from": "Nullius: no agent runs; the answer is empty and the workspace is left as setup made it."
|
|
31
|
+
},
|
|
32
|
+
"oracle": {
|
|
33
|
+
"source": "nullius",
|
|
34
|
+
"from": "The kit's own fix: result.txt written, and calc.py adding instead of subtracting."
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"score": {
|
|
38
|
+
"kind": "pass",
|
|
39
|
+
"says": "the file holds the content; the test passes"
|
|
40
|
+
},
|
|
41
|
+
"limits": {
|
|
42
|
+
"turn_seconds": 600
|
|
43
|
+
},
|
|
44
|
+
"proven": {
|
|
45
|
+
"kit": {
|
|
46
|
+
"units": {
|
|
47
|
+
"units": 2,
|
|
48
|
+
"passed": "2 ids, the same twice, unique",
|
|
49
|
+
"on": "2026-10-12"
|
|
50
|
+
},
|
|
51
|
+
"leak": {
|
|
52
|
+
"units": 2,
|
|
53
|
+
"looked_for": 1,
|
|
54
|
+
"short": 1,
|
|
55
|
+
"passed": "2/2",
|
|
56
|
+
"note": "1 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
|
|
57
|
+
"on": "2026-10-12"
|
|
58
|
+
},
|
|
59
|
+
"nop": {
|
|
60
|
+
"passed": "0/2",
|
|
61
|
+
"value": 0.0,
|
|
62
|
+
"expect": {
|
|
63
|
+
"at_most": 0.0
|
|
64
|
+
},
|
|
65
|
+
"units": 2,
|
|
66
|
+
"job": "2026-10-12__16-13-26__agent-workspace-v1__kit__prove-nop",
|
|
67
|
+
"source": "nullius",
|
|
68
|
+
"on": "2026-10-12"
|
|
69
|
+
},
|
|
70
|
+
"oracle": {
|
|
71
|
+
"passed": "2/2",
|
|
72
|
+
"value": 1.0,
|
|
73
|
+
"expect": {
|
|
74
|
+
"at_least": 1.0
|
|
75
|
+
},
|
|
76
|
+
"units": 2,
|
|
77
|
+
"job": "2026-10-12__16-13-26__agent-workspace-v1__kit__prove-oracle",
|
|
78
|
+
"source": "nullius",
|
|
79
|
+
"on": "2026-10-12"
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"check_details": {
|
|
84
|
+
"write": {
|
|
85
|
+
"label": "Create a file",
|
|
86
|
+
"task": "Create result.txt in the working directory with the content nullius-ok.",
|
|
87
|
+
"passes": "The file exists and its content, after trimming whitespace, equals nullius-ok."
|
|
88
|
+
},
|
|
89
|
+
"fix": {
|
|
90
|
+
"label": "Fix a function",
|
|
91
|
+
"task": "Fix calc.py so add(2, 3) returns 5.",
|
|
92
|
+
"passes": "The original test_calc.py is restored and run; it exits successfully and prints OK."
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Kit for cap.memory/v1: add sessions, recall for a question, the fact must come back."""
|
|
2
|
+
from nullius.standard.jsonl import serve
|
|
3
|
+
|
|
4
|
+
FILLER = ["We discussed the train timetable for the weekend.", "The weather was cold and grey all morning.",
|
|
5
|
+
"I watched a documentary about volcanoes.", "Dinner plans changed to a quiet night in.",
|
|
6
|
+
"The neighbours painted their fence blue."]
|
|
7
|
+
CHECKS = {
|
|
8
|
+
"m1": ("My sister Ines moved to Lisbon in March.", "Where did Ines move to?", "Lisbon"),
|
|
9
|
+
"m2": ("The boat at the cabin is named tern-river-9.", "What is the boat at the cabin named?", "tern-river-9"),
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def units(dataset):
|
|
14
|
+
return {"units": list(CHECKS)}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def setup(unit, dir):
|
|
18
|
+
fact, q, _ = CHECKS[unit]
|
|
19
|
+
sessions = [{"id": f"s{i + 1}", "date": f"2026-0{i + 1}-01", "turns": [{"role": "user", "text": t}]}
|
|
20
|
+
for i, t in enumerate(FILLER[:1] + [fact] + FILLER[1:])]
|
|
21
|
+
for s in sessions:
|
|
22
|
+
s["text"] = " ".join(t["text"] for t in s["turns"])
|
|
23
|
+
return {"memory": {"add": sessions, "recall": [{"id": unit, "question": q}]}}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def produce(unit, dir, recalled, **kw):
|
|
27
|
+
return {"answer": recalled[0]["memory"] if recalled else ""}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def score(unit, dir, transcript, answer):
|
|
31
|
+
ok = CHECKS[unit][2].lower() in (answer or "").lower()
|
|
32
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"recalled": (answer or "")[:500]}}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def oracle(unit, dir):
|
|
36
|
+
return {"answer": CHECKS[unit][0]} # the fact itself, as it was added
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
serve({"units": units, "setup": setup, "produce": produce, "score": score, "oracle": oracle})
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "cap-memory-v1",
|
|
4
|
+
"kit_version": 1,
|
|
5
|
+
"label": "Kit: cap.memory/v1",
|
|
6
|
+
"summary": "Six sessions, one fact among distractors: what the memory recalls for a question about it must carry the fact.",
|
|
7
|
+
"family": "capability",
|
|
8
|
+
"needs": [
|
|
9
|
+
"cap.memory/v1"
|
|
10
|
+
],
|
|
11
|
+
"driver": {
|
|
12
|
+
"command": [
|
|
13
|
+
"python3",
|
|
14
|
+
"{here}/driver.py"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
"datasets": [
|
|
18
|
+
{
|
|
19
|
+
"id": "kit",
|
|
20
|
+
"units": 2
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"units": {
|
|
24
|
+
"noun": "check",
|
|
25
|
+
"nouns": "checks"
|
|
26
|
+
},
|
|
27
|
+
"controls": {
|
|
28
|
+
"nop": {
|
|
29
|
+
"source": "nullius",
|
|
30
|
+
"from": "Nullius: no capability runs and nothing is recalled; the answer is empty."
|
|
31
|
+
},
|
|
32
|
+
"oracle": {
|
|
33
|
+
"source": "nullius",
|
|
34
|
+
"from": "The fact the kit added among its distractors, given back as the answer."
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"score": {
|
|
38
|
+
"kind": "pass",
|
|
39
|
+
"says": "the recalled memory holds the fact"
|
|
40
|
+
},
|
|
41
|
+
"proven": {
|
|
42
|
+
"kit": {
|
|
43
|
+
"units": {
|
|
44
|
+
"units": 2,
|
|
45
|
+
"passed": "2 ids, the same twice, unique",
|
|
46
|
+
"on": "2026-10-12"
|
|
47
|
+
},
|
|
48
|
+
"leak": {
|
|
49
|
+
"units": 2,
|
|
50
|
+
"looked_for": 2,
|
|
51
|
+
"short": 0,
|
|
52
|
+
"passed": "2/2",
|
|
53
|
+
"on": "2026-10-12"
|
|
54
|
+
},
|
|
55
|
+
"nop": {
|
|
56
|
+
"passed": "0/2",
|
|
57
|
+
"value": 0.0,
|
|
58
|
+
"expect": {
|
|
59
|
+
"at_most": 0.0
|
|
60
|
+
},
|
|
61
|
+
"units": 2,
|
|
62
|
+
"job": "2026-10-12__16-13-29__feature-memory-v1__kit__prove-nop",
|
|
63
|
+
"source": "nullius",
|
|
64
|
+
"on": "2026-10-12"
|
|
65
|
+
},
|
|
66
|
+
"oracle": {
|
|
67
|
+
"passed": "2/2",
|
|
68
|
+
"value": 1.0,
|
|
69
|
+
"expect": {
|
|
70
|
+
"at_least": 1.0
|
|
71
|
+
},
|
|
72
|
+
"units": 2,
|
|
73
|
+
"job": "2026-10-12__16-13-29__feature-memory-v1__kit__prove-oracle",
|
|
74
|
+
"source": "nullius",
|
|
75
|
+
"on": "2026-10-12"
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
"check_details": {
|
|
80
|
+
"m1": {
|
|
81
|
+
"label": "Recall a place",
|
|
82
|
+
"task": "Store six sessions, including “My sister Ines moved to Lisbon in March.” Ask where Ines moved.",
|
|
83
|
+
"passes": "The recalled memory contains Lisbon, ignoring case."
|
|
84
|
+
},
|
|
85
|
+
"m2": {
|
|
86
|
+
"label": "Recall an exact name",
|
|
87
|
+
"task": "Store six sessions, including “The boat at the cabin is named tern-river-9.” Ask what the boat is named.",
|
|
88
|
+
"passes": "The recalled memory contains tern-river-9, ignoring case."
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""A run's spending cap: one small file every relay of the job books its calls in.
|
|
2
|
+
|
|
3
|
+
A capped run (``run --max-usd``) gives each of its relays - the units' own,
|
|
4
|
+
in this process or in their containers, and the benchmark's - the same
|
|
5
|
+
ledger. Before a call goes out, its relay books what the call is likely to
|
|
6
|
+
cost (:func:`bound`). The call is sent only while what was spent, what the
|
|
7
|
+
calls in flight have booked and this booking stay within the cap. The first
|
|
8
|
+
call that does not fit is refused and closes the ledger: every later call is
|
|
9
|
+
refused too, so a run stops at one point instead of letting its small calls
|
|
10
|
+
through.
|
|
11
|
+
|
|
12
|
+
A reply turns the booking into what the call cost (``settle``). A call the
|
|
13
|
+
provider answered with an error gives its booking back (``release``). One
|
|
14
|
+
whose cost was never known - a reply with no token count, a reply cut off -
|
|
15
|
+
keeps its whole booking, as ``unsettled``: it may have been billed.
|
|
16
|
+
|
|
17
|
+
What was spent is each reply's real cost. Only a call in flight is an
|
|
18
|
+
estimate, since its cost is known when its reply comes back: the prompt's
|
|
19
|
+
characters over three as input tokens, and as output the average reply
|
|
20
|
+
the run has had from that model so far (the ledger keeps their count and
|
|
21
|
+
sum, ``seen``) - ``OUTPUT`` before any - and never more than the limit the
|
|
22
|
+
call names or the model's own (its price row's ``max_output``). The
|
|
23
|
+
model's limit alone would book a short answer at a hundred times its
|
|
24
|
+
cost, and a few calls at a time would fill a cap nothing had been spent
|
|
25
|
+
of; the largest reply so far would let one long answer do the same to
|
|
26
|
+
every call after it. A call that costs more than was booked for it takes
|
|
27
|
+
the spend past the cap by the difference, and the next call is refused.
|
|
28
|
+
|
|
29
|
+
A cap per unit (``run --max-usd-per-unit``) is kept in the same file: what
|
|
30
|
+
each unit spent and has in flight, beside the run's. A call that does not
|
|
31
|
+
fit its unit's cap is refused, and so is every later call of that unit; the
|
|
32
|
+
run goes on. A unit's calls are its own and its simulated user's - part of
|
|
33
|
+
the task - and never its judge's, which scores it (:func:`unit_of`).
|
|
34
|
+
|
|
35
|
+
A call that lacks room only because of calls in flight is not refused: its
|
|
36
|
+
relay waits for them to settle (``WAIT``), since their bookings are
|
|
37
|
+
estimates and mostly come back. Only a call that would not fit with nothing
|
|
38
|
+
in flight, or that waited in vain, closes the ledger.
|
|
39
|
+
|
|
40
|
+
Before any reply from its model, a call's output is a guess (``OUTPUT``),
|
|
41
|
+
and no call is refused on a guess: one whose request alone fits is booked
|
|
42
|
+
the room there is. The calls beside it wait for its reply, and are booked
|
|
43
|
+
from it. A cap under one guessed call would else refuse every call with
|
|
44
|
+
nothing spent. Nor is a unit refused before it has spent anything: for its
|
|
45
|
+
first call the average is other units' replies, a guess of its own, and a
|
|
46
|
+
unit refused on it would count against its agent with no call sent.
|
|
47
|
+
|
|
48
|
+
The file is rewritten whole under a lock held for one read and one write
|
|
49
|
+
(``flock`` on a file beside it, as an install is locked): a relay that dies
|
|
50
|
+
holding it lets it go.
|
|
51
|
+
"""
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import contextlib
|
|
55
|
+
import fcntl
|
|
56
|
+
import json
|
|
57
|
+
import math
|
|
58
|
+
import os
|
|
59
|
+
import uuid
|
|
60
|
+
from typing import Optional, Tuple
|
|
61
|
+
|
|
62
|
+
OUTPUT = 4096 # output tokens booked for a call before the run has had a reply from its model
|
|
63
|
+
ERROR = "SpendingCapError" # what a unit the cap ended carries: its own outcome outside the rate, never a miss (runs.classify)
|
|
64
|
+
# what a unit that reached its own cap carries: it had its attempt and spent its allowance, as one that spends its
|
|
65
|
+
# time does - an agent error, counted against the agent (runs.classify)
|
|
66
|
+
UNIT_ERROR = "UnitSpendingCapError"
|
|
67
|
+
WAIT = 120.0 # seconds a call waits for the calls in flight to settle before it is refused
|
|
68
|
+
_LIMITS = ("max_completion_tokens", "max_output_tokens", "max_tokens")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Unavailable(Exception):
|
|
72
|
+
"""The ledger could not be read or written: a capped call is then not sent."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def models(other: dict, model, judge, simulator, embed, control: bool = False) -> list:
|
|
76
|
+
"""Every model a run's relays may send a call to, as sent: the run's own, the judge's, the simulated
|
|
77
|
+
user's (never a control's: it holds no conversation), an option's embeddings and the side models of
|
|
78
|
+
``other`` (the agent, capability or setup). What a capped run must have a price for; the run and the
|
|
79
|
+
builder's Limits step both read it here."""
|
|
80
|
+
side = [(to or name) for name, _, to in (str(m).partition("=") for m in (other.get("model") or {}).get("side_models") or ())]
|
|
81
|
+
return [m for m in dict.fromkeys([model, judge, None if control else simulator, embed, *side]) if m]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def unit_of(tag: str) -> Optional[str]:
|
|
85
|
+
"""The unit a relay's route books for, by its tag (``run``): the unit's own calls and its simulated user's
|
|
86
|
+
(``<tag>-simulator``) are the task; its judge's (``<tag>-judge``) score it and count toward no unit."""
|
|
87
|
+
if tag.endswith("-judge"):
|
|
88
|
+
return None
|
|
89
|
+
return tag[:-len("-simulator")] if tag.endswith("-simulator") else tag
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def bound(req: dict, raw: bytes, row: dict, path: str = "", seen: Optional[int] = None, output: bool = True) -> float:
|
|
93
|
+
"""What a call is likely to cost, in USD, at ``row`` (a price row, USD
|
|
94
|
+
per million tokens). Input: the request's characters over three - its
|
|
95
|
+
JSON whole, less what it carries as base64 - at the dearer of the input
|
|
96
|
+
and the cache-write price. Output: ``seen``, the average reply the run
|
|
97
|
+
has had from this model, or ``OUTPUT`` before any; never more than the
|
|
98
|
+
limit the call names or the row's own. An embeddings call has none.
|
|
99
|
+
``output`` False: the request alone, which is no guess."""
|
|
100
|
+
tokens = math.ceil(max(0, len(raw) - _blobs(req)) / 3)
|
|
101
|
+
rate = max(row["input"], row.get("cache_write") or 0, row.get("cache_write_1h") or 0)
|
|
102
|
+
out = 0
|
|
103
|
+
if output and not path.split("?")[0].endswith("/embeddings"):
|
|
104
|
+
named = [req.get(k) for k in _LIMITS if isinstance(req.get(k), int) and not isinstance(req.get(k), bool) and req.get(k) > 0]
|
|
105
|
+
out = min(x for x in (named[0] if named else None, row.get("max_output"), seen or OUTPUT) if x)
|
|
106
|
+
return (tokens * rate + out * row["output"]) / 1e6
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _blobs(v) -> int:
|
|
110
|
+
"""Characters of a request that are an encoded file, not a prompt: a
|
|
111
|
+
``data:`` address, a block's base64 ``data``."""
|
|
112
|
+
if isinstance(v, str):
|
|
113
|
+
return len(v) if v.startswith("data:") else 0
|
|
114
|
+
if isinstance(v, list):
|
|
115
|
+
return sum(_blobs(x) for x in v)
|
|
116
|
+
if isinstance(v, dict):
|
|
117
|
+
if v.get("type") == "base64" and isinstance(v.get("data"), str):
|
|
118
|
+
return len(v["data"])
|
|
119
|
+
return sum(_blobs(x) for x in v.values())
|
|
120
|
+
return 0
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class Ledger:
|
|
124
|
+
def __init__(self, path: str):
|
|
125
|
+
self.path = path
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def create(cls, path: str, max_usd: Optional[float] = None, unit_max_usd: Optional[float] = None) -> "Ledger":
|
|
129
|
+
"""``max_usd``: the run's cap, or None for a run with a cap per unit alone. ``unit_max_usd``: each unit's."""
|
|
130
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
131
|
+
led = cls(path)
|
|
132
|
+
led._write({"max_usd": None if max_usd is None else float(max_usd),
|
|
133
|
+
"unit_max_usd": None if unit_max_usd is None else float(unit_max_usd),
|
|
134
|
+
"spent_usd": 0.0, "unsettled_usd": 0.0, "held": {}, "closed": False, "seen": {}, "units": {}, "of": {}})
|
|
135
|
+
return led
|
|
136
|
+
|
|
137
|
+
@contextlib.contextmanager
|
|
138
|
+
def _locked(self):
|
|
139
|
+
try:
|
|
140
|
+
with open(self.path + ".lock", "a") as f:
|
|
141
|
+
fcntl.flock(f, fcntl.LOCK_EX)
|
|
142
|
+
try:
|
|
143
|
+
yield self._read()
|
|
144
|
+
finally:
|
|
145
|
+
fcntl.flock(f, fcntl.LOCK_UN)
|
|
146
|
+
except (OSError, ValueError, KeyError, TypeError) as e:
|
|
147
|
+
raise Unavailable(f"the spending cap's ledger could not be used: {type(e).__name__}: {e}")
|
|
148
|
+
|
|
149
|
+
def _read(self) -> dict:
|
|
150
|
+
with open(self.path, encoding="utf-8") as f:
|
|
151
|
+
doc = json.load(f)
|
|
152
|
+
float(doc["max_usd"] or 0), float(doc["spent_usd"]), float(doc["unsettled_usd"]), sum(doc["held"].values())
|
|
153
|
+
doc["unit_max_usd"], dict(doc["seen"]), dict(doc["units"]), dict(doc["of"]) # all there, or it is no ledger
|
|
154
|
+
return doc
|
|
155
|
+
|
|
156
|
+
def _write(self, doc: dict) -> None:
|
|
157
|
+
"""Written whole beside the file, then renamed: a reader that takes no lock never finds half of it."""
|
|
158
|
+
part = f"{self.path}.{os.getpid()}.{uuid.uuid4().hex[:8]}.part"
|
|
159
|
+
with open(part, "w", encoding="utf-8") as f:
|
|
160
|
+
json.dump(doc, f)
|
|
161
|
+
os.replace(part, self.path)
|
|
162
|
+
|
|
163
|
+
def reserve(self, amount, unit: Optional[str] = None, model: Optional[str] = None, wait: bool = False,
|
|
164
|
+
least: Optional[float] = None) -> Tuple[Optional[str], dict]:
|
|
165
|
+
"""(the booking's id, the state before it) when ``amount`` fits;
|
|
166
|
+
(None, the state) when it does not, or the ledger is closed already -
|
|
167
|
+
and it is closed from then on. ``unit``: the unit the call is part
|
|
168
|
+
of (:func:`unit_of`): with a cap per unit, a call that does not fit
|
|
169
|
+
its unit's is refused (the state says ``by: unit``) and closes that
|
|
170
|
+
unit alone - the run's ledger stays open for the others. ``amount``
|
|
171
|
+
may be a function of what ``model``'s replies have been on average
|
|
172
|
+
(:func:`bound`): it is then read here, under the same lock, and
|
|
173
|
+
the state says the ``amount`` booked. ``wait``: a call that would
|
|
174
|
+
fit with nothing in flight is not refused but told to wait (``by:
|
|
175
|
+
wait``), and nothing is closed. ``least``: the part of ``amount``
|
|
176
|
+
that is no guess, the request's own (:func:`bound`): before any
|
|
177
|
+
reply from ``model``, a call whose ``least`` fits is booked the room
|
|
178
|
+
there is with nothing in flight, never refused on its guess; and so
|
|
179
|
+
is the call of a unit that has spent nothing, within its unit's cap."""
|
|
180
|
+
with self._locked() as doc:
|
|
181
|
+
usual = _usual(doc, model)
|
|
182
|
+
if callable(amount):
|
|
183
|
+
amount = amount(usual)
|
|
184
|
+
guess = least is not None and usual is None # no reply from its model yet: its output is a guess
|
|
185
|
+
each = doc["unit_max_usd"] if unit is not None else None
|
|
186
|
+
u = doc["units"].setdefault(unit, {"spent_usd": 0.0, "unsettled_usd": 0.0, "closed": False}) if each is not None else None
|
|
187
|
+
# a unit that has spent nothing: the average is other units' replies, a guess for its own first call
|
|
188
|
+
first = u is not None and least is not None and not u["spent_usd"] and not u["unsettled_usd"]
|
|
189
|
+
for cap, count in ((each, u), (doc["max_usd"], doc)):
|
|
190
|
+
room = None if cap is None else cap - count["spent_usd"] - count["unsettled_usd"]
|
|
191
|
+
if (guess or (first and count is u)) and room is not None and least <= room < amount:
|
|
192
|
+
amount = room # booked the room there is, never refused on the guess
|
|
193
|
+
seen = dict(_state(doc), by="cap", amount=amount)
|
|
194
|
+
if each is not None:
|
|
195
|
+
flight = sum(doc["held"].get(b, 0.0) for b, w in doc["of"].items() if w == unit) + u["unsettled_usd"]
|
|
196
|
+
seen["unit"] = {"max_usd": each, "spent_usd": u["spent_usd"], "held_usd": flight, "closed": u["closed"]}
|
|
197
|
+
if u["closed"] or amount > each - u["spent_usd"] - flight:
|
|
198
|
+
if wait and not u["closed"] and flight > u["unsettled_usd"] and amount <= each - u["spent_usd"] - u["unsettled_usd"]:
|
|
199
|
+
return None, dict(seen, by="wait") # only its own calls in flight are in the way
|
|
200
|
+
u["closed"] = True
|
|
201
|
+
self._write(doc)
|
|
202
|
+
return None, dict(seen, by="unit")
|
|
203
|
+
if doc["closed"] or amount > seen["left_usd"]:
|
|
204
|
+
if wait and not doc["closed"] and seen["held_usd"] > 0 and amount <= doc["max_usd"] - doc["spent_usd"] - doc["unsettled_usd"]:
|
|
205
|
+
return None, dict(seen, by="wait") # only the calls in flight are in the way
|
|
206
|
+
if not doc["closed"]:
|
|
207
|
+
doc["closed"] = True
|
|
208
|
+
self._write(doc)
|
|
209
|
+
return None, seen
|
|
210
|
+
booking = uuid.uuid4().hex[:12]
|
|
211
|
+
doc["held"][booking] = amount
|
|
212
|
+
if each is not None:
|
|
213
|
+
doc["of"][booking] = unit
|
|
214
|
+
self._write(doc)
|
|
215
|
+
return booking, seen
|
|
216
|
+
|
|
217
|
+
def settle(self, booking: str, cost: Optional[float], model: Optional[str] = None, output: Optional[int] = None) -> None:
|
|
218
|
+
"""The call's cost in place of its booking; None, its cost was never
|
|
219
|
+
known: the whole booking is kept, as unsettled. ``output``: the
|
|
220
|
+
reply's output tokens, added to what ``model`` has answered in this
|
|
221
|
+
run - its next calls are booked at their average."""
|
|
222
|
+
with self._locked() as doc:
|
|
223
|
+
amount = doc["held"].pop(booking, None)
|
|
224
|
+
if amount is None:
|
|
225
|
+
return
|
|
226
|
+
if model and isinstance(output, int) and not isinstance(output, bool) and output > 0:
|
|
227
|
+
replies, tokens = doc["seen"].get(model) or (0, 0)
|
|
228
|
+
doc["seen"][model] = [replies + 1, tokens + output]
|
|
229
|
+
unit = doc["units"].get(doc["of"].pop(booking, None))
|
|
230
|
+
if cost is None:
|
|
231
|
+
doc["unsettled_usd"] += amount
|
|
232
|
+
else:
|
|
233
|
+
doc["spent_usd"] += max(0.0, float(cost))
|
|
234
|
+
if unit is not None: # the same, on its unit's own count
|
|
235
|
+
unit["unsettled_usd" if cost is None else "spent_usd"] += amount if cost is None else max(0.0, float(cost))
|
|
236
|
+
self._write(doc)
|
|
237
|
+
|
|
238
|
+
def release(self, booking: str) -> None:
|
|
239
|
+
"""The booking given back: the provider answered with an error, or was never reached."""
|
|
240
|
+
with self._locked() as doc:
|
|
241
|
+
doc["of"].pop(booking, None)
|
|
242
|
+
if doc["held"].pop(booking, None) is not None:
|
|
243
|
+
self._write(doc)
|
|
244
|
+
|
|
245
|
+
def forget(self, unit: str) -> None:
|
|
246
|
+
"""A unit that ended: its own count goes, so the file stays the size of the units in flight."""
|
|
247
|
+
with self._locked() as doc:
|
|
248
|
+
if doc["units"].pop(unit, None) is not None:
|
|
249
|
+
self._write(doc)
|
|
250
|
+
|
|
251
|
+
def usual(self, model) -> Optional[int]:
|
|
252
|
+
"""The output tokens ``model`` has answered a call of this run with, on average; None before any. Read without the lock."""
|
|
253
|
+
try:
|
|
254
|
+
return _usual(self._read(), model)
|
|
255
|
+
except (OSError, ValueError, KeyError, TypeError) as e:
|
|
256
|
+
raise Unavailable(f"the spending cap's ledger could not be read: {type(e).__name__}: {e}")
|
|
257
|
+
|
|
258
|
+
def state(self) -> dict:
|
|
259
|
+
"""``max_usd``, ``spent_usd``, ``unsettled_usd``, ``held_usd`` (the
|
|
260
|
+
calls in flight), ``left_usd`` and ``closed``, read without the lock."""
|
|
261
|
+
try:
|
|
262
|
+
return _state(self._read())
|
|
263
|
+
except (OSError, ValueError, KeyError, TypeError) as e:
|
|
264
|
+
raise Unavailable(f"the spending cap's ledger could not be read: {type(e).__name__}: {e}")
|
|
265
|
+
|
|
266
|
+
def closed(self) -> bool:
|
|
267
|
+
"""Whether a call was refused: nothing more of the run is begun. A
|
|
268
|
+
ledger that cannot be read is closed - no call it guards is sent."""
|
|
269
|
+
try:
|
|
270
|
+
return bool(self.state()["closed"])
|
|
271
|
+
except Unavailable:
|
|
272
|
+
return True
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _usual(doc: dict, model) -> Optional[int]:
|
|
276
|
+
replies, tokens = doc["seen"].get(model) or (0, 0)
|
|
277
|
+
return math.ceil(tokens / replies) if replies else None
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _state(doc: dict) -> dict:
|
|
281
|
+
held = sum(doc["held"].values())
|
|
282
|
+
left = float("inf") if doc["max_usd"] is None else doc["max_usd"] - doc["spent_usd"] - doc["unsettled_usd"] - held # no cap on the run: a cap per unit alone
|
|
283
|
+
return {"max_usd": doc["max_usd"], "spent_usd": doc["spent_usd"], "unsettled_usd": doc["unsettled_usd"], "held_usd": held,
|
|
284
|
+
"left_usd": left, "closed": bool(doc["closed"])}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _places(v: float) -> int:
|
|
288
|
+
return min(12, max(4, 2 - math.floor(math.log10(abs(v))))) if v and math.isfinite(v) else 4
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def usd(v: float, places: Optional[int] = None) -> str:
|
|
292
|
+
"""An amount as a person reads it: cents, and the digits past them it has, to four decimals or to its third
|
|
293
|
+
figure where that is finer (0.395, 0.00202, 0.00005): a cap under a cent is not rounded to another.
|
|
294
|
+
``places``: to that many decimals instead."""
|
|
295
|
+
whole, _, cents = f"{v:.{_places(v) if places is None else places}f}".rstrip("0").partition(".")
|
|
296
|
+
return f"{whole}.{cents.ljust(2, '0')}"
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _together(cap: float, *amounts: float) -> list:
|
|
300
|
+
"""Amounts that pass ``cap`` together, printed so that they still do: all to the decimals the finest of
|
|
301
|
+
them needs, and to more while their rounding hides it. 0.0118 spent and a call of 0.000157 would read as
|
|
302
|
+
fitting a cap of 0.012 that 0.011849 and 0.000157 pass."""
|
|
303
|
+
places = max(_places(a) for a in amounts)
|
|
304
|
+
while places < 12 and sum(round(a, places) for a in amounts) <= cap < sum(amounts):
|
|
305
|
+
places += 1
|
|
306
|
+
return [usd(a, places) for a in amounts]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def refusal(state: dict, amount: float) -> str:
|
|
310
|
+
"""Why a call was not sent, as its caller and the record read it."""
|
|
311
|
+
if state.get("by") == "unit":
|
|
312
|
+
u = state["unit"]
|
|
313
|
+
if u["closed"]:
|
|
314
|
+
return (f"an earlier call of this unit did not fit its spending cap of {usd(u['max_usd'])} USD "
|
|
315
|
+
f"({usd(u['spent_usd'])} spent by the unit): no further call of this unit is sent")
|
|
316
|
+
spent, held, call = _together(u["max_usd"], u["spent_usd"], u["held_usd"], amount)
|
|
317
|
+
return (f"this unit's spending cap of {usd(u['max_usd'])} USD: {spent} spent by the unit"
|
|
318
|
+
+ (f", {held} booked by its calls in flight or never settled" if u["held_usd"] else "")
|
|
319
|
+
+ f", and this call is counted at {call}; it was not sent, and no further call of this unit is")
|
|
320
|
+
if state["closed"]:
|
|
321
|
+
return (f"an earlier call did not fit the run's spending cap of {usd(state['max_usd'])} USD "
|
|
322
|
+
f"({usd(state['spent_usd'])} spent): no further call of this run is sent")
|
|
323
|
+
spent, held, call = _together(state["max_usd"], state["spent_usd"], state["held_usd"] + state["unsettled_usd"], amount)
|
|
324
|
+
return (f"the run's spending cap of {usd(state['max_usd'])} USD: {spent} spent"
|
|
325
|
+
+ (f", {held} booked by calls in flight or never settled" if state["held_usd"] + state["unsettled_usd"] else "")
|
|
326
|
+
+ f", and this call is counted at {call}; it was not sent, and no further call of this run is")
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Which pairs fit, and why the others do not.
|
|
2
|
+
|
|
3
|
+
A benchmark of the agent family takes an agent, one of the capability family a
|
|
4
|
+
capability, and the pair fits when the other side offers - and has proven -
|
|
5
|
+
every port the benchmark needs - and, for a port whose kit has skills, has
|
|
6
|
+
shown each skill the benchmark relies on (``needs_skills``). An agent with
|
|
7
|
+
``only_for`` - a benchmark's own reference agent - fits those benchmarks
|
|
8
|
+
alone, whatever it has proven. The reasons are sentences a page can show
|
|
9
|
+
beside a greyed-out choice.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Dict, List
|
|
14
|
+
|
|
15
|
+
from nullius.standard import ports
|
|
16
|
+
from nullius.standard.definitions import CONTROLS, port_skills, ref_of, skills_shown
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def fit(bench: dict, other: dict) -> List[str]:
|
|
20
|
+
"""Why ``other`` (an agent or a capability definition, or a control) cannot
|
|
21
|
+
run ``bench``; empty when it can. A control fits the benchmark that
|
|
22
|
+
declares it, whatever its family: it needs no port, it answers itself."""
|
|
23
|
+
side = bench.get("family")
|
|
24
|
+
kind = other.get("_kind") or ("agent" if other.get("schema", "").startswith("nullius.agent") else "capability")
|
|
25
|
+
if kind == "upstream":
|
|
26
|
+
return [] if other.get("key") in (bench.get("setups") or {}) else [f"{bench['label']} has no {other.get('key')} setup"]
|
|
27
|
+
if kind == "control":
|
|
28
|
+
declared = (bench.get("controls") or {}).get(other.get("key"))
|
|
29
|
+
return [] if declared else [f"{bench['label']} declares no {other.get('key')} control"]
|
|
30
|
+
if kind != side:
|
|
31
|
+
return [f"{bench['label']} takes {'an agent' if side == 'agent' else 'a capability'}, and {other['label']} is {'an agent' if kind == 'agent' else 'a capability'}"]
|
|
32
|
+
offers, proven = set(other.get("offers") or []), other.get("proven") or {}
|
|
33
|
+
version = str((other.get("upstream") or {}).get("version") or "")
|
|
34
|
+
name = f"{other['label']} {version}" if "@" in str(other.get("_ref") or "") else other["label"]
|
|
35
|
+
only = other.get("only_for") or []
|
|
36
|
+
if only and bench.get("key") not in only:
|
|
37
|
+
return [f"{name} is the reference agent of {', '.join(only)}, and runs no other benchmark"]
|
|
38
|
+
why = []
|
|
39
|
+
for p in bench.get("needs") or []:
|
|
40
|
+
on = str((proven.get(p) or {}).get("version") or "")
|
|
41
|
+
if p not in offers:
|
|
42
|
+
why.append(f"needs {p} ({ports.PORTS.get(p, {}).get('summary', '')}), which {name} does not offer")
|
|
43
|
+
elif p not in proven:
|
|
44
|
+
why.append(f"needs {p}, which {name} claims but has not passed its kit")
|
|
45
|
+
elif on and version and on != version:
|
|
46
|
+
why.append(f"needs {p}, which {name} passed on version {on}, not {version}: prove it again")
|
|
47
|
+
else:
|
|
48
|
+
shown, skills = skills_shown(p, proven[p]), port_skills(p)
|
|
49
|
+
why += [f"{name} has not shown the {s} skill of {p}" + (f" ({skills[s]})" if s in skills else "")
|
|
50
|
+
for s in (bench.get("needs_skills") or {}).get(p) or [] if s not in shown]
|
|
51
|
+
return why
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def table(benches: List[dict], agents: List[dict], capabilities: List[dict]) -> Dict[str, Dict[str, List[str]]]:
|
|
55
|
+
"""For every benchmark, every agent or capability of its family - each
|
|
56
|
+
version by its ``key@version`` when it has several - and the reasons it
|
|
57
|
+
does not fit (empty: offered); then the controls it declares, ``nop`` and
|
|
58
|
+
``oracle``, which always fit. A benchmark is keyed as an agent is: its
|
|
59
|
+
key for its default version, ``key@version`` for another."""
|
|
60
|
+
out = {}
|
|
61
|
+
for b in benches:
|
|
62
|
+
others = agents if b.get("family") == "agent" else capabilities
|
|
63
|
+
k = ref_of(b) if "_version" in b else b["key"]
|
|
64
|
+
out[k] = {(o.get("_ref") or o["key"]): fit(b, o) for o in others}
|
|
65
|
+
out[k].update({c: [] for c in CONTROLS if c in (b.get("controls") or {})})
|
|
66
|
+
return out
|