nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "swe-bench-verified",
|
|
4
|
+
"label": "SWE-bench Verified",
|
|
5
|
+
"summary": "Each instance is a GitHub issue and the authors' container for it, the repository at the issue's base commit in /testbed; the agent fixes it there, and the harness's own eval script and grading say whether the failing tests now pass and the passing ones still do.",
|
|
6
|
+
"family": "agent",
|
|
7
|
+
"needs": [
|
|
8
|
+
"agent.workspace/v1"
|
|
9
|
+
],
|
|
10
|
+
"upstream": {
|
|
11
|
+
"repository": "https://github.com/SWE-bench/SWE-bench",
|
|
12
|
+
"tag": "v5.0.1",
|
|
13
|
+
"tested": "87ab1f6ced28f75ba73ca899dc759b019310944a",
|
|
14
|
+
"tested_date": "2026-10-23",
|
|
15
|
+
"license": "MIT"
|
|
16
|
+
},
|
|
17
|
+
"driver": {
|
|
18
|
+
"command": [
|
|
19
|
+
"{install}/.venv/bin/python",
|
|
20
|
+
"{here}/driver.py"
|
|
21
|
+
],
|
|
22
|
+
"install": {
|
|
23
|
+
"python": "3.12",
|
|
24
|
+
"pip": [
|
|
25
|
+
"swebench==5.0.1"
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"datasets": [
|
|
30
|
+
{
|
|
31
|
+
"id": "verified",
|
|
32
|
+
"label": "SWE-bench Verified",
|
|
33
|
+
"url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified/resolve/78f471bf655a3137b2e8a75af1501690ec009ec3/data/test-00000-of-00001.parquet",
|
|
34
|
+
"sha256": "030cfd7f2a704c4c0226e7f104c725a3b41230b1d3517f9c915ad7ea5be3fa25",
|
|
35
|
+
"units": 500,
|
|
36
|
+
"environment": "task"
|
|
37
|
+
}
|
|
38
|
+
],
|
|
39
|
+
"units": {
|
|
40
|
+
"noun": "instance",
|
|
41
|
+
"nouns": "instances"
|
|
42
|
+
},
|
|
43
|
+
"environments": {
|
|
44
|
+
"task": {
|
|
45
|
+
"summary": "The authors' prebuilt image for the instance (swebench/sweb.eval.x86_64.<instance>), pulled by the digest its tag had when pinned here, with the agent added at run time.",
|
|
46
|
+
"image": "own"
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
"controls": {
|
|
50
|
+
"nop": {
|
|
51
|
+
"source": "nullius",
|
|
52
|
+
"from": "Nullius: no agent runs; the instance's tests run on its container as its image left it."
|
|
53
|
+
},
|
|
54
|
+
"oracle": {
|
|
55
|
+
"source": "official",
|
|
56
|
+
"from": "The instance's gold patch, the pull request that closed the issue minus its tests, applied in the container with git apply."
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"score": {
|
|
60
|
+
"kind": "pass",
|
|
61
|
+
"says": "the harness's own eval script, run in the instance's container after the agent ends, and its grading: every FAIL_TO_PASS test passes and every PASS_TO_PASS test still passes"
|
|
62
|
+
},
|
|
63
|
+
"limits": {
|
|
64
|
+
"turn_seconds": 1800
|
|
65
|
+
},
|
|
66
|
+
"proven": {
|
|
67
|
+
"verified": {
|
|
68
|
+
"units": {
|
|
69
|
+
"units": 500,
|
|
70
|
+
"passed": "500 ids, the same twice, unique",
|
|
71
|
+
"on": "2026-10-23"
|
|
72
|
+
},
|
|
73
|
+
"leak": {
|
|
74
|
+
"units": 2,
|
|
75
|
+
"looked_for": 0,
|
|
76
|
+
"short": 0,
|
|
77
|
+
"passed": "2/2",
|
|
78
|
+
"note": "2 unit(s) whose oracle gave nothing to look for",
|
|
79
|
+
"on": "2026-10-23",
|
|
80
|
+
"limit": 2
|
|
81
|
+
},
|
|
82
|
+
"nop": {
|
|
83
|
+
"passed": "0/2",
|
|
84
|
+
"value": 0.0,
|
|
85
|
+
"expect": {
|
|
86
|
+
"at_most": 0.0
|
|
87
|
+
},
|
|
88
|
+
"units": 2,
|
|
89
|
+
"job": "2026-10-23__15-24-36__swe-bench-verified__verified__prove-nop",
|
|
90
|
+
"source": "nullius",
|
|
91
|
+
"on": "2026-10-23",
|
|
92
|
+
"limit": 2
|
|
93
|
+
},
|
|
94
|
+
"oracle": {
|
|
95
|
+
"passed": "2/2",
|
|
96
|
+
"value": 1.0,
|
|
97
|
+
"expect": {
|
|
98
|
+
"at_least": 1.0
|
|
99
|
+
},
|
|
100
|
+
"units": 2,
|
|
101
|
+
"job": "2026-10-23__15-24-36__swe-bench-verified__verified__prove-oracle",
|
|
102
|
+
"source": "official",
|
|
103
|
+
"on": "2026-10-23",
|
|
104
|
+
"limit": 2
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-d/--dataset_name", "values": "SWE-bench/SWE-bench_Verified, SWE-bench/SWE-bench, SWE-bench/SWE-bench_Lite, SWE-bench/SWE-bench_Multimodal, a local file", "default": "SWE-bench/SWE-bench_Lite",
|
|
3
|
+
"nullius": "dataset:verified"},
|
|
4
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-d/--dataset_name: the full, Lite and Multimodal sets", "values": "", "default": "",
|
|
5
|
+
"nullius": "not offered - this entry is SWE-bench Verified, the human-validated 500; the other sets are benchmarks of their own"},
|
|
6
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-s/--split", "values": "test", "default": "test",
|
|
7
|
+
"nullius": "fixed:test - the only split the Verified dataset has"},
|
|
8
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-i/--instance_ids", "values": "any subset", "default": "all",
|
|
9
|
+
"nullius": "fixed:the run's units - cli run --limit, the builder's Tasks, in the dataset's order"},
|
|
10
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-p/--predictions_path (a patch per instance, or gold)", "values": "a predictions file, gold", "default": "",
|
|
11
|
+
"nullius": "fixed:the agent's changes, made in the container and tested there - never a patch re-applied in a fresh container; gold is the oracle control; said in the limitations"},
|
|
12
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-t/--timeout (one instance's tests)", "values": "seconds", "default": "1800",
|
|
13
|
+
"nullius": "fixed:1800 - the harness's own default, the check's seconds"},
|
|
14
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "--max_workers", "values": "", "default": "4",
|
|
15
|
+
"nullius": "fixed:the run's concurrency - how many units run at once, not what is measured"},
|
|
16
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "--open_file_limit", "values": "", "default": "4096",
|
|
17
|
+
"nullius": "not offered - a limit of the harness's own process, not of what is measured"},
|
|
18
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "-id/--run_id", "values": "", "default": "",
|
|
19
|
+
"nullius": "fixed:the job's name - the folder a run writes, the same role"},
|
|
20
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "--report_dir, --rewrite_reports", "values": "", "default": ". / false",
|
|
21
|
+
"nullius": "fixed:the trial's verifier/ folder - the harness's report.json written there once, by its own grading"},
|
|
22
|
+
{"entry": "python -m swebench.harness.run_evaluation", "setting": "--modal", "values": "true, false", "default": "false",
|
|
23
|
+
"nullius": "not offered - where the containers run, not what is measured; the standard's own container backend runs them"},
|
|
24
|
+
{"entry": "the dataset", "setting": "image (swebench/sweb.eval.x86_64.<instance>:latest)", "values": "the authors' prebuilt image per instance", "default": "latest",
|
|
25
|
+
"nullius": "fixed:the authors' image - pulled by the digest its latest tag had when pinned here (images.json), so a re-push does not move the measure"},
|
|
26
|
+
{"entry": "the dataset", "setting": "eval_script, log_parser, eval_type", "values": "per instance", "default": "per instance",
|
|
27
|
+
"nullius": "fixed:the dataset's own - the script written into the container and run there, the parser the harness names reading its output"},
|
|
28
|
+
{"entry": "the dataset", "setting": "problem_statement as the task", "values": "", "default": "",
|
|
29
|
+
"nullius": "fixed:verbatim - the issue's title and body as the prompt, with no template of ours; an agent's own prompt wraps it as that agent does"},
|
|
30
|
+
{"entry": "the dataset", "setting": "hints_text (comments on the issue before the fix)", "values": "", "default": "not shown",
|
|
31
|
+
"nullius": "fixed:not shown - as the leaderboard runs it; said in the limitations"}
|
|
32
|
+
]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "tau3-bench",
|
|
3
|
+
"title": "τ³-bench",
|
|
4
|
+
"category": "Customer service",
|
|
5
|
+
"domain": "Customer service with domain tools and a simulated user: airline, retail, telecom, banking.",
|
|
6
|
+
"scoring": "τ² evaluator",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard",
|
|
9
|
+
"harbor"
|
|
10
|
+
],
|
|
11
|
+
"origin": "official",
|
|
12
|
+
"good_to_know": [
|
|
13
|
+
"Scores depend on the simulated user's and the evaluator's models: keep both fixed when comparing.",
|
|
14
|
+
"Conversations run in the upstream's own runner, release v1.0.1, at its defaults: 200 steps, 10 tool errors, seed 300.",
|
|
15
|
+
"Original is the upstream's own agent with your model, as in a leaderboard submission.",
|
|
16
|
+
"Several attempts per task report pass^k, as the leaderboard's 4 trials do. Each attempt is a new conversation.",
|
|
17
|
+
"A task whose retries all fail is an infrastructure error, not a score.",
|
|
18
|
+
"A tool call after the conversation has ended returns an error. An agent that keeps calling until its time runs out is not scored."
|
|
19
|
+
],
|
|
20
|
+
"upstream_limitations": [
|
|
21
|
+
"The agent must answer the user through the user tool, or the episode ends early.",
|
|
22
|
+
"Tool calls must be one at a time: the evaluator refuses concurrent ones."
|
|
23
|
+
],
|
|
24
|
+
"limitations": [
|
|
25
|
+
"Voice mode is not offered.",
|
|
26
|
+
"banking_knowledge's search method is an option, bm25 by default. Scores compare only between runs with the same search. Reranker methods use gpt-5.2 through the relay. Qwen methods require OpenRouter.",
|
|
27
|
+
"Another agent replaces the upstream's agent. It gets the upstream's instructions and policy, plus two lines from Nullius: where its tools are, and that its last message in a turn goes to the user."
|
|
28
|
+
],
|
|
29
|
+
"datasets": [
|
|
30
|
+
{
|
|
31
|
+
"key": "tau3-bench",
|
|
32
|
+
"title": "τ³-bench",
|
|
33
|
+
"tasks": 375,
|
|
34
|
+
"official": "https://github.com/sierra-research/tau2-bench",
|
|
35
|
+
"registry": "sierra-research/tau3-bench",
|
|
36
|
+
"hub": "sierra-research/tau3-bench",
|
|
37
|
+
"verifier": "τ²'s evaluator: the database state and the conversation's assertions.",
|
|
38
|
+
"source": "Harbor hub"
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
"timeouts": {
|
|
42
|
+
"env_setup": 600,
|
|
43
|
+
"agent_setup": 1200,
|
|
44
|
+
"agent_run": 3600,
|
|
45
|
+
"verify": 300
|
|
46
|
+
},
|
|
47
|
+
"citation": "@misc{barres2025tau2,\n title={$\\tau^2$-Bench: Evaluating Conversational Agents in a Dual-Control Environment},\n author={Victor Barres and Honghua Dong and Soham Ray and Xujie Si and Karthik Narasimhan},\n year={2025},\n eprint={2506.07982},\n archivePrefix={arXiv},\n primaryClass={cs.AI},\n url={https://arxiv.org/abs/2506.07982},\n}\n\n@misc{yao2024tau,\n title={$\\tau$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains},\n author={Shunyu Yao and Noah Shinn and Pedram Razavi and Karthik Narasimhan},\n year={2024},\n eprint={2406.12045},\n archivePrefix={arXiv},\n primaryClass={cs.AI},\n url={https://arxiv.org/abs/2406.12045},\n}\n\n@article{shi2026tau,\n title={$\\tau$-Knowledge: Evaluating Conversational Agents over Unstructured Knowledge},\n author={Shi, Quan and Zytek, Alexandra and Razavi, Pedram and Narasimhan, Karthik and Barres, Victor},\n journal={arXiv preprint arXiv:2603.04370},\n year={2026}\n}",
|
|
48
|
+
"citation_source": "https://github.com/sierra-research/tau2-bench"
|
|
49
|
+
}
|