nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"entry": "GAIA dataset card",
|
|
4
|
+
"setting": "2023_all / validation",
|
|
5
|
+
"values": "metadata.parquet",
|
|
6
|
+
"default": "validation",
|
|
7
|
+
"nullius": "dataset:gaia-validation"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"entry": "GAIA dataset card",
|
|
11
|
+
"setting": "2023_level1 / validation",
|
|
12
|
+
"values": "metadata.level1.parquet",
|
|
13
|
+
"default": "validation",
|
|
14
|
+
"nullius": "dataset:gaia-validation-level1"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"entry": "GAIA dataset card",
|
|
18
|
+
"setting": "2023_level2 / validation",
|
|
19
|
+
"values": "metadata.level2.parquet",
|
|
20
|
+
"default": "validation",
|
|
21
|
+
"nullius": "dataset:gaia-validation-level2"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"entry": "GAIA dataset card",
|
|
25
|
+
"setting": "2023_level3 / validation",
|
|
26
|
+
"values": "metadata.level3.parquet",
|
|
27
|
+
"default": "validation",
|
|
28
|
+
"nullius": "dataset:gaia-validation-level3"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"entry": "GAIA dataset card",
|
|
32
|
+
"setting": "test split",
|
|
33
|
+
"values": "301 questions; answers withheld",
|
|
34
|
+
"default": "test",
|
|
35
|
+
"nullius": "not offered - reference answers are private and the local scorer cannot evaluate them"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"entry": "GAIA dataset card",
|
|
39
|
+
"setting": "revision",
|
|
40
|
+
"values": "immutable dataset revision",
|
|
41
|
+
"default": "682dd723ee1e1697e00360edccf2366dc8418dd9",
|
|
42
|
+
"nullius": "fixed:682dd723ee1e1697e00360edccf2366dc8418dd9 - pinned data and attachments"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"entry": "GAIA dataset card",
|
|
46
|
+
"setting": "loader",
|
|
47
|
+
"values": "Hugging Face datasets; Parquet metadata and relative attachment paths",
|
|
48
|
+
"default": "Parquet",
|
|
49
|
+
"nullius": "fixed:Hugging Face Parquet loader - reads the published split without a rewritten parser"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"entry": "content.py",
|
|
53
|
+
"setting": "formatting prompt",
|
|
54
|
+
"values": "SUBMISSION_TEXT first fenced block",
|
|
55
|
+
"default": "published prompt",
|
|
56
|
+
"nullius": "fixed:imported verbatim - official formatting instructions"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"entry": "scorer.py",
|
|
60
|
+
"setting": "answer type dispatch",
|
|
61
|
+
"values": "number, ordered list, normalized string",
|
|
62
|
+
"default": "ground truth determines type",
|
|
63
|
+
"nullius": "fixed:question_scorer - imported unchanged from pinned upstream"
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"entry": "scorer.py",
|
|
67
|
+
"setting": "numeric normalization",
|
|
68
|
+
"values": "remove dollar sign, percent sign and commas; compare floats",
|
|
69
|
+
"default": "official implementation",
|
|
70
|
+
"nullius": "fixed:official normalization - no local scorer implementation"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"entry": "scorer.py",
|
|
74
|
+
"setting": "list separators and order",
|
|
75
|
+
"values": "comma or semicolon; compare elements in order",
|
|
76
|
+
"default": "comma and semicolon",
|
|
77
|
+
"nullius": "fixed:official ordered-list comparison - retain punctuation in string list elements"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"entry": "scorer.py",
|
|
81
|
+
"setting": "string normalization",
|
|
82
|
+
"values": "lowercase; remove whitespace and optionally punctuation",
|
|
83
|
+
"default": "remove_punct=True",
|
|
84
|
+
"nullius": "fixed:official normalization - called by question_scorer"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"entry": "paper section 4",
|
|
88
|
+
"setting": "no-tools API procedure",
|
|
89
|
+
"values": "GPT-4, GPT-4 Turbo",
|
|
90
|
+
"default": "published prefix then question",
|
|
91
|
+
"nullius": "setup:original"
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"entry": "paper section 4",
|
|
95
|
+
"setting": "API repetitions",
|
|
96
|
+
"values": "3 runs; mean score",
|
|
97
|
+
"default": "3",
|
|
98
|
+
"nullius": "fixed:runner repeat control - choose three repeats to match the paper; this entry does not change the runner default"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"entry": "paper section 4",
|
|
102
|
+
"setting": "generation temperature",
|
|
103
|
+
"values": "not specified",
|
|
104
|
+
"default": "not specified",
|
|
105
|
+
"nullius": "option:temperature"
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"entry": "paper section 4",
|
|
109
|
+
"setting": "ChatGPT plugins and Advanced Data Analysis",
|
|
110
|
+
"values": "manual choice of mode or up to three plugins for each question",
|
|
111
|
+
"default": "manual",
|
|
112
|
+
"nullius": "not offered - no API or stable per-question configuration published for the historical procedure"
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"entry": "paper section 4",
|
|
116
|
+
"setting": "AutoGPT",
|
|
117
|
+
"values": "ed172dec1947466cc0942abf75bb77b027cd433d with GPT-4",
|
|
118
|
+
"default": "paper commit",
|
|
119
|
+
"nullius": "not offered - this import evaluates the chosen Nullius agent or direct model, not the historical AutoGPT stack"
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
"entry": "driver.py (Nullius)",
|
|
123
|
+
"setting": "final-answer extraction",
|
|
124
|
+
"values": "text after the last literal FINAL ANSWER:, else the whole reply; trim whitespace; fail empty",
|
|
125
|
+
"default": "marker when present",
|
|
126
|
+
"nullius": "fixed:Nullius extraction - authorized adaptation around the unchanged scorer"
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"entry": "driver.py (Nullius)",
|
|
130
|
+
"setting": "attachment staging",
|
|
131
|
+
"values": "agent workspace; original no-tools baseline has no attachment input",
|
|
132
|
+
"default": "unmodified file",
|
|
133
|
+
"nullius": "fixed:one question’s attachment only - keep answers and annotator metadata outside agent workspaces"
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"entry": "driver.py (Nullius)",
|
|
137
|
+
"setting": "agent prompt role",
|
|
138
|
+
"values": "published prefix and question delivered together over ACP",
|
|
139
|
+
"default": "ACP user prompt",
|
|
140
|
+
"nullius": "fixed:agent protocol - agent system instructions stay its own"
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"entry": "driver.py (Nullius)",
|
|
144
|
+
"setting": "HF_TOKEN",
|
|
145
|
+
"values": "driver-only credential",
|
|
146
|
+
"default": "Settings → Data providers → Hugging Face",
|
|
147
|
+
"nullius": "fixed:Settings → Data providers → Hugging Face - the driver alone receives it"
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"entry": "driver.py (Nullius)",
|
|
151
|
+
"setting": "cache",
|
|
152
|
+
"values": "pinned Hugging Face artifacts under upstream/.gaia-cache",
|
|
153
|
+
"default": "shared driver cache",
|
|
154
|
+
"nullius": "fixed:shared pinned cache - outside agent workspace and catalogue payload"
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"entry": "app.py",
|
|
158
|
+
"setting": "TOKEN",
|
|
159
|
+
"values": "Space service credential",
|
|
160
|
+
"default": "Space service credential",
|
|
161
|
+
"nullius": "not offered - the local driver uses its declared HF_TOKEN and never accesses internal repositories"
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"entry": "app.py",
|
|
165
|
+
"setting": "OWNER",
|
|
166
|
+
"values": "gaia-benchmark",
|
|
167
|
+
"default": "gaia-benchmark",
|
|
168
|
+
"nullius": "fixed:gaia-benchmark - organisation constant"
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
"entry": "app.py",
|
|
172
|
+
"setting": "DATA_DATASET",
|
|
173
|
+
"values": "gaia-benchmark/GAIA",
|
|
174
|
+
"default": "gaia-benchmark/GAIA",
|
|
175
|
+
"nullius": "fixed:gaia-benchmark/GAIA - public gated dataset used by this import"
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
"entry": "app.py",
|
|
179
|
+
"setting": "INTERNAL_DATA_DATASET",
|
|
180
|
+
"values": "gaia-benchmark/GAIA_internal",
|
|
181
|
+
"default": "gaia-benchmark/GAIA_internal",
|
|
182
|
+
"nullius": "not offered - private reference answers"
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"entry": "app.py",
|
|
186
|
+
"setting": "SUBMISSION_DATASET",
|
|
187
|
+
"values": "gaia-benchmark/submissions_internal",
|
|
188
|
+
"default": "gaia-benchmark/submissions_internal",
|
|
189
|
+
"nullius": "not offered - online publication is not part of a run"
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
"entry": "app.py",
|
|
193
|
+
"setting": "SUBMISSION_DATASET_PUBLIC",
|
|
194
|
+
"values": "gaia-benchmark/submissions_public",
|
|
195
|
+
"default": "gaia-benchmark/submissions_public",
|
|
196
|
+
"nullius": "not offered - online publication is not part of a run"
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
"entry": "app.py",
|
|
200
|
+
"setting": "CONTACT_DATASET",
|
|
201
|
+
"values": "gaia-benchmark/contact_info",
|
|
202
|
+
"default": "gaia-benchmark/contact_info",
|
|
203
|
+
"nullius": "not offered - online account records are not part of a run"
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"entry": "app.py",
|
|
207
|
+
"setting": "RESULTS_DATASET",
|
|
208
|
+
"values": "gaia-benchmark/results_public",
|
|
209
|
+
"default": "gaia-benchmark/results_public",
|
|
210
|
+
"nullius": "not offered - online leaderboard display is not part of a run"
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
"entry": "app.py",
|
|
214
|
+
"setting": "LEADERBOARD_PATH",
|
|
215
|
+
"values": "gaia-benchmark/leaderboard",
|
|
216
|
+
"default": "gaia-benchmark/leaderboard",
|
|
217
|
+
"nullius": "not offered - online leaderboard display is not part of a run"
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
"entry": "app.py",
|
|
221
|
+
"setting": "YEAR_VERSION",
|
|
222
|
+
"values": "2023",
|
|
223
|
+
"default": "2023",
|
|
224
|
+
"nullius": "fixed:2023 - only published year"
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"entry": "app.py",
|
|
228
|
+
"setting": "ref_scores_len",
|
|
229
|
+
"values": "validation 165; test 301",
|
|
230
|
+
"default": "validation 165; test 301",
|
|
231
|
+
"nullius": "fixed:validation 165; test 301 - published full-split counts"
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
"entry": "app.py",
|
|
235
|
+
"setting": "ref_level_len",
|
|
236
|
+
"values": "validation 53/86/26; test 93/159/49",
|
|
237
|
+
"default": "validation 53/86/26; test 93/159/49",
|
|
238
|
+
"nullius": "fixed:validation 53/86/26; test 93/159/49 - published per-level counts"
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
"entry": "app.py",
|
|
242
|
+
"setting": "LOCAL_DEBUG",
|
|
243
|
+
"values": "False",
|
|
244
|
+
"default": "False",
|
|
245
|
+
"nullius": "not offered - Space upload/debug behavior"
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
"entry": "app.py",
|
|
249
|
+
"setting": "submission fields",
|
|
250
|
+
"values": "model, model_family, system_prompt, url, file, organisation, mail, OAuth profile",
|
|
251
|
+
"default": "model, model_family, system_prompt, url, file, organisation, mail, OAuth profile",
|
|
252
|
+
"nullius": "not offered - online submission metadata"
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
"entry": "app.py",
|
|
256
|
+
"setting": "submission split",
|
|
257
|
+
"values": "test",
|
|
258
|
+
"default": "test",
|
|
259
|
+
"nullius": "not offered - validation leaderboard is closed"
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
"entry": "app.py",
|
|
263
|
+
"setting": "submission checks",
|
|
264
|
+
"values": "account age 60 days; one submission/day; unique task ids; complete counts; reject all-perfect split scores",
|
|
265
|
+
"default": "account age 60 days; one submission/day; unique task ids; complete counts; reject all-perfect split scores",
|
|
266
|
+
"nullius": "not offered - online anti-abuse controls, not local per-question scoring"
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
"entry": "app.py",
|
|
270
|
+
"setting": "UI options",
|
|
271
|
+
"values": "top 100; percentage display; Gradio launch debug=True",
|
|
272
|
+
"default": "top 100; percentage display; Gradio launch debug=True",
|
|
273
|
+
"nullius": "not offered - leaderboard interface"
|
|
274
|
+
}
|
|
275
|
+
]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "gsm8k",
|
|
3
|
+
"title": "GSM8K",
|
|
4
|
+
"category": "Math",
|
|
5
|
+
"domain": "Grade-school math word problems that take two to eight steps of basic arithmetic, each with a single numeric answer.",
|
|
6
|
+
"scoring": "Solve rate (exact final answer)",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard"
|
|
9
|
+
],
|
|
10
|
+
"origin": "official",
|
|
11
|
+
"limitations": [
|
|
12
|
+
"The prompt asking for the '#### <number>' line is Nullius's wording. The paper's models were fine-tuned to write it.",
|
|
13
|
+
"The paper's trained verifier, which picks the best of several answers, is not reproduced. Several attempts per problem report pass@k instead."
|
|
14
|
+
],
|
|
15
|
+
"datasets": [
|
|
16
|
+
{
|
|
17
|
+
"key": "test",
|
|
18
|
+
"title": "GSM8K test",
|
|
19
|
+
"tasks": 1319,
|
|
20
|
+
"official": "https://github.com/openai/grade-school-math",
|
|
21
|
+
"source": "GitHub",
|
|
22
|
+
"registry": "",
|
|
23
|
+
"hub": "",
|
|
24
|
+
"verifier": "The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer.",
|
|
25
|
+
"notes": "1,319 problems. MIT License."
|
|
26
|
+
}
|
|
27
|
+
],
|
|
28
|
+
"citation": "@article{cobbe2021gsm8k,\n title={Training Verifiers to Solve Math Word Problems},\n author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},\n journal={arXiv preprint arXiv:2110.14168},\n year={2021}\n}",
|
|
29
|
+
"citation_source": "https://github.com/openai/grade-school-math",
|
|
30
|
+
"good_to_know": [
|
|
31
|
+
"Answers must end with a line '#### <number>'. The number is compared as text, so 18.00 does not match 18.",
|
|
32
|
+
"The test split is run. The Socratic variant has the same questions and answers, with different worked explanations."
|
|
33
|
+
]
|
|
34
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""GSM8K's driver for the Nullius standard (JSON lines on stdin and stdout).
|
|
2
|
+
|
|
3
|
+
units - one per problem of the test file, named by its line number ("0001")
|
|
4
|
+
setup - the question goes in the prompt; the workspace stays empty
|
|
5
|
+
score - the upstream's own extract_answer and is_correct, imported from
|
|
6
|
+
grade_school_math/dataset.py at the tested commit ($NULLIUS_UPSTREAM),
|
|
7
|
+
applied to the agent's last message
|
|
8
|
+
oracle - the reference control: the problem's own worked solution, the test
|
|
9
|
+
file's "answer" field, which ends in '#### <number>'
|
|
10
|
+
answer - the original setup: the problem asked of a model in one call through
|
|
11
|
+
the unit's relay, in the format the paper's solutions end with
|
|
12
|
+
('#### <number>'), the reply scored as any agent's last message is.
|
|
13
|
+
The paper's own answerers were its fine-tuned models, never released,
|
|
14
|
+
so the setup takes any model; the paper decoded greedily, which the
|
|
15
|
+
temperature option asks for (default: the provider's own, since a
|
|
16
|
+
model that reasons refuses one)
|
|
17
|
+
|
|
18
|
+
The upstream's module imports torch (for a training dataset class the score
|
|
19
|
+
never uses), so the driver runs in the environment of its own that
|
|
20
|
+
standard.json declares (driver.install), never in Nullius's.
|
|
21
|
+
|
|
22
|
+
The reference solution is never in the workspace: it is read from the
|
|
23
|
+
dataset only when the score is taken.
|
|
24
|
+
"""
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import sys
|
|
28
|
+
import urllib.request
|
|
29
|
+
|
|
30
|
+
sys.path.insert(0, os.path.join(os.environ["NULLIUS_UPSTREAM"], "grade_school_math"))
|
|
31
|
+
import dataset as upstream # noqa: E402 grade_school_math/dataset.py, as published
|
|
32
|
+
|
|
33
|
+
PROBLEMS = {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load():
|
|
37
|
+
if not PROBLEMS:
|
|
38
|
+
with open(os.environ["NULLIUS_DATASET"]) as f:
|
|
39
|
+
for n, line in enumerate(f, 1):
|
|
40
|
+
if line.strip():
|
|
41
|
+
PROBLEMS[f"{n:04d}"] = json.loads(line)
|
|
42
|
+
return PROBLEMS
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
TEMPERATURE = os.environ.get("NULLIUS_OPTION_TEMPERATURE", "default")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def units(dataset, setup=None):
|
|
49
|
+
return {"units": list(load())}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def prompt(unit):
|
|
53
|
+
q = load()[unit]["question"]
|
|
54
|
+
return (f"Solve this grade-school math word problem. Work it out step by step, then end your reply with "
|
|
55
|
+
f"a final line of the form `#### <number>`: the answer as a plain number, with no units and no "
|
|
56
|
+
f"other text on that line.\n\n{q}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def setup(unit, dir):
|
|
60
|
+
return {"prompt": prompt(unit)}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def answer(unit, dir, setup, model, base_url, key):
|
|
64
|
+
"""The original: one chat call, the same words an agent is given, no agent loop around the model."""
|
|
65
|
+
if setup != "original":
|
|
66
|
+
raise ValueError(f"GSM8K has no {setup} setup")
|
|
67
|
+
body = {"model": model, "messages": [{"role": "user", "content": prompt(unit)}]}
|
|
68
|
+
if TEMPERATURE != "default":
|
|
69
|
+
body["temperature"] = float(TEMPERATURE)
|
|
70
|
+
req = urllib.request.Request(base_url.rstrip("/") + "/chat/completions", data=json.dumps(body).encode(),
|
|
71
|
+
headers={"Content-Type": "application/json", "Authorization": "Bearer " + key})
|
|
72
|
+
with urllib.request.urlopen(req, timeout=600) as r:
|
|
73
|
+
reply = json.load(r)
|
|
74
|
+
text = ((reply.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
|
|
75
|
+
if not text.strip():
|
|
76
|
+
raise RuntimeError("the reply had no text")
|
|
77
|
+
return {"answer": text, "model": model}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def score(unit, dir, transcript, answer, setup=None, **_):
|
|
81
|
+
problem = load()[unit]
|
|
82
|
+
if upstream.extract_answer(problem["answer"]) == upstream.INVALID_ANS:
|
|
83
|
+
raise ValueError(f"problem {unit} has no '####' reference answer")
|
|
84
|
+
ok = upstream.is_correct(answer or "", problem)
|
|
85
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok,
|
|
86
|
+
"detail": {"expected": upstream.extract_answer(problem["answer"]), "extracted": upstream.extract_answer(answer or ""),
|
|
87
|
+
"answer": (answer or "")[-4000:]}}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def oracle(unit, dir):
|
|
91
|
+
return {"answer": load()[unit]["answer"]}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score, "oracle": oracle}
|
|
95
|
+
for line in sys.stdin:
|
|
96
|
+
try:
|
|
97
|
+
req = json.loads(line)
|
|
98
|
+
out = HANDLERS[req.pop("op")](**req)
|
|
99
|
+
except Exception as e:
|
|
100
|
+
out = {"error": f"{type(e).__name__}: {e}"}
|
|
101
|
+
print(json.dumps(out), flush=True)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "gsm8k",
|
|
4
|
+
"label": "GSM8K",
|
|
5
|
+
"summary": "Each problem is a grade-school math word problem given in the prompt; the agent's last message ends with '#### <number>', and that number must be the reference answer's.",
|
|
6
|
+
"family": "agent",
|
|
7
|
+
"needs": [
|
|
8
|
+
"agent.task/v1"
|
|
9
|
+
],
|
|
10
|
+
"upstream": {
|
|
11
|
+
"repository": "https://github.com/openai/grade-school-math",
|
|
12
|
+
"tested": "3101c7d5072418e28b9008a6636bde82a006892c",
|
|
13
|
+
"tested_date": "2021-11-19",
|
|
14
|
+
"license": "MIT"
|
|
15
|
+
},
|
|
16
|
+
"driver": {
|
|
17
|
+
"command": [
|
|
18
|
+
"{install}/.venv/bin/python",
|
|
19
|
+
"{here}/driver.py"
|
|
20
|
+
],
|
|
21
|
+
"install": {
|
|
22
|
+
"python": "3.12",
|
|
23
|
+
"indexes": [
|
|
24
|
+
{
|
|
25
|
+
"url": "https://download.pytorch.org/whl/cpu",
|
|
26
|
+
"packages": [
|
|
27
|
+
"torch==2.5.1"
|
|
28
|
+
]
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"datasets": [
|
|
34
|
+
{
|
|
35
|
+
"id": "test",
|
|
36
|
+
"label": "GSM8K test",
|
|
37
|
+
"path": "grade_school_math/data/test.jsonl",
|
|
38
|
+
"sha256": "3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14",
|
|
39
|
+
"units": 1319
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
"units": {
|
|
43
|
+
"noun": "problem",
|
|
44
|
+
"nouns": "problems"
|
|
45
|
+
},
|
|
46
|
+
"controls": {
|
|
47
|
+
"nop": {
|
|
48
|
+
"source": "nullius",
|
|
49
|
+
"from": "Nullius: no agent runs; the answer is empty and the workspace is left as setup made it."
|
|
50
|
+
},
|
|
51
|
+
"oracle": {
|
|
52
|
+
"source": "official",
|
|
53
|
+
"from": "The test file's answer field: the authors' worked solution, ending in '#### <number>'."
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"score": {
|
|
57
|
+
"kind": "pass",
|
|
58
|
+
"says": "the number after '####' in the agent's last message, commas removed, is the same string as the reference answer's (the upstream's extract_answer and is_correct)"
|
|
59
|
+
},
|
|
60
|
+
"proven": {
|
|
61
|
+
"test": {
|
|
62
|
+
"units": {
|
|
63
|
+
"units": 1319,
|
|
64
|
+
"passed": "1319 ids, the same twice, unique",
|
|
65
|
+
"on": "2026-10-13"
|
|
66
|
+
},
|
|
67
|
+
"leak": {
|
|
68
|
+
"units": 1319,
|
|
69
|
+
"looked_for": 1319,
|
|
70
|
+
"short": 0,
|
|
71
|
+
"passed": "1319/1319",
|
|
72
|
+
"on": "2026-10-13"
|
|
73
|
+
},
|
|
74
|
+
"nop": {
|
|
75
|
+
"passed": "0/1319",
|
|
76
|
+
"value": 0.0,
|
|
77
|
+
"expect": {
|
|
78
|
+
"at_most": 0.0
|
|
79
|
+
},
|
|
80
|
+
"units": 1319,
|
|
81
|
+
"job": "2026-10-13__11-54-11__gsm8k__test__prove-nop",
|
|
82
|
+
"source": "nullius",
|
|
83
|
+
"on": "2026-10-13"
|
|
84
|
+
},
|
|
85
|
+
"oracle": {
|
|
86
|
+
"passed": "1319/1319",
|
|
87
|
+
"value": 1.0,
|
|
88
|
+
"expect": {
|
|
89
|
+
"at_least": 1.0
|
|
90
|
+
},
|
|
91
|
+
"units": 1319,
|
|
92
|
+
"job": "2026-10-13__11-54-11__gsm8k__test__prove-oracle",
|
|
93
|
+
"source": "official",
|
|
94
|
+
"on": "2026-10-13"
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
"setups": {
|
|
99
|
+
"original": {
|
|
100
|
+
"summary": "The problem asked of the model in one call, in the format the paper's solutions end with ('#### <number>'), and the reply scored by the upstream's own extract_answer and is_correct, as published. The paper's answerers were its own fine-tuned models, never released: the setup takes any model, in their place.",
|
|
101
|
+
"providers": {
|
|
102
|
+
"openai": "",
|
|
103
|
+
"openrouter": "",
|
|
104
|
+
"custom": ""
|
|
105
|
+
},
|
|
106
|
+
"turn_seconds": 300,
|
|
107
|
+
"paper": "the paper's test-set accuracy (table 1, its fine-tuned models), which no served model reproduces",
|
|
108
|
+
"comparable": false
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
"options": {
|
|
112
|
+
"temperature": {
|
|
113
|
+
"summary": "Sampling randomness. Provider default sends no temperature; enter a non-negative decimal to override it. Accepted values depend on the model.",
|
|
114
|
+
"default": "default",
|
|
115
|
+
"setups": [
|
|
116
|
+
"original"
|
|
117
|
+
],
|
|
118
|
+
"pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?"
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
"limits": {
|
|
122
|
+
"turn_seconds": 300
|
|
123
|
+
}
|
|
124
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"entry": "grade_school_math/dataset.py",
|
|
4
|
+
"setting": "split",
|
|
5
|
+
"values": "test, train",
|
|
6
|
+
"default": "test",
|
|
7
|
+
"nullius": "dataset:test"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"entry": "grade_school_math/dataset.py",
|
|
11
|
+
"setting": "train split",
|
|
12
|
+
"values": "train.jsonl (7,473 problems)",
|
|
13
|
+
"default": "",
|
|
14
|
+
"nullius": "not offered - a training set: the paper fine-tunes on it and evaluates on test"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"entry": "grade_school_math/data",
|
|
18
|
+
"setting": "socratic format",
|
|
19
|
+
"values": "test_socratic.jsonl, train_socratic.jsonl",
|
|
20
|
+
"default": "",
|
|
21
|
+
"nullius": "not offered - the same 1,319 questions and final answers as test.jsonl, only the reference solution is written as sub-questions: a score would not change"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"entry": "grade_school_math/dataset.py",
|
|
25
|
+
"setting": "answer extraction (ANS_RE, INVALID_ANS)",
|
|
26
|
+
"values": "#### (-?[0-9.,]+), commas removed",
|
|
27
|
+
"default": "",
|
|
28
|
+
"nullius": "fixed:the upstream's extract_answer and is_correct, imported - scored as published"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"entry": "grade_school_math/sample.py",
|
|
32
|
+
"setting": "model",
|
|
33
|
+
"values": "a fine-tuned GPT-2 checkpoint (model_ckpts), on CUDA",
|
|
34
|
+
"default": "",
|
|
35
|
+
"nullius": "not offered - the README says sample.py was not used for any experiment of the paper, and it needs a checkpoint the repository does not ship and a CUDA GPU"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"entry": "grade_school_math/sample.py",
|
|
39
|
+
"setting": "sample_len",
|
|
40
|
+
"values": "100",
|
|
41
|
+
"default": "100",
|
|
42
|
+
"nullius": "not offered - sample.py's, not used for the paper's experiments"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"entry": "grade_school_math/calculator.py",
|
|
46
|
+
"setting": "calculator annotations (use_calculator)",
|
|
47
|
+
"values": "<<expr=value>> evaluated while sampling",
|
|
48
|
+
"default": "",
|
|
49
|
+
"nullius": "not offered - a sampling aid of sample.py, which the paper's experiments did not use; an agent may compute however it likes"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"entry": "grade_school_math/train.py",
|
|
53
|
+
"setting": "fine-tuning (batch 16, AdamW lr 1e-5, 20 epochs)",
|
|
54
|
+
"values": "",
|
|
55
|
+
"default": "",
|
|
56
|
+
"nullius": "not offered - training, not evaluation"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"entry": "paper",
|
|
60
|
+
"setting": "verifier reranking over 100 samples",
|
|
61
|
+
"values": "",
|
|
62
|
+
"default": "",
|
|
63
|
+
"nullius": "not offered - not in the repository's code"
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"entry": "driver (ours)",
|
|
67
|
+
"setting": "prompt",
|
|
68
|
+
"values": "the question, asked to end with '#### <number>'",
|
|
69
|
+
"default": "",
|
|
70
|
+
"nullius": "fixed:Nullius's wording - the upstream has no prompt (the paper fine-tunes), so the agent, and the original's model, are told the answer format its scorer reads; said in the limitations"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"entry": "paper",
|
|
74
|
+
"setting": "original setup (the paper's answerer)",
|
|
75
|
+
"values": "fine-tuned GPT-3 175B / 6B",
|
|
76
|
+
"default": "",
|
|
77
|
+
"nullius": "setup:original"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"entry": "paper",
|
|
81
|
+
"setting": "decoding (greedy at test time)",
|
|
82
|
+
"values": "temperature 0",
|
|
83
|
+
"default": "0",
|
|
84
|
+
"nullius": "option:temperature"
|
|
85
|
+
}
|
|
86
|
+
]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "humaneval",
|
|
3
|
+
"title": "HumanEval",
|
|
4
|
+
"category": "Coding",
|
|
5
|
+
"domain": "Python functions written from their signature and docstring, each checked by its unit tests.",
|
|
6
|
+
"scoring": "Pass rate (pass@1)",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard"
|
|
9
|
+
],
|
|
10
|
+
"origin": "official",
|
|
11
|
+
"upstream_limitations": [
|
|
12
|
+
"Original asks a chat model for the code. The paper's Codex models, used through raw completion, are not available."
|
|
13
|
+
],
|
|
14
|
+
"limitations": [
|
|
15
|
+
"Only solution.py is tested. Other files the agent writes are ignored.",
|
|
16
|
+
"The task given to agents, completing the function in solution.py, is Nullius's wording."
|
|
17
|
+
],
|
|
18
|
+
"datasets": [
|
|
19
|
+
{
|
|
20
|
+
"key": "humaneval",
|
|
21
|
+
"title": "HumanEval",
|
|
22
|
+
"tasks": 164,
|
|
23
|
+
"official": "https://github.com/openai/human-eval",
|
|
24
|
+
"source": "GitHub",
|
|
25
|
+
"registry": "",
|
|
26
|
+
"hub": "",
|
|
27
|
+
"verifier": "The upstream's check_correctness: the problem's prompt, the agent's completion and the problem's own check(candidate), run as one program.",
|
|
28
|
+
"notes": "164 problems. MIT License."
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"citation": "@article{chen2021codex,\n title={Evaluating Large Language Models Trained on Code},\n author={Chen, Mark and Tworek, Jerry and Jun, Heewoo and Yuan, Qiming and Pinto, Henrique Ponde de Oliveira and Kaplan, Jared and others},\n journal={arXiv preprint arXiv:2107.03374},\n year={2021}\n}",
|
|
32
|
+
"citation_source": "https://github.com/openai/human-eval",
|
|
33
|
+
"good_to_know": [
|
|
34
|
+
"One attempt per problem gives pass@1. With n attempts per problem, the run reports pass@1 to pass@n. The paper sampled 200."
|
|
35
|
+
]
|
|
36
|
+
}
|