nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""GAIA on the Nullius standard, using the pinned authors' prompt and scorer.
|
|
2
|
+
|
|
3
|
+
Our glue loads pinned Parquet splits with Hugging Face datasets, stages only a
|
|
4
|
+
question's attachment, and gives the official scorer, imported unchanged, the
|
|
5
|
+
text after the last literal FINAL ANSWER: marker, or the whole reply without one.
|
|
6
|
+
Original is the paper's no-tools API procedure, implemented here: one system
|
|
7
|
+
prefix and user question per call, with no file conversion or agent loop.
|
|
8
|
+
Agent runs receive that prefix and question through ACP and use their own tools.
|
|
9
|
+
"""
|
|
10
|
+
import contextlib
|
|
11
|
+
import hashlib
|
|
12
|
+
import importlib.util
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
from pathlib import Path, PurePosixPath
|
|
16
|
+
import shutil
|
|
17
|
+
import sys
|
|
18
|
+
import urllib.request
|
|
19
|
+
|
|
20
|
+
HERE = Path(__file__).resolve().parent
|
|
21
|
+
UPSTREAM = Path(os.environ["NULLIUS_UPSTREAM"])
|
|
22
|
+
SOURCE = json.loads((HERE / "data-source.json").read_text())
|
|
23
|
+
CACHE = UPSTREAM / ".gaia-cache"
|
|
24
|
+
ROWS = {}
|
|
25
|
+
MARKER = "FINAL ANSWER:"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def upstream_module(name):
|
|
29
|
+
spec = importlib.util.spec_from_file_location("gaia_official_" + name, UPSTREAM / (name + ".py"))
|
|
30
|
+
module = importlib.util.module_from_spec(spec)
|
|
31
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
32
|
+
spec.loader.exec_module(module)
|
|
33
|
+
return module
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
SCORER = upstream_module("scorer")
|
|
37
|
+
CONTENT = upstream_module("content")
|
|
38
|
+
PREFIX = CONTENT.SUBMISSION_TEXT.split("```")[1].strip()
|
|
39
|
+
if not PREFIX.startswith("You are a general AI assistant.") or MARKER not in PREFIX:
|
|
40
|
+
raise RuntimeError("The pinned official GAIA formatting prompt could not be located")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def dataset_id():
|
|
44
|
+
key = os.environ["NULLIUS_DATASET_ID"]
|
|
45
|
+
if key not in SOURCE["datasets"]:
|
|
46
|
+
raise ValueError("Unknown GAIA validation dataset")
|
|
47
|
+
return key
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def cached_file(filename):
|
|
51
|
+
from huggingface_hub import hf_hub_download, try_to_load_from_cache
|
|
52
|
+
common = dict(repo_id=SOURCE["repository"], repo_type="dataset",
|
|
53
|
+
revision=SOURCE["revision"], cache_dir=str(CACHE / "hub"))
|
|
54
|
+
cached = try_to_load_from_cache(filename=filename, **common)
|
|
55
|
+
if isinstance(cached, str) and Path(cached).is_file():
|
|
56
|
+
return Path(cached)
|
|
57
|
+
token = os.environ.get("HF_TOKEN")
|
|
58
|
+
if not token:
|
|
59
|
+
raise RuntimeError("GAIA data are not cached: accept the dataset’s access conditions, then add a token in Settings → Data providers → Hugging Face")
|
|
60
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
61
|
+
return Path(hf_hub_download(filename=filename, token=token, **common))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def load():
|
|
65
|
+
if not ROWS:
|
|
66
|
+
from datasets import load_dataset
|
|
67
|
+
definition = SOURCE["datasets"][dataset_id()]
|
|
68
|
+
file = cached_file(definition["file"])
|
|
69
|
+
expected = definition["sha256"]
|
|
70
|
+
if hashlib.sha256(file.read_bytes()).hexdigest() != expected:
|
|
71
|
+
raise ValueError("The GAIA metadata checksum differs from the verified pin")
|
|
72
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
73
|
+
data = load_dataset("parquet", data_files=str(file), split="train", cache_dir=str(CACHE / "datasets"))
|
|
74
|
+
loaded = {}
|
|
75
|
+
for row in data:
|
|
76
|
+
uid = row["task_id"]
|
|
77
|
+
if not isinstance(uid, str) or not uid or uid in loaded:
|
|
78
|
+
raise ValueError("GAIA task ids must be nonempty and unique")
|
|
79
|
+
if not isinstance(row["Question"], str) or not row["Question"].strip():
|
|
80
|
+
raise ValueError("GAIA question is missing")
|
|
81
|
+
if not isinstance(row["Final answer"], str) or not row["Final answer"].strip():
|
|
82
|
+
raise ValueError("GAIA reference answer is missing; private test answers cannot be scored here")
|
|
83
|
+
if definition.get("level") and int(row["Level"]) != definition["level"]:
|
|
84
|
+
raise ValueError("GAIA level differs from its selected split")
|
|
85
|
+
loaded[uid] = row
|
|
86
|
+
if len(loaded) != definition["units"]:
|
|
87
|
+
raise ValueError("GAIA task count differs from the pinned definition")
|
|
88
|
+
ROWS.update(loaded)
|
|
89
|
+
return ROWS
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def units(dataset, setup=None):
|
|
93
|
+
if dataset != dataset_id():
|
|
94
|
+
raise ValueError("The requested dataset differs from the driver environment")
|
|
95
|
+
return {"units": list(load())}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def attachment(row):
|
|
99
|
+
name = row.get("file_name") or ""
|
|
100
|
+
path = row.get("file_path") or ""
|
|
101
|
+
if not name and not path:
|
|
102
|
+
return None
|
|
103
|
+
rel = PurePosixPath(path)
|
|
104
|
+
if (not name or PurePosixPath(name).name != name or name in (".", "..") or "\\" in name
|
|
105
|
+
or rel.is_absolute() or ".." in rel.parts or "\\" in path
|
|
106
|
+
or rel.parts[:2] != ("2023", "validation") or rel.name != name
|
|
107
|
+
or name.startswith("metadata")):
|
|
108
|
+
raise ValueError("GAIA attachment is not a plain file in the pinned validation split")
|
|
109
|
+
return name, cached_file(str(rel))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def setup(unit, dir):
|
|
113
|
+
row = load()[unit]
|
|
114
|
+
prompt = PREFIX + "\n\n" + row["Question"]
|
|
115
|
+
attached = attachment(row)
|
|
116
|
+
if attached:
|
|
117
|
+
name, src = attached
|
|
118
|
+
destination = Path(dir) / name
|
|
119
|
+
if destination.exists() or destination.is_symlink():
|
|
120
|
+
raise ValueError("GAIA attachment destination already exists")
|
|
121
|
+
shutil.copyfile(src, destination)
|
|
122
|
+
prompt += "\n\nAttached file in your working directory: " + name
|
|
123
|
+
return {"prompt": prompt, "environment": "data"}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def extracted(answer):
|
|
127
|
+
if not isinstance(answer, str):
|
|
128
|
+
return None
|
|
129
|
+
value = answer.rsplit(MARKER, 1)[-1].strip()
|
|
130
|
+
return value or None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def score(unit, dir, transcript, answer, setup=None, **_):
|
|
134
|
+
row = load()[unit]
|
|
135
|
+
value = extracted(answer)
|
|
136
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
137
|
+
passed = value is not None and bool(SCORER.question_scorer(value, row["Final answer"]))
|
|
138
|
+
return {"reward": float(passed), "passed": passed,
|
|
139
|
+
"detail": {"model_answer": value, "level": row["Level"],
|
|
140
|
+
"marker": isinstance(answer, str) and MARKER in answer,
|
|
141
|
+
"dataset_revision": SOURCE["revision"],
|
|
142
|
+
"reason": "official GAIA matching" if value is not None else "empty answer"}}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def oracle(unit, dir):
|
|
146
|
+
reference = load()[unit]["Final answer"]
|
|
147
|
+
# The standard's leak check also examines files added only by the oracle.
|
|
148
|
+
# Include the raw reference so the answer marker cannot hide a reference leak.
|
|
149
|
+
destination = Path(dir) / ".gaia-oracle-answer.txt"
|
|
150
|
+
with destination.open("x", encoding="utf-8") as f:
|
|
151
|
+
f.write(reference)
|
|
152
|
+
return {"answer": MARKER + " " + reference}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def answer(unit, dir, setup, model, base_url, key, **_):
|
|
156
|
+
if setup != "original":
|
|
157
|
+
raise ValueError("GAIA exposes only the original no-tools API procedure")
|
|
158
|
+
body = {"model": model, "messages": [{"role": "system", "content": PREFIX},
|
|
159
|
+
{"role": "user", "content": load()[unit]["Question"]}]}
|
|
160
|
+
temperature = os.environ.get("NULLIUS_OPTION_TEMPERATURE", "default")
|
|
161
|
+
if temperature != "default":
|
|
162
|
+
body["temperature"] = float(temperature)
|
|
163
|
+
request = urllib.request.Request(base_url.rstrip("/") + "/chat/completions",
|
|
164
|
+
data=json.dumps(body).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + key})
|
|
165
|
+
with urllib.request.urlopen(request, timeout=600) as response:
|
|
166
|
+
reply = json.load(response)
|
|
167
|
+
value = ((reply.get("choices") or [{}])[0].get("message") or {}).get("content")
|
|
168
|
+
if not isinstance(value, str) or not value.strip():
|
|
169
|
+
raise RuntimeError("The model returned no textual answer")
|
|
170
|
+
return {"answer": value, "model": model}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
HANDLERS = {"units": units, "setup": setup, "score": score, "oracle": oracle, "answer": answer}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def main():
|
|
177
|
+
for line in sys.stdin:
|
|
178
|
+
try:
|
|
179
|
+
request = json.loads(line)
|
|
180
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
181
|
+
result = HANDLERS[request.pop("op")](**request)
|
|
182
|
+
except Exception as exc:
|
|
183
|
+
result = {"error": f"{type(exc).__name__}: {exc}"}
|
|
184
|
+
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
if __name__ == "__main__":
|
|
188
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Nullius’s base image unchanged. HF_TOKEN comes from Settings → Data providers → Hugging Face, to the driver alone, never into the image.
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "gaia",
|
|
4
|
+
"label": "GAIA",
|
|
5
|
+
"summary": "Answer real-world questions with optional attachments; the authors\u2019 scorer checks the final answer.",
|
|
6
|
+
"family": "agent",
|
|
7
|
+
"needs": [
|
|
8
|
+
"agent.task/v1",
|
|
9
|
+
"agent.workspace/v1",
|
|
10
|
+
"agent.search/v1"
|
|
11
|
+
],
|
|
12
|
+
"upstream": {
|
|
13
|
+
"repository": "https://huggingface.co/spaces/gaia-benchmark/leaderboard",
|
|
14
|
+
"tested": "9f133d71362e77b3539f1514f31b9c101a545fec",
|
|
15
|
+
"license": "Apache-2.0",
|
|
16
|
+
"tested_date": "2026-11-06"
|
|
17
|
+
},
|
|
18
|
+
"driver": {
|
|
19
|
+
"command": [
|
|
20
|
+
"{install}/.venv/bin/python",
|
|
21
|
+
"{here}/driver.py"
|
|
22
|
+
],
|
|
23
|
+
"install": {
|
|
24
|
+
"python": "3.12",
|
|
25
|
+
"pip": [
|
|
26
|
+
"numpy==2.5.4",
|
|
27
|
+
"datasets==5.1.0",
|
|
28
|
+
"huggingface-hub==2.4.0"
|
|
29
|
+
]
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"datasets": [
|
|
33
|
+
{
|
|
34
|
+
"id": "gaia-validation",
|
|
35
|
+
"label": "Validation \u00b7 All levels",
|
|
36
|
+
"units": 165,
|
|
37
|
+
"environment": "data"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "gaia-validation-level1",
|
|
41
|
+
"label": "Validation \u00b7 Level 1",
|
|
42
|
+
"units": 53,
|
|
43
|
+
"environment": "data"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"id": "gaia-validation-level2",
|
|
47
|
+
"label": "Validation \u00b7 Level 2",
|
|
48
|
+
"units": 86,
|
|
49
|
+
"environment": "data"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"id": "gaia-validation-level3",
|
|
53
|
+
"label": "Validation \u00b7 Level 3",
|
|
54
|
+
"units": 26,
|
|
55
|
+
"environment": "data"
|
|
56
|
+
}
|
|
57
|
+
],
|
|
58
|
+
"units": {
|
|
59
|
+
"noun": "question",
|
|
60
|
+
"nouns": "questions",
|
|
61
|
+
"redistribute": false
|
|
62
|
+
},
|
|
63
|
+
"environments": {
|
|
64
|
+
"data": {
|
|
65
|
+
"summary": "Access to the gated GAIA dataset.",
|
|
66
|
+
"dockerfile": "env/data.Dockerfile",
|
|
67
|
+
"local_ok": true,
|
|
68
|
+
"needs_secrets": [
|
|
69
|
+
{
|
|
70
|
+
"name": "HF_TOKEN",
|
|
71
|
+
"to": "driver",
|
|
72
|
+
"provider": "huggingface",
|
|
73
|
+
"summary": "Hugging Face access token."
|
|
74
|
+
}
|
|
75
|
+
],
|
|
76
|
+
"network": {
|
|
77
|
+
"access": "public",
|
|
78
|
+
"probe": "https://en.wikipedia.org/"
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
"controls": {
|
|
83
|
+
"nop": {
|
|
84
|
+
"source": "nullius",
|
|
85
|
+
"from": "No agent runs and the answer is empty."
|
|
86
|
+
},
|
|
87
|
+
"oracle": {
|
|
88
|
+
"source": "adapter",
|
|
89
|
+
"from": "The official validation Final answer field, prefixed with FINAL ANSWER:. The raw reference is also written only by the oracle so the leak check can inspect it.",
|
|
90
|
+
"in_prompt": "A question that asks for one of the choices it lists names its answer among those choices."
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
"score": {
|
|
94
|
+
"kind": "pass",
|
|
95
|
+
"says": "The authors' scorer, unchanged, compares the text after the last FINAL ANSWER: marker, or the whole reply without one, with the reference."
|
|
96
|
+
},
|
|
97
|
+
"setups": {
|
|
98
|
+
"original": {
|
|
99
|
+
"summary": "The paper's no-tools API procedure: its formatting prompt as the system message, the question as the user message, one call, the authors' scorer unchanged. Attachments are not read.",
|
|
100
|
+
"providers": {
|
|
101
|
+
"openai": "",
|
|
102
|
+
"openrouter": "",
|
|
103
|
+
"custom": ""
|
|
104
|
+
},
|
|
105
|
+
"paper": "GAIA, section 4: direct GPT-4 and GPT-4 Turbo evaluation without plugins; API runs repeated three times.",
|
|
106
|
+
"comparable": false
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
"options": {
|
|
110
|
+
"temperature": {
|
|
111
|
+
"label": "Temperature",
|
|
112
|
+
"summary": "Provider default sends none; a non-negative decimal sets it. The paper does not state it.",
|
|
113
|
+
"pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?",
|
|
114
|
+
"default": "default",
|
|
115
|
+
"setups": [
|
|
116
|
+
"original"
|
|
117
|
+
]
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
"proven": {
|
|
121
|
+
"gaia-validation": {
|
|
122
|
+
"units": {
|
|
123
|
+
"units": 165,
|
|
124
|
+
"passed": "165 ids, the same twice, unique",
|
|
125
|
+
"on": "2026-11-11"
|
|
126
|
+
},
|
|
127
|
+
"leak": {
|
|
128
|
+
"units": 165,
|
|
129
|
+
"looked_for": 210,
|
|
130
|
+
"short": 120,
|
|
131
|
+
"passed": "165/165",
|
|
132
|
+
"in_prompt": {
|
|
133
|
+
"units": 1,
|
|
134
|
+
"why": "A question that asks for one of the choices it lists names its answer among those choices."
|
|
135
|
+
},
|
|
136
|
+
"note": "120 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 1 prompt(s) name their answer, as the benchmark says they may: A question that asks for one of the choices it lists names its answer among those choices.",
|
|
137
|
+
"on": "2026-11-11"
|
|
138
|
+
},
|
|
139
|
+
"nop": {
|
|
140
|
+
"passed": "0/165",
|
|
141
|
+
"value": 0.0,
|
|
142
|
+
"expect": {
|
|
143
|
+
"at_most": 0.0
|
|
144
|
+
},
|
|
145
|
+
"units": 165,
|
|
146
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation__prove-nop",
|
|
147
|
+
"source": "nullius",
|
|
148
|
+
"on": "2026-11-11"
|
|
149
|
+
},
|
|
150
|
+
"oracle": {
|
|
151
|
+
"passed": "165/165",
|
|
152
|
+
"value": 1.0,
|
|
153
|
+
"expect": {
|
|
154
|
+
"at_least": 1.0
|
|
155
|
+
},
|
|
156
|
+
"units": 165,
|
|
157
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation__prove-oracle",
|
|
158
|
+
"source": "adapter",
|
|
159
|
+
"on": "2026-11-11"
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
"gaia-validation-level1": {
|
|
163
|
+
"units": {
|
|
164
|
+
"units": 53,
|
|
165
|
+
"passed": "53 ids, the same twice, unique",
|
|
166
|
+
"on": "2026-11-11"
|
|
167
|
+
},
|
|
168
|
+
"leak": {
|
|
169
|
+
"units": 53,
|
|
170
|
+
"looked_for": 69,
|
|
171
|
+
"short": 37,
|
|
172
|
+
"passed": "53/53",
|
|
173
|
+
"in_prompt": {
|
|
174
|
+
"units": 1,
|
|
175
|
+
"why": "A question that asks for one of the choices it lists names its answer among those choices."
|
|
176
|
+
},
|
|
177
|
+
"note": "37 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 1 prompt(s) name their answer, as the benchmark says they may: A question that asks for one of the choices it lists names its answer among those choices.",
|
|
178
|
+
"on": "2026-11-11"
|
|
179
|
+
},
|
|
180
|
+
"nop": {
|
|
181
|
+
"passed": "0/53",
|
|
182
|
+
"value": 0.0,
|
|
183
|
+
"expect": {
|
|
184
|
+
"at_most": 0.0
|
|
185
|
+
},
|
|
186
|
+
"units": 53,
|
|
187
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation-level1__prove-nop",
|
|
188
|
+
"source": "nullius",
|
|
189
|
+
"on": "2026-11-11"
|
|
190
|
+
},
|
|
191
|
+
"oracle": {
|
|
192
|
+
"passed": "53/53",
|
|
193
|
+
"value": 1.0,
|
|
194
|
+
"expect": {
|
|
195
|
+
"at_least": 1.0
|
|
196
|
+
},
|
|
197
|
+
"units": 53,
|
|
198
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation-level1__prove-oracle",
|
|
199
|
+
"source": "adapter",
|
|
200
|
+
"on": "2026-11-11"
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
"gaia-validation-level2": {
|
|
204
|
+
"units": {
|
|
205
|
+
"units": 86,
|
|
206
|
+
"passed": "86 ids, the same twice, unique",
|
|
207
|
+
"on": "2026-11-11"
|
|
208
|
+
},
|
|
209
|
+
"leak": {
|
|
210
|
+
"units": 86,
|
|
211
|
+
"looked_for": 106,
|
|
212
|
+
"short": 66,
|
|
213
|
+
"passed": "86/86",
|
|
214
|
+
"note": "66 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
|
|
215
|
+
"on": "2026-11-11"
|
|
216
|
+
},
|
|
217
|
+
"nop": {
|
|
218
|
+
"passed": "0/86",
|
|
219
|
+
"value": 0.0,
|
|
220
|
+
"expect": {
|
|
221
|
+
"at_most": 0.0
|
|
222
|
+
},
|
|
223
|
+
"units": 86,
|
|
224
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation-level2__prove-nop",
|
|
225
|
+
"source": "nullius",
|
|
226
|
+
"on": "2026-11-11"
|
|
227
|
+
},
|
|
228
|
+
"oracle": {
|
|
229
|
+
"passed": "86/86",
|
|
230
|
+
"value": 1.0,
|
|
231
|
+
"expect": {
|
|
232
|
+
"at_least": 1.0
|
|
233
|
+
},
|
|
234
|
+
"units": 86,
|
|
235
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation-level2__prove-oracle",
|
|
236
|
+
"source": "adapter",
|
|
237
|
+
"on": "2026-11-11"
|
|
238
|
+
}
|
|
239
|
+
},
|
|
240
|
+
"gaia-validation-level3": {
|
|
241
|
+
"units": {
|
|
242
|
+
"units": 26,
|
|
243
|
+
"passed": "26 ids, the same twice, unique",
|
|
244
|
+
"on": "2026-11-11"
|
|
245
|
+
},
|
|
246
|
+
"leak": {
|
|
247
|
+
"units": 26,
|
|
248
|
+
"looked_for": 35,
|
|
249
|
+
"short": 17,
|
|
250
|
+
"passed": "26/26",
|
|
251
|
+
"note": "17 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
|
|
252
|
+
"on": "2026-11-11"
|
|
253
|
+
},
|
|
254
|
+
"nop": {
|
|
255
|
+
"passed": "0/26",
|
|
256
|
+
"value": 0.0,
|
|
257
|
+
"expect": {
|
|
258
|
+
"at_most": 0.0
|
|
259
|
+
},
|
|
260
|
+
"units": 26,
|
|
261
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation-level3__prove-nop",
|
|
262
|
+
"source": "nullius",
|
|
263
|
+
"on": "2026-11-11"
|
|
264
|
+
},
|
|
265
|
+
"oracle": {
|
|
266
|
+
"passed": "26/26",
|
|
267
|
+
"value": 1.0,
|
|
268
|
+
"expect": {
|
|
269
|
+
"at_least": 1.0
|
|
270
|
+
},
|
|
271
|
+
"units": 26,
|
|
272
|
+
"job": "2026-11-11__23-16-20__gaia__gaia-validation-level3__prove-oracle",
|
|
273
|
+
"source": "adapter",
|
|
274
|
+
"on": "2026-11-11"
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|