nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.agent/v1",
|
|
3
|
+
"key": "reference",
|
|
4
|
+
"label": "Example agent",
|
|
5
|
+
"example": true,
|
|
6
|
+
"summary": "Nullius's example of the protocol, to read or copy.",
|
|
7
|
+
"upstream": {
|
|
8
|
+
"version": "1",
|
|
9
|
+
"license": "same as Nullius"
|
|
10
|
+
},
|
|
11
|
+
"start": {
|
|
12
|
+
"acp": [
|
|
13
|
+
"python3",
|
|
14
|
+
"{here}/agent.py"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
"model": {
|
|
18
|
+
"apis": [
|
|
19
|
+
"chat"
|
|
20
|
+
],
|
|
21
|
+
"env": {
|
|
22
|
+
"OPENAI_BASE_URL": "{base_url}",
|
|
23
|
+
"OPENAI_API_KEY": "{key}",
|
|
24
|
+
"REFERENCE_MODEL": "{model}"
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"offers": [
|
|
28
|
+
"agent.task/v1",
|
|
29
|
+
"agent.workspace/v1",
|
|
30
|
+
"agent.tools/v1",
|
|
31
|
+
"agent.conversation/v1"
|
|
32
|
+
],
|
|
33
|
+
"proven": {
|
|
34
|
+
"agent.task/v1": {
|
|
35
|
+
"passed": "3/3",
|
|
36
|
+
"on": "2026-11-02",
|
|
37
|
+
"model": "openai/gpt-5-mini",
|
|
38
|
+
"provider": "openrouter",
|
|
39
|
+
"version": "1",
|
|
40
|
+
"kit_version": 1,
|
|
41
|
+
"backend": "docker",
|
|
42
|
+
"job": "2026-11-02__14-14-35__kit__agent-task-v1__reference"
|
|
43
|
+
},
|
|
44
|
+
"agent.workspace/v1": {
|
|
45
|
+
"passed": "2/2",
|
|
46
|
+
"on": "2026-11-02",
|
|
47
|
+
"model": "openai/gpt-5-mini",
|
|
48
|
+
"provider": "openrouter",
|
|
49
|
+
"version": "1",
|
|
50
|
+
"kit_version": 1,
|
|
51
|
+
"backend": "docker",
|
|
52
|
+
"job": "2026-11-02__14-15-19__kit__agent-workspace-v1__reference"
|
|
53
|
+
},
|
|
54
|
+
"agent.tools/v1": {
|
|
55
|
+
"passed": "2/2",
|
|
56
|
+
"on": "2026-11-02",
|
|
57
|
+
"model": "openai/gpt-5-mini",
|
|
58
|
+
"provider": "openrouter",
|
|
59
|
+
"version": "1",
|
|
60
|
+
"kit_version": 2,
|
|
61
|
+
"backend": "docker",
|
|
62
|
+
"job": "2026-11-02__14-15-59__kit__agent-tools-v1__reference",
|
|
63
|
+
"skills": {
|
|
64
|
+
"decoys": {
|
|
65
|
+
"passed": "1/1",
|
|
66
|
+
"ok": true
|
|
67
|
+
},
|
|
68
|
+
"chain": {
|
|
69
|
+
"passed": "1/1",
|
|
70
|
+
"ok": true
|
|
71
|
+
},
|
|
72
|
+
"retry": {
|
|
73
|
+
"passed": "1/1",
|
|
74
|
+
"ok": true
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"agent.conversation/v1": {
|
|
79
|
+
"passed": "2/2",
|
|
80
|
+
"on": "2026-11-02",
|
|
81
|
+
"model": "openai/gpt-5-mini",
|
|
82
|
+
"provider": "openrouter",
|
|
83
|
+
"version": "1",
|
|
84
|
+
"kit_version": 1,
|
|
85
|
+
"backend": "docker",
|
|
86
|
+
"job": "2026-11-02__14-17-27__kit__agent-conversation-v1__reference"
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""The reference agent: the smallest complete agent in the Nullius standard.
|
|
2
|
+
|
|
3
|
+
Python's standard library only, so it is also the one to copy. It speaks ACP
|
|
4
|
+
on stdin and stdout, keeps one conversation per session, and works in a loop:
|
|
5
|
+
ask the model (OpenAI Chat Completions at $OPENAI_BASE_URL), run the tools it
|
|
6
|
+
asks for, give it their results, until it answers without a tool. Its tools
|
|
7
|
+
are four of its own, working in the session's directory - read_file,
|
|
8
|
+
write_file, list_files, run - and every tool of the MCP servers the session
|
|
9
|
+
was given, named ``<server>__<tool>``.
|
|
10
|
+
|
|
11
|
+
Every step is told to the client as it happens: the model's text as
|
|
12
|
+
``agent_message_chunk``, each tool call as ``tool_call`` then
|
|
13
|
+
``tool_call_update`` with its result.
|
|
14
|
+
"""
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import threading
|
|
20
|
+
import urllib.error
|
|
21
|
+
import urllib.request
|
|
22
|
+
|
|
23
|
+
MAX_STEPS = int(os.environ.get("REFERENCE_MAX_STEPS") or 30)
|
|
24
|
+
OUT_LIMIT = 8000
|
|
25
|
+
SYSTEM = ("You are a careful software agent working in the directory {cwd}. Use the tools to inspect and change files "
|
|
26
|
+
"and to run commands there. When the task is done, reply with a short final message and no tool call. "
|
|
27
|
+
"If the task asks for a specific reply, give exactly that reply as your final message.")
|
|
28
|
+
OWN_TOOLS = [
|
|
29
|
+
{"type": "function", "function": {"name": "read_file", "description": "Read a text file in the working directory.",
|
|
30
|
+
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}},
|
|
31
|
+
{"type": "function", "function": {"name": "write_file", "description": "Write a whole text file in the working directory.",
|
|
32
|
+
"parameters": {"type": "object", "properties": {"path": {"type": "string"}, "content": {"type": "string"}},
|
|
33
|
+
"required": ["path", "content"]}}},
|
|
34
|
+
{"type": "function", "function": {"name": "list_files", "description": "List the files under a folder of the working directory.",
|
|
35
|
+
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}}}},
|
|
36
|
+
{"type": "function", "function": {"name": "run", "description": "Run a shell command in the working directory (60 s at most).",
|
|
37
|
+
"parameters": {"type": "object", "properties": {"command": {"type": "string"}}, "required": ["command"]}}},
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
lock = threading.Lock()
|
|
41
|
+
sessions = {}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def send(msg):
|
|
45
|
+
with lock:
|
|
46
|
+
sys.stdout.write(json.dumps(msg) + "\n")
|
|
47
|
+
sys.stdout.flush()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def update(sid, u):
|
|
51
|
+
send({"jsonrpc": "2.0", "method": "session/update", "params": {"sessionId": sid, "update": u}})
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Mcp:
|
|
55
|
+
"""A stdio MCP server the session was given: started once, asked for its
|
|
56
|
+
tools, called by name."""
|
|
57
|
+
|
|
58
|
+
def __init__(self, spec, cwd):
|
|
59
|
+
env = dict(os.environ, **{e["name"]: e["value"] for e in spec.get("env") or []})
|
|
60
|
+
self.name = spec["name"]
|
|
61
|
+
self.p = subprocess.Popen([spec["command"]] + list(spec.get("args") or []), cwd=cwd, env=env, stdin=subprocess.PIPE,
|
|
62
|
+
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)
|
|
63
|
+
self.n = 0
|
|
64
|
+
self.ask("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "reference", "version": "1"}})
|
|
65
|
+
self.p.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
|
|
66
|
+
self.p.stdin.flush()
|
|
67
|
+
self.tools = self.ask("tools/list", {}).get("tools") or []
|
|
68
|
+
|
|
69
|
+
def ask(self, method, params):
|
|
70
|
+
self.n += 1
|
|
71
|
+
self.p.stdin.write(json.dumps({"jsonrpc": "2.0", "id": self.n, "method": method, "params": params}) + "\n")
|
|
72
|
+
self.p.stdin.flush()
|
|
73
|
+
for line in self.p.stdout:
|
|
74
|
+
m = json.loads(line)
|
|
75
|
+
if m.get("id") == self.n:
|
|
76
|
+
if "error" in m:
|
|
77
|
+
raise RuntimeError(m["error"].get("message"))
|
|
78
|
+
return m.get("result") or {}
|
|
79
|
+
raise RuntimeError(f"MCP server {self.name} ended")
|
|
80
|
+
|
|
81
|
+
def call(self, tool, args):
|
|
82
|
+
r = self.ask("tools/call", {"name": tool, "arguments": args})
|
|
83
|
+
text = "\n".join(c.get("text") or json.dumps(c) for c in r.get("content") or [])
|
|
84
|
+
return ("Error: " if r.get("isError") else "") + text
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def inside(cwd, path):
|
|
88
|
+
p = os.path.realpath(os.path.join(cwd, path or "."))
|
|
89
|
+
if p != os.path.realpath(cwd) and not p.startswith(os.path.realpath(cwd) + os.sep):
|
|
90
|
+
raise ValueError(f"{path} is outside the working directory")
|
|
91
|
+
return p
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def run_tool(s, name, args):
|
|
95
|
+
cwd = s["cwd"]
|
|
96
|
+
if name == "read_file":
|
|
97
|
+
with open(inside(cwd, args["path"]), errors="replace") as f:
|
|
98
|
+
return f.read()[:OUT_LIMIT]
|
|
99
|
+
if name == "write_file":
|
|
100
|
+
p = inside(cwd, args["path"])
|
|
101
|
+
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
102
|
+
with open(p, "w") as f:
|
|
103
|
+
f.write(args.get("content") or "")
|
|
104
|
+
return f"wrote {len(args.get('content') or '')} characters to {args['path']}"
|
|
105
|
+
if name == "list_files":
|
|
106
|
+
base = inside(cwd, args.get("path") or ".")
|
|
107
|
+
out = []
|
|
108
|
+
for root, dirs, files in os.walk(base):
|
|
109
|
+
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
|
110
|
+
out += [os.path.relpath(os.path.join(root, f), cwd) for f in files]
|
|
111
|
+
return "\n".join(sorted(out)[:500]) or "(empty)"
|
|
112
|
+
if name == "run":
|
|
113
|
+
try:
|
|
114
|
+
r = subprocess.run(args["command"], shell=True, cwd=cwd, capture_output=True, text=True, timeout=60)
|
|
115
|
+
out = (r.stdout + r.stderr)[-OUT_LIMIT:]
|
|
116
|
+
return f"exit {r.returncode}\n{out}"
|
|
117
|
+
except subprocess.TimeoutExpired:
|
|
118
|
+
return "Error: the command did not end in 60 s"
|
|
119
|
+
server, _, tool = name.partition("__")
|
|
120
|
+
for m in s["mcp"]:
|
|
121
|
+
if m.name == server:
|
|
122
|
+
return m.call(tool, args)
|
|
123
|
+
raise ValueError(f"no tool {name}")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def model(messages, tools):
|
|
127
|
+
body = {"model": os.environ.get("REFERENCE_MODEL") or "default", "messages": messages, "tools": tools}
|
|
128
|
+
req = urllib.request.Request(os.environ["OPENAI_BASE_URL"].rstrip("/") + "/chat/completions", data=json.dumps(body).encode(),
|
|
129
|
+
headers={"Content-Type": "application/json", "Authorization": "Bearer " + os.environ.get("OPENAI_API_KEY", "")})
|
|
130
|
+
with urllib.request.urlopen(req, timeout=600) as r:
|
|
131
|
+
return json.loads(r.read())["choices"][0]["message"]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def turn(rid, sid, text):
|
|
135
|
+
s = sessions[sid]
|
|
136
|
+
s["messages"].append({"role": "user", "content": text})
|
|
137
|
+
tools = OWN_TOOLS + [{"type": "function", "function": {"name": f"{m.name}__{t['name']}", "description": t.get("description") or "",
|
|
138
|
+
"parameters": t.get("inputSchema") or {"type": "object", "properties": {}}}}
|
|
139
|
+
for m in s["mcp"] for t in m.tools]
|
|
140
|
+
try:
|
|
141
|
+
for step in range(MAX_STEPS):
|
|
142
|
+
if s["cancelled"]:
|
|
143
|
+
send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "cancelled"}})
|
|
144
|
+
return
|
|
145
|
+
msg = model(s["messages"], tools)
|
|
146
|
+
calls = msg.get("tool_calls") or []
|
|
147
|
+
s["messages"].append({k: v for k, v in msg.items() if k in ("role", "content", "tool_calls")})
|
|
148
|
+
if msg.get("content"):
|
|
149
|
+
update(sid, {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": msg["content"]}})
|
|
150
|
+
if not calls:
|
|
151
|
+
send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "end_turn"}})
|
|
152
|
+
return
|
|
153
|
+
for c in calls:
|
|
154
|
+
name = c["function"]["name"]
|
|
155
|
+
try:
|
|
156
|
+
args = json.loads(c["function"].get("arguments") or "{}")
|
|
157
|
+
except ValueError:
|
|
158
|
+
args = {}
|
|
159
|
+
kind = {"read_file": "read", "write_file": "edit", "list_files": "search", "run": "execute"}.get(name, "other")
|
|
160
|
+
update(sid, {"sessionUpdate": "tool_call", "toolCallId": c["id"], "title": name, "kind": kind,
|
|
161
|
+
"status": "in_progress", "rawInput": args})
|
|
162
|
+
try:
|
|
163
|
+
out, status = run_tool(s, name, args), "completed"
|
|
164
|
+
except Exception as e:
|
|
165
|
+
out, status = f"Error: {e}", "failed"
|
|
166
|
+
update(sid, {"sessionUpdate": "tool_call_update", "toolCallId": c["id"], "status": status,
|
|
167
|
+
"content": [{"type": "content", "content": {"type": "text", "text": out}}]})
|
|
168
|
+
s["messages"].append({"role": "tool", "tool_call_id": c["id"], "content": out})
|
|
169
|
+
send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "max_turn_requests"}})
|
|
170
|
+
except urllib.error.HTTPError as e:
|
|
171
|
+
send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": f"model: HTTP {e.code}: {e.read()[:300].decode('utf-8', 'replace')}"}})
|
|
172
|
+
except Exception as e:
|
|
173
|
+
send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": f"{type(e).__name__}: {e}"}})
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def main():
|
|
177
|
+
for line in sys.stdin:
|
|
178
|
+
try:
|
|
179
|
+
m = json.loads(line)
|
|
180
|
+
except ValueError:
|
|
181
|
+
continue
|
|
182
|
+
method, params, rid = m.get("method"), m.get("params") or {}, m.get("id")
|
|
183
|
+
if method == "initialize":
|
|
184
|
+
send({"jsonrpc": "2.0", "id": rid, "result": {"protocolVersion": 1, "agentCapabilities": {
|
|
185
|
+
"loadSession": False, "promptCapabilities": {"image": False, "audio": False, "embeddedContext": False}},
|
|
186
|
+
"agentInfo": {"name": "reference", "version": "1"}}})
|
|
187
|
+
elif method == "session/new":
|
|
188
|
+
sid = f"ref-{len(sessions) + 1}"
|
|
189
|
+
try:
|
|
190
|
+
servers = [Mcp(x, params["cwd"]) for x in params.get("mcpServers") or []]
|
|
191
|
+
except Exception as e:
|
|
192
|
+
send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": f"MCP server: {e}"}})
|
|
193
|
+
continue
|
|
194
|
+
sessions[sid] = {"cwd": params["cwd"], "mcp": servers, "cancelled": False,
|
|
195
|
+
"messages": [{"role": "system", "content": SYSTEM.format(cwd=params["cwd"])}]}
|
|
196
|
+
send({"jsonrpc": "2.0", "id": rid, "result": {"sessionId": sid}})
|
|
197
|
+
elif method == "session/prompt":
|
|
198
|
+
text = "\n".join(c.get("text") or "" for c in params.get("prompt") or [] if c.get("type") == "text")
|
|
199
|
+
sessions[params["sessionId"]]["cancelled"] = False
|
|
200
|
+
threading.Thread(target=turn, args=(rid, params["sessionId"], text), daemon=True).start()
|
|
201
|
+
elif method == "session/cancel":
|
|
202
|
+
if params.get("sessionId") in sessions:
|
|
203
|
+
sessions[params["sessionId"]]["cancelled"] = True
|
|
204
|
+
elif rid is not None and method:
|
|
205
|
+
send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": f"{method} is not supported"}})
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
if __name__ == "__main__":
|
|
209
|
+
main()
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.agent/v1",
|
|
3
|
+
"key": "terminus-2",
|
|
4
|
+
"label": "Terminus 2",
|
|
5
|
+
"summary": "Harbor's terminal agent, used by Terminal-Bench.",
|
|
6
|
+
"upstream": {
|
|
7
|
+
"package": "harbor",
|
|
8
|
+
"version": "0.24.0",
|
|
9
|
+
"license": "Apache-2.0"
|
|
10
|
+
},
|
|
11
|
+
"install": {
|
|
12
|
+
"python": "3.12",
|
|
13
|
+
"pip": [
|
|
14
|
+
"harbor==0.24.0"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
"start": {
|
|
18
|
+
"command": [
|
|
19
|
+
"{install}/.venv/bin/python",
|
|
20
|
+
"{here}/run.py",
|
|
21
|
+
"{prompt}"
|
|
22
|
+
]
|
|
23
|
+
},
|
|
24
|
+
"model": {
|
|
25
|
+
"apis": [
|
|
26
|
+
"chat"
|
|
27
|
+
],
|
|
28
|
+
"env": {
|
|
29
|
+
"OPENAI_API_BASE": "{base_url}",
|
|
30
|
+
"OPENAI_API_KEY": "{key}",
|
|
31
|
+
"NULLIUS_MODEL": "{model}",
|
|
32
|
+
"NULLIUS_REASONING_EFFORT": "{opt_reasoning_effort}"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"options": {
|
|
36
|
+
"reasoning_effort": {
|
|
37
|
+
"summary": "How much reasoning the model uses before answering, with `default` leaving the choice to the provider.",
|
|
38
|
+
"values": [
|
|
39
|
+
"default",
|
|
40
|
+
"minimal",
|
|
41
|
+
"low",
|
|
42
|
+
"medium",
|
|
43
|
+
"high"
|
|
44
|
+
],
|
|
45
|
+
"default": "default"
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
"only_for": ["terminal-bench"],
|
|
49
|
+
"offers": [
|
|
50
|
+
"agent.workspace/v1"
|
|
51
|
+
],
|
|
52
|
+
"proven": {
|
|
53
|
+
"agent.workspace/v1": {
|
|
54
|
+
"passed": "2/2",
|
|
55
|
+
"on": "2026-11-02",
|
|
56
|
+
"model": "openai/gpt-5-mini",
|
|
57
|
+
"provider": "openrouter",
|
|
58
|
+
"version": "0.24.0",
|
|
59
|
+
"kit_version": 1,
|
|
60
|
+
"backend": "docker",
|
|
61
|
+
"job": "2026-11-02__14-57-08__kit__agent-workspace-v1__terminus-2"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Terminus 2, Harbor's own terminal agent, as a command agent of the Nullius standard.
|
|
2
|
+
|
|
3
|
+
Harbor runs Terminus 2 against a BaseEnvironment: a container it reaches by
|
|
4
|
+
``exec``, ``upload_file`` and ``download_file``. Here the environment is the
|
|
5
|
+
machine this runs on - the unit's workspace on the local backend, or the
|
|
6
|
+
inside of the task's own container when the standard runs the unit in one -
|
|
7
|
+
so ``Here`` is that interface over the local shell, and nothing else. The
|
|
8
|
+
agent itself is Harbor's, untouched: its prompt, its parser, its tmux
|
|
9
|
+
session, its summarisation, its LiteLLM client.
|
|
10
|
+
|
|
11
|
+
run.py "<the task>"
|
|
12
|
+
|
|
13
|
+
Terminus works in the terminal and says nothing back: the answer is what it
|
|
14
|
+
left in the workspace, so nothing is printed. Its own trajectory.json and
|
|
15
|
+
pane log land in ``agent/terminus-2/`` beside the unit's workspace, the
|
|
16
|
+
rest of what it says in the agent's log (stderr).
|
|
17
|
+
|
|
18
|
+
The model, its endpoint and key come from the environment the definition
|
|
19
|
+
sets (NULLIUS_MODEL, OPENAI_API_BASE, OPENAI_API_KEY): the relay's; the
|
|
20
|
+
reasoning effort from NULLIUS_REASONING_EFFORT (the option reasoning_effort).
|
|
21
|
+
"""
|
|
22
|
+
import asyncio
|
|
23
|
+
import logging
|
|
24
|
+
import os
|
|
25
|
+
import shutil
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import tempfile
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
from harbor.agents.terminus_2.terminus_2 import Terminus2
|
|
32
|
+
from harbor.environments.base import BaseEnvironment, ExecResult
|
|
33
|
+
from harbor.models.agent.context import AgentContext
|
|
34
|
+
from harbor.models.task.config import EnvironmentConfig
|
|
35
|
+
from harbor.models.trial.paths import EnvironmentPaths, TrialPaths
|
|
36
|
+
|
|
37
|
+
SESSION = "terminus-2"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Here(BaseEnvironment):
|
|
41
|
+
"""This machine, as the environment Terminus works in.
|
|
42
|
+
|
|
43
|
+
Harbor mounts the trial's agent folder at ``/logs/agent`` in its
|
|
44
|
+
containers; Terminus pipes its pane there. Here that one path is spelled
|
|
45
|
+
as ``logs``, the agent's own folder. tmux gets a server of its own
|
|
46
|
+
(``TMUX_TMPDIR``), so two units on one machine never share a session.
|
|
47
|
+
``user`` is ignored: Terminus asks root only to install tmux, and runs as
|
|
48
|
+
whoever runs this otherwise - the same here."""
|
|
49
|
+
|
|
50
|
+
def __init__(self, workspace: str, logs: str, tmux_dir: str):
|
|
51
|
+
self.workspace = workspace
|
|
52
|
+
self.logs = logs
|
|
53
|
+
self.tmux_dir = tmux_dir
|
|
54
|
+
super().__init__(environment_dir=Path(workspace), environment_name="here", session_id=SESSION,
|
|
55
|
+
trial_paths=TrialPaths(trial_dir=Path(logs)), task_env_config=EnvironmentConfig())
|
|
56
|
+
|
|
57
|
+
@staticmethod
|
|
58
|
+
def type() -> str:
|
|
59
|
+
return "here"
|
|
60
|
+
|
|
61
|
+
def _validate_definition(self):
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
async def start(self, force_build: bool) -> None:
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
async def stop(self, delete: bool) -> None:
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
def _spell(self, path: str) -> str:
|
|
71
|
+
agent_dir = str(EnvironmentPaths.agent_dir)
|
|
72
|
+
return self.logs + path[len(agent_dir):] if path == agent_dir or path.startswith(agent_dir + "/") else path
|
|
73
|
+
|
|
74
|
+
async def exec(self, command: str, cwd=None, env=None, timeout_sec=None, user=None) -> ExecResult:
|
|
75
|
+
command = command.replace(str(EnvironmentPaths.agent_dir) + "/", self.logs + "/")
|
|
76
|
+
full = dict(os.environ, TMUX_TMPDIR=self.tmux_dir, **(env or {}))
|
|
77
|
+
proc = await asyncio.create_subprocess_exec("bash", "-c", command, cwd=cwd or self.workspace, env=full,
|
|
78
|
+
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
79
|
+
try:
|
|
80
|
+
out, err = await asyncio.wait_for(proc.communicate(), timeout_sec)
|
|
81
|
+
except asyncio.TimeoutError:
|
|
82
|
+
proc.kill()
|
|
83
|
+
out, err = await proc.communicate()
|
|
84
|
+
return ExecResult(stdout=out.decode("utf-8", "replace"), stderr="timed out", return_code=124)
|
|
85
|
+
return ExecResult(stdout=out.decode("utf-8", "replace"), stderr=err.decode("utf-8", "replace"), return_code=proc.returncode)
|
|
86
|
+
|
|
87
|
+
async def upload_file(self, source_path, target_path: str):
|
|
88
|
+
shutil.copy(str(source_path), self._spell(target_path))
|
|
89
|
+
|
|
90
|
+
async def upload_dir(self, source_dir, target_dir: str):
|
|
91
|
+
shutil.copytree(str(source_dir), self._spell(target_dir), dirs_exist_ok=True)
|
|
92
|
+
|
|
93
|
+
async def download_file(self, source_path: str, target_path):
|
|
94
|
+
shutil.copy(self._spell(source_path), str(target_path))
|
|
95
|
+
|
|
96
|
+
async def download_dir(self, source_dir: str, target_dir):
|
|
97
|
+
shutil.copytree(self._spell(source_dir), str(target_dir), dirs_exist_ok=True)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def logs_dir(workspace: str) -> str:
|
|
101
|
+
"""Where Terminus writes its own files: ``agent/terminus-2/`` beside the
|
|
102
|
+
workspace when the unit has an agent folder there, else a folder of its own."""
|
|
103
|
+
beside = os.path.join(os.path.dirname(workspace), "agent")
|
|
104
|
+
where = os.path.join(beside, SESSION) if os.path.isdir(beside) else tempfile.mkdtemp(prefix="terminus-2-")
|
|
105
|
+
os.makedirs(where, exist_ok=True)
|
|
106
|
+
return where
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def main(prompt: str) -> int:
|
|
110
|
+
workspace = os.getcwd()
|
|
111
|
+
logs = logs_dir(workspace)
|
|
112
|
+
tmux_dir = tempfile.mkdtemp(prefix="tmux-")
|
|
113
|
+
effort = os.environ.get("NULLIUS_REASONING_EFFORT", "default")
|
|
114
|
+
agent = Terminus2(logs_dir=Path(logs), model_name="openai/" + os.environ["NULLIUS_MODEL"],
|
|
115
|
+
api_base=os.environ["OPENAI_API_BASE"], record_terminal_session=False,
|
|
116
|
+
reasoning_effort=None if effort in ("", "default") else effort)
|
|
117
|
+
env = Here(workspace, logs, tmux_dir)
|
|
118
|
+
rc = 0
|
|
119
|
+
try:
|
|
120
|
+
await agent.setup(env)
|
|
121
|
+
await agent.run(prompt, env, AgentContext())
|
|
122
|
+
except Exception as e:
|
|
123
|
+
logging.getLogger(SESSION).error("terminus-2: %s: %s", type(e).__name__, e)
|
|
124
|
+
rc = 1
|
|
125
|
+
finally:
|
|
126
|
+
await env.exec("tmux kill-server", timeout_sec=30)
|
|
127
|
+
shutil.rmtree(tmux_dir, ignore_errors=True)
|
|
128
|
+
return rc
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
if __name__ == "__main__":
|
|
132
|
+
# stdout is the answer and Terminus has none: everything it and litellm print goes to the log
|
|
133
|
+
answer = os.dup(1)
|
|
134
|
+
os.dup2(2, 1)
|
|
135
|
+
sys.stdout = sys.stderr
|
|
136
|
+
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
|
|
137
|
+
code = asyncio.run(main(sys.argv[1]))
|
|
138
|
+
os.dup2(answer, 1)
|
|
139
|
+
sys.exit(code)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "gaia",
|
|
3
|
+
"title": "GAIA",
|
|
4
|
+
"category": "General assistance",
|
|
5
|
+
"domain": "Real-world questions needing reasoning, web research and file understanding.",
|
|
6
|
+
"scoring": "Answer accuracy",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard"
|
|
9
|
+
],
|
|
10
|
+
"origin": "official",
|
|
11
|
+
"good_to_know": [
|
|
12
|
+
"Agents need public internet access.",
|
|
13
|
+
"Accept GAIA's access conditions on Hugging Face, then add a token in Settings → Data providers. Do not redistribute its questions or answers in a crawlable form.",
|
|
14
|
+
"Levels 1, 2 and 3 (53, 86 and 26 questions) split the 165 validation questions.",
|
|
15
|
+
"The authors' scorer gets the text after the last FINAL ANSWER: marker, or the whole reply without one.",
|
|
16
|
+
"Original is the paper's no-tools API baseline; choose an agent for files and web research. The paper ran it 3 times: 3 attempts per question."
|
|
17
|
+
],
|
|
18
|
+
"upstream_limitations": [
|
|
19
|
+
"The 301 test questions have private answers and are excluded from local scoring.",
|
|
20
|
+
"GAIA uses the live web; some evidence may have changed or become unavailable."
|
|
21
|
+
],
|
|
22
|
+
"limitations": [],
|
|
23
|
+
"datasets": [
|
|
24
|
+
{
|
|
25
|
+
"key": "gaia-validation",
|
|
26
|
+
"title": "Validation · All levels",
|
|
27
|
+
"tasks": 165,
|
|
28
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
29
|
+
"source": "Hugging Face",
|
|
30
|
+
"registry": "",
|
|
31
|
+
"hub": "",
|
|
32
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
33
|
+
"notes": "Official 2023 validation split."
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"key": "gaia-validation-level1",
|
|
37
|
+
"title": "Validation · Level 1",
|
|
38
|
+
"tasks": 53,
|
|
39
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
40
|
+
"source": "Hugging Face",
|
|
41
|
+
"registry": "",
|
|
42
|
+
"hub": "",
|
|
43
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
44
|
+
"notes": "Official 2023 validation split."
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"key": "gaia-validation-level2",
|
|
48
|
+
"title": "Validation · Level 2",
|
|
49
|
+
"tasks": 86,
|
|
50
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
51
|
+
"source": "Hugging Face",
|
|
52
|
+
"registry": "",
|
|
53
|
+
"hub": "",
|
|
54
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
55
|
+
"notes": "Official 2023 validation split."
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"key": "gaia-validation-level3",
|
|
59
|
+
"title": "Validation · Level 3",
|
|
60
|
+
"tasks": 26,
|
|
61
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
62
|
+
"source": "Hugging Face",
|
|
63
|
+
"registry": "",
|
|
64
|
+
"hub": "",
|
|
65
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
66
|
+
"notes": "Official 2023 validation split."
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
"citation": "@misc{mialon2023gaia,\n title={GAIA: a benchmark for General AI Assistants},\n author={Grégoire Mialon and Clémentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom},\n year={2023},\n eprint={2311.12983},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}",
|
|
70
|
+
"citation_source": "https://huggingface.co/spaces/gaia-benchmark/leaderboard/blob/9f133d71362e77b3539f1514f31b9c101a545fec/content.py",
|
|
71
|
+
"task_count": 165
|
|
72
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"repository": "gaia-benchmark/GAIA",
|
|
3
|
+
"revision": "682dd723ee1e1697e00360edccf2366dc8418dd9",
|
|
4
|
+
"datasets": {
|
|
5
|
+
"gaia-validation": {
|
|
6
|
+
"file": "2023/validation/metadata.parquet",
|
|
7
|
+
"units": 165,
|
|
8
|
+
"sha256": "d9141bad0081af438d1ee36d188c5197035dc22a01f64e3f2ee17e45469518e9"
|
|
9
|
+
},
|
|
10
|
+
"gaia-validation-level1": {
|
|
11
|
+
"file": "2023/validation/metadata.level1.parquet",
|
|
12
|
+
"units": 53,
|
|
13
|
+
"level": 1,
|
|
14
|
+
"sha256": "5e574b0faeb4603b816e426cf7c7aefb1fe398d32f9c4861e1a4e3304f2b1281"
|
|
15
|
+
},
|
|
16
|
+
"gaia-validation-level2": {
|
|
17
|
+
"file": "2023/validation/metadata.level2.parquet",
|
|
18
|
+
"units": 86,
|
|
19
|
+
"level": 2,
|
|
20
|
+
"sha256": "36fe47d728ef4e4a4477ab21bbddb88e6739ac81a885ededf99f8b395b196f2c"
|
|
21
|
+
},
|
|
22
|
+
"gaia-validation-level3": {
|
|
23
|
+
"file": "2023/validation/metadata.level3.parquet",
|
|
24
|
+
"units": 26,
|
|
25
|
+
"level": 3,
|
|
26
|
+
"sha256": "5d004e101d8fa8b071cc50d323f8437051975aea7e894892e7d5584e9920790d"
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|