nullius-eval 0.1.0rc2__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/PKG-INFO +7 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/README.md +22 -20
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/.env.example +14 -5
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/codex/agent.json +20 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/goose/agent.json +12 -1
- nullius_eval-0.1.1/app/nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/jiuwenswarm/agent.py +45 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/opencode/agent.json +12 -1
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/benchmark.json +15 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/driver.py +25 -10
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/standard.json +49 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +7 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/benchmark.json +0 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/driver.py +42 -7
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/standard.json +1 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/benchmark.json +2 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/upstream-settings.json +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/benchmark.json +4 -3
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/driver.py +63 -7
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/standard.json +186 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +13 -11
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/agents.json +393 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +48 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/driver.py +332 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/nop.py +46 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/standard.json +274 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json +3061 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/driver.py +20 -9
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/benchmark.json +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/driver.py +5 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +3 -0
- nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/standard.json +58 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +10 -10
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/benchmark.json +5 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/driver.py +84 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/standard.json +6 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +5 -5
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/mem0/capability.py +6 -3
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/catalog.py +26 -13
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/harbor_entry.py +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/sources.py +28 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/cli.py +2 -0
- nullius_eval-0.1.1/app/nullius/constraints.py +713 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/execution.py +86 -55
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/hosts.py +91 -13
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/client.py +37 -20
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/reconcile.py +4 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/onboarding.py +10 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/paths.py +6 -1
- nullius_eval-0.1.1/app/nullius/prerequisites.py +167 -0
- nullius_eval-0.1.1/app/nullius/prices.py +283 -0
- nullius_eval-0.1.1/app/nullius/providers.py +335 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/classify.py +30 -3
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/launches.py +64 -3
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/live.py +14 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/parse.py +26 -1
- nullius_eval-0.1.1/app/nullius/sandbox.py +109 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/secrets.py +30 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/settings.py +127 -17
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/acp.py +10 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/cli.py +100 -14
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/conform.py +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/container.py +799 -273
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/definitions.py +104 -30
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/fidelity.py +2 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/forward.py +57 -10
- nullius_eval-0.1.1/app/nullius/standard/hostreq.py +115 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/install.py +120 -48
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/job.py +28 -12
- nullius_eval-0.1.1/app/nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius_eval-0.1.1/app/nullius/standard/kits/agent-search-v1/standard.json +99 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/cap-memory-v1/driver.py +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/cap-memory-v1/standard.json +2 -2
- nullius_eval-0.1.1/app/nullius/standard/ledger.py +326 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/mcp_bridge.py +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/mcp_machine.py +4 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/merge.py +80 -20
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/names.py +8 -1
- nullius_eval-0.1.1/app/nullius/standard/network_requirements.py +44 -0
- nullius_eval-0.1.1/app/nullius/standard/pace.py +53 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/payload.py +24 -1
- nullius_eval-0.1.1/app/nullius/standard/pins.py +130 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/ports.py +12 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/prove.py +22 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/recipe.py +136 -46
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/relay.py +184 -28
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/run.py +666 -129
- nullius_eval-0.1.1/app/nullius/standard/setup_container.py +210 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/shard.py +2 -0
- nullius_eval-0.1.1/app/nullius/standard/web_services.py +30 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/workspace.py +12 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/packaging/PYPI.md +6 -5
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/README.md +4 -0
- nullius_eval-0.1.1/app/static/brand/docker.svg +1 -0
- nullius_eval-0.1.1/app/static/brand/huggingface.svg +37 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/command-builder-core.js +30 -11
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/command-builder.css +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/command-builder.js +87 -78
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/model-picker.js +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/catalogue.css +5 -35
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/marks.svg +8 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/new.js +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/controls.js +8 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/notifications.js +15 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/onboarding.js +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/board.css +5 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/launches.css +22 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/live-board.css +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/live-board.js +16 -9
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/path-suggestions.js +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/autosave.js +26 -6
- nullius_eval-0.1.1/app/static/settings/constraints.js +130 -0
- nullius_eval-0.1.1/app/static/settings/custom-keys.js +100 -0
- nullius_eval-0.1.1/app/static/settings/data-providers.js +111 -0
- nullius_eval-0.1.1/app/static/settings/hosts.js +115 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/judges.js +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/navigation.js +1 -1
- nullius_eval-0.1.1/app/static/settings/prices.js +68 -0
- nullius_eval-0.1.1/app/static/settings/providers.js +141 -0
- nullius_eval-0.1.1/app/static/settings/sandbox.js +20 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/settings.css +124 -11
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/style.css +68 -28
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/traces/conversation.js +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/agents/entry.html +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/base.html +8 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/benchmark.html +7 -38
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/benchmarks.html +2 -3
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/new.html +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/compare.html +2 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/_execution.html +4 -4
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/index.html +97 -40
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/job.html +17 -3
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/setup.html +1 -1
- nullius_eval-0.1.1/app/templates/settings/custom_keys.html +82 -0
- nullius_eval-0.1.1/app/templates/settings/data_providers.html +28 -0
- nullius_eval-0.1.1/app/templates/settings/machine_info.html +22 -0
- nullius_eval-0.1.1/app/templates/settings/settings.html +612 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/_judgement.html +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/trial.html +1 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/agents.py +7 -7
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/builder.py +42 -22
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/catalogue.py +19 -23
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/execute.py +38 -20
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/filters.py +8 -1
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/runs.py +106 -17
- nullius_eval-0.1.1/app/web/settings.py +480 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/shared.py +41 -2
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/PKG-INFO +7 -6
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/SOURCES.txt +33 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/pyproject.toml +2 -2
- nullius_eval-0.1.0rc2/app/nullius/catalogue/agents/jiuwenswarm/agent.json +0 -115
- nullius_eval-0.1.0rc2/app/nullius/prerequisites.py +0 -111
- nullius_eval-0.1.0rc2/app/nullius/providers.py +0 -115
- nullius_eval-0.1.0rc2/app/nullius/sandbox.py +0 -146
- nullius_eval-0.1.0rc2/app/nullius/standard/hostreq.py +0 -65
- nullius_eval-0.1.0rc2/app/static/settings/providers.js +0 -78
- nullius_eval-0.1.0rc2/app/static/settings/sandbox.js +0 -83
- nullius_eval-0.1.0rc2/app/templates/settings/settings.html +0 -508
- nullius_eval-0.1.0rc2/app/web/settings.py +0 -210
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/LICENSE +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/app.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/__main__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/codex/config.toml +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/goose/config.yaml +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/mini-swe-agent/agent.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/mini-swe-agent/run.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/opencode/opencode.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/reference/agent.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/reference/agent.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/terminus-2/agent.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/terminus-2/run.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/upstream-settings.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/driver.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/standard.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/images.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/standard.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/keyword-memory/capability.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/keyword-memory/capability.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/mem0/capability.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/citation.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/storage.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/compare/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/compare/cli.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/compare/comparison.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/gitrefs.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/harbor/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/harbor/feed.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/harbor/source.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/atif.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/cli.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/otel.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/batch.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/floors.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/lanes.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/questions.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/state.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/store.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/migrate.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/progress.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/recipes/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/recipes/launch.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/redact.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/artifacts.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/cli.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/feasibility.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/verifier.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/cancellation.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/egress.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/howto.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/jsonl.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-conversation-v1/driver.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-conversation-v1/standard.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-task-v1/driver.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-task-v1/standard.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-tools-v1/driver.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-tools-v1/mcp_kit.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-tools-v1/standard.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-workspace-v1/driver.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-workspace-v1/standard.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/match.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/behavior.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/charts.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/conversation.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/otel.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/trace.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/trajectory.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/workspace_cli.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/requirements.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/favicon.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/mark-dark.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/marks.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/tokens.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/typesafe.png +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/wordmark-dark.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/add.js +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/README.md +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/goose.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/jiuwenswarm.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/marks.json +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/mini-swe-agent.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/opencode.svg +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/catalogue.js +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/versions.js +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/compare/comparison.css +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/disclosures.js +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/0569c822cab6c510.ttf +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/DMSans-Medium.woff2 +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/DMSans-Regular.woff2 +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/README.md +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/dmsans-OFL.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/ibmplexmono-OFL.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/instrumentserif-OFL.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts.css +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/onboarding.css +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/board.js +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/start.css +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/modal.css +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/modal.js +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/traces/conversation.css +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/agents/kits.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/agents/standard.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/builder/new_run.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/add_dialog.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/use_bundle.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/_detail.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/compare_picker.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/compare_task.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/_runtime.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/ascii-mark.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/feasibility.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/preparing.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/reconcile.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/start.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/_charts.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/_conversation.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/artifact.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/workspaces/workspaces.html +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/__init__.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/compare.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/core.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/feasibility.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/judge.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/judge_tasks.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/onboarding.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/reconcile.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/traces.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/workspaces.py +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/dependency_links.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/entry_points.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/requires.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/top_level.txt +0 -0
- {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nullius-eval
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: An evaluation framework for AI agents. Bring any benchmark, agent or capability, run it on your machines, and inspect the evidence behind every score. Bring anything, run it, prove it.
|
|
5
5
|
Maintainer-email: ThinkFlowLab <nullius.thinkflowlab@proton.me>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -27,7 +27,9 @@ Dynamic: license-file
|
|
|
27
27
|
Bring anything. Run it. Prove it.
|
|
28
28
|
|
|
29
29
|
Nullius runs AI evaluations on your machines and opens each score into the
|
|
30
|
-
evidence behind it
|
|
30
|
+
evidence behind it, so you know whether the agent failed or the benchmark did.\
|
|
31
|
+
Add your own benchmark, agent or capability with a single prompt and iterate
|
|
32
|
+
faster.
|
|
31
33
|
|
|
32
34
|
## Install and launch
|
|
33
35
|
|
|
@@ -43,10 +45,9 @@ the application and command are `nullius`.
|
|
|
43
45
|
|
|
44
46
|
Python 3.12 or later is required. Reading existing results needs no model key.
|
|
45
47
|
Running benchmarks requires the appropriate provider credentials and benchmark
|
|
46
|
-
prerequisites; Docker is needed for Harbor and the standard's
|
|
48
|
+
prerequisites; Docker is needed for Harbor and the standard's container backend.
|
|
47
49
|
|
|
48
|
-
|
|
49
|
-
validation targets Linux with Python 3.12. The wheel is a Python package with
|
|
50
|
+
Release validation targets Linux with Python 3.12. The wheel is a Python package with
|
|
50
51
|
a local web UI, not a standalone desktop installer.
|
|
51
52
|
|
|
52
53
|
## What it does
|
|
@@ -72,7 +73,7 @@ Update to the latest version with:
|
|
|
72
73
|
uv tool upgrade nullius-eval
|
|
73
74
|
```
|
|
74
75
|
|
|
75
|
-
To select a specific version, use `uv tool install nullius-eval==0.1.
|
|
76
|
+
To select a specific version, use `uv tool install nullius-eval==0.1.1`.
|
|
76
77
|
Add `--force` when replacing an existing installation. Explicit version pins
|
|
77
78
|
are retained by `uv tool upgrade`; use `uv tool install --force nullius-eval`
|
|
78
79
|
to return to the latest version without a pin.
|
|
@@ -8,29 +8,31 @@
|
|
|
8
8
|
</p>
|
|
9
9
|
|
|
10
10
|
Nullius runs AI evaluations on your machines and opens every score into the
|
|
11
|
-
evidence behind it
|
|
11
|
+
evidence behind it, so you know whether the agent failed or the benchmark did.\
|
|
12
|
+
Add your own benchmark, agent or capability with a single prompt and iterate
|
|
13
|
+
faster.
|
|
12
14
|
|
|
13
15
|
## Quickstart
|
|
14
16
|
|
|
15
17
|
Python 3.12+ and [uv](https://docs.astral.sh/uv/); a model key to run
|
|
16
|
-
benchmarks; Docker for the
|
|
17
|
-
|
|
18
|
-
Install the latest version with `uv`. The PyPI package is `nullius-eval`;
|
|
19
|
-
the application and its command remain `nullius`:
|
|
18
|
+
benchmarks; Docker or Podman for the container backend; Docker for Harbor.
|
|
20
19
|
|
|
21
20
|
```bash
|
|
22
21
|
uv tool install nullius-eval
|
|
23
22
|
nullius serve
|
|
24
23
|
```
|
|
25
24
|
|
|
26
|
-
|
|
27
|
-
The
|
|
28
|
-
|
|
25
|
+
A specific version: `uv tool install nullius-eval==0.1.1`.\
|
|
26
|
+
The main branch, from source:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
git clone https://github.com/thinkflowlab/nullius.git
|
|
30
|
+
cd nullius
|
|
31
|
+
uv run --no-project --with-requirements app/requirements.txt app/app.py
|
|
32
|
+
```
|
|
29
33
|
|
|
30
|
-
Open **[127.0.0.1:5057](http://127.0.0.1:5057)** to go straight to **Runs
|
|
31
|
-
|
|
32
|
-
[Quickstart guide](docs/getting-started.md): open results, a first run, the app
|
|
33
|
-
on another machine.
|
|
34
|
+
Open **[127.0.0.1:5057](http://127.0.0.1:5057)** to go straight to **Runs**.\
|
|
35
|
+
The [Quickstart guide](docs/getting-started.md) covers open results, a first run, the app on another machine, upgrades, wheel installation and migration from a checkout.
|
|
34
36
|
|
|
35
37
|
```bash
|
|
36
38
|
# open job folders from the command line
|
|
@@ -48,9 +50,9 @@ profile. Provider keys can be saved in Settings without creating an `.env` file.
|
|
|
48
50
|
|
|
49
51
|
### Bring anything
|
|
50
52
|
|
|
51
|
-
*+ Add* on Benchmarks, Agents or Capabilities writes a prompt for your coding assistant. Its import skill pins the upstream, wraps the upstream's own code, proves it and reports what fits; it asks before any code of
|
|
53
|
+
*+ Add* on Benchmarks, Agents or Capabilities writes a prompt for your coding assistant. Its import skill pins the upstream, wraps the upstream's own code, proves it and reports what fits; it asks before adding any code of its own. Additions go in your user catalogue and survive application upgrades.
|
|
52
54
|
|
|
53
|
-
The import skills: `/import-benchmark
|
|
55
|
+
The import skills: `/import-benchmark`,
|
|
54
56
|
`/import-standard`, `/import-agent`, `/import-capability`.
|
|
55
57
|
|
|
56
58
|
<img src="docs/assets/nullius/readme/bring.svg" alt="Benchmarks, agents and capabilities come into one catalogue." width="800">
|
|
@@ -69,7 +71,7 @@ With no agent, the nop control must score nothing and the oracle everything; the
|
|
|
69
71
|
|
|
70
72
|
### Run anywhere
|
|
71
73
|
|
|
72
|
-
Several agents and datasets in one run. This machine, a host over ssh, or several machines, one shard each, merged into one job by `cli merge`. The
|
|
74
|
+
Several agents and datasets in one run. This machine, a host over ssh, or several machines, one shard each, merged into one job by `cli merge`. The container backend gives each unit's agent or capability its own container. On the Nullius standard a relay per unit holds the key, which the agent never sees, and records every model call.
|
|
73
75
|
|
|
74
76
|
The Run builder: **Runner → Benchmark → Dataset → Agent → Model → Limits → Where →
|
|
75
77
|
Review**. Review checks each machine before Run; follow each trial in the job's
|
|
@@ -156,7 +158,7 @@ uv run --no-project --with-requirements app/requirements.txt \
|
|
|
156
158
|
```
|
|
157
159
|
|
|
158
160
|
The [tests workflow](https://github.com/thinkflowlab/nullius/actions/workflows/tests.yml)
|
|
159
|
-
runs both suites on pull requests
|
|
161
|
+
runs both suites on pull requests; they can also be run locally.
|
|
160
162
|
|
|
161
163
|
## License
|
|
162
164
|
|
|
@@ -173,10 +175,10 @@ Built by the Nullius contributors. To cite it:
|
|
|
173
175
|
|
|
174
176
|
```bibtex
|
|
175
177
|
@software{Nullius,
|
|
176
|
-
author = {{Nullius contributors}},
|
|
177
|
-
title = {{Nullius: the run behind a benchmark score}},
|
|
178
|
-
year = {2026},
|
|
179
|
-
url = {https://github.com/thinkflowlab/nullius}
|
|
178
|
+
author = {{Nullius contributors}},
|
|
179
|
+
title = {{Nullius: the run behind a benchmark score}},
|
|
180
|
+
year = {2026},
|
|
181
|
+
url = {https://github.com/thinkflowlab/nullius}
|
|
180
182
|
}
|
|
181
183
|
```
|
|
182
184
|
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
# Start-up options, not on the Settings page:
|
|
7
7
|
# NULLIUS_HOME: a separate profile with config/, data/ and cache/ directories
|
|
8
8
|
# NULLIUS_CATALOGUE: the user catalogue directory (default catalogue/ in the user data directory)
|
|
9
|
+
# NULLIUS_PRICES: the model price list file (default prices.json in the user configuration directory); set for a run on a host
|
|
9
10
|
# APP_HOST: the address the server binds (default 127.0.0.1, or --host)
|
|
10
11
|
# APP_PORT: the port the server listens on (default 5057, or --port)
|
|
11
12
|
# JOB_ROOTS: job roots to open at start (or the command-line argument)
|
|
@@ -17,13 +18,13 @@
|
|
|
17
18
|
# HARBOR_PINS: where the Harbor a recipe pins is built and kept on this machine (default harbor-pins/ beside the runs)
|
|
18
19
|
# STANDARD_DIR: where the Nullius standard installs agents, capabilities and benchmark upstreams (default standard/ in the user data directory)
|
|
19
20
|
# STANDARD_DATA_DIR: where the Nullius standard keeps the datasets it downloads, which can be gigabytes (default datasets/ in STANDARD_DIR)
|
|
20
|
-
# SECRETS_DIR: where each benchmark's licences
|
|
21
|
+
# SECRETS_DIR: where each benchmark's licences (Settings → Custom keys) and the Hugging Face token (Settings → Data providers) are kept (default secrets/ in the user configuration directory)
|
|
21
22
|
|
|
22
23
|
# --- Model credentials ---
|
|
23
|
-
# The
|
|
24
|
+
# The key of the OpenAI-compatible endpoint at MODEL_API_URL: a recipe on that endpoint is sent there with this key. Optional, for a server that checks none: without it the run is sent the word 'local' as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log.
|
|
24
25
|
# e.g. sk-…
|
|
25
26
|
MODEL_API_KEY=
|
|
26
|
-
# The
|
|
27
|
+
# The URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More endpoints, each with a name, are MODEL_<ID>_NAME, MODEL_<ID>_API_URL and MODEL_<ID>_API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.
|
|
27
28
|
# e.g. https://…/v1
|
|
28
29
|
MODEL_API_URL=
|
|
29
30
|
# An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own endpoint, https://api.openai.com/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.
|
|
@@ -39,7 +40,7 @@ TYPESAFE_API_KEY=
|
|
|
39
40
|
SYSTEM_ONE_PROVIDER=
|
|
40
41
|
# The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest, typesafe/jev-router, jaredpalmer/kev-4b... Unset: the provider's Jev.
|
|
41
42
|
SYSTEM_ONE_MODEL=
|
|
42
|
-
# The provider of the LLM judgement, the second judgement beside System One: openrouter, openai or local, asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.
|
|
43
|
+
# The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, local (the endpoint at MODEL_API_URL) or local:<ID> (the endpoint MODEL_<ID>_API_URL), asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.
|
|
43
44
|
LLM_JUDGE_PROVIDER=
|
|
44
45
|
# The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.
|
|
45
46
|
LLM_JUDGE_MODEL=
|
|
@@ -48,7 +49,15 @@ LLM_JUDGE_MODEL=
|
|
|
48
49
|
# Machines a recipe can run on, over ssh: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]. Only the name is stored; keys stay where they are.
|
|
49
50
|
# e.g. user@host[:port][:/path/to/key]
|
|
50
51
|
HOSTS=
|
|
52
|
+
# What Nullius may use on a machine: a JSON object from local (this machine) or a host's entry to its constraints (cpus, memory_mb, processes, concurrency, cpu_set, gpu_ids). A constraint not set is no constraint.
|
|
53
|
+
# e.g. {"local": {"cpus": 4, "memory_mb": 8192}}
|
|
54
|
+
HOST_CONSTRAINTS=
|
|
51
55
|
|
|
52
|
-
# ---
|
|
56
|
+
# --- Custom keys ---
|
|
53
57
|
# The names of the agent keys this file holds, comma-separated: the keys an agent needs of its own vendor, beside the model's (Cursor's CLI agent a CURSOR_API_KEY, Copilot's a GH_TOKEN). Each is a line of its own in this file. A Harbor run gets every one of them in its environment - over stdin to a host - and a Nullius standard run's never; none is ever written into a recipe or a log.
|
|
54
58
|
AGENT_KEYS=
|
|
59
|
+
|
|
60
|
+
# --- Data providers ---
|
|
61
|
+
# Where Hugging Face downloads go: https://huggingface.co when unset, or a mirror's address. A run's dataset downloads, installs and drivers get it as HF_ENDPOINT before they load huggingface_hub; the environment wins over this file, a run on a host is sent it.
|
|
62
|
+
# e.g. https://hf-mirror.com
|
|
63
|
+
HF_ENDPOINT=
|
|
@@ -128,6 +128,16 @@
|
|
|
128
128
|
"ok": true
|
|
129
129
|
}
|
|
130
130
|
}
|
|
131
|
+
},
|
|
132
|
+
"agent.search/v1": {
|
|
133
|
+
"passed": "3/3",
|
|
134
|
+
"on": "2026-11-12",
|
|
135
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
136
|
+
"provider": "openrouter",
|
|
137
|
+
"version": "2.2.2",
|
|
138
|
+
"kit_version": 1,
|
|
139
|
+
"backend": "docker",
|
|
140
|
+
"job": "2026-11-12__10-24-26__kit__agent-search-v1__codex-2.2.2"
|
|
131
141
|
}
|
|
132
142
|
}
|
|
133
143
|
},
|
|
@@ -191,6 +201,16 @@
|
|
|
191
201
|
"ok": true
|
|
192
202
|
}
|
|
193
203
|
}
|
|
204
|
+
},
|
|
205
|
+
"agent.search/v1": {
|
|
206
|
+
"passed": "3/3",
|
|
207
|
+
"on": "2026-11-12",
|
|
208
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
209
|
+
"provider": "openrouter",
|
|
210
|
+
"version": "0.16.0",
|
|
211
|
+
"kit_version": 1,
|
|
212
|
+
"backend": "docker",
|
|
213
|
+
"job": "2026-11-12__11-08-33__kit__agent-search-v1__codex-0.16.0"
|
|
194
214
|
}
|
|
195
215
|
},
|
|
196
216
|
"upstream": {
|
|
@@ -59,7 +59,8 @@
|
|
|
59
59
|
"agent.task/v1",
|
|
60
60
|
"agent.workspace/v1",
|
|
61
61
|
"agent.tools/v1",
|
|
62
|
-
"agent.conversation/v1"
|
|
62
|
+
"agent.conversation/v1",
|
|
63
|
+
"agent.search/v1"
|
|
63
64
|
],
|
|
64
65
|
"proven": {
|
|
65
66
|
"agent.workspace/v1": {
|
|
@@ -115,6 +116,16 @@
|
|
|
115
116
|
"kit_version": 1,
|
|
116
117
|
"backend": "docker",
|
|
117
118
|
"job": "2026-11-02__13-27-36__kit__agent-conversation-v1__goose"
|
|
119
|
+
},
|
|
120
|
+
"agent.search/v1": {
|
|
121
|
+
"passed": "3/3",
|
|
122
|
+
"on": "2026-11-12",
|
|
123
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
124
|
+
"provider": "openrouter",
|
|
125
|
+
"version": "1.53.0",
|
|
126
|
+
"kit_version": 1,
|
|
127
|
+
"backend": "docker",
|
|
128
|
+
"job": "2026-11-12__11-09-30__kit__agent-search-v1__goose"
|
|
118
129
|
}
|
|
119
130
|
}
|
|
120
131
|
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.agent/v1",
|
|
3
|
+
"key": "jiuwenswarm",
|
|
4
|
+
"label": "JiuwenSwarm",
|
|
5
|
+
"summary": "openJiuwen's agent.",
|
|
6
|
+
"start": {
|
|
7
|
+
"acp": [
|
|
8
|
+
"{install}/.venv/bin/python",
|
|
9
|
+
"{here}/agent.py"
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
"model": {
|
|
13
|
+
"apis": [
|
|
14
|
+
"chat"
|
|
15
|
+
],
|
|
16
|
+
"env": {
|
|
17
|
+
"NULLIUS_BASE_URL": "{base_url}",
|
|
18
|
+
"NULLIUS_KEY": "{key}",
|
|
19
|
+
"NULLIUS_MODEL": "{model}",
|
|
20
|
+
"NULLIUS_REASONING": "{opt_reasoning_effort}"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"options": {
|
|
24
|
+
"reasoning_effort": {
|
|
25
|
+
"summary": "How much reasoning the model uses before answering, with `default` leaving the choice to the provider.",
|
|
26
|
+
"values": [
|
|
27
|
+
"default",
|
|
28
|
+
"off",
|
|
29
|
+
"minimal",
|
|
30
|
+
"low",
|
|
31
|
+
"medium",
|
|
32
|
+
"high",
|
|
33
|
+
"xhigh",
|
|
34
|
+
"max"
|
|
35
|
+
],
|
|
36
|
+
"default": "default"
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"default": "0.2.8",
|
|
40
|
+
"versions": {
|
|
41
|
+
"0.2.7": {
|
|
42
|
+
"upstream": {
|
|
43
|
+
"repository": "https://github.com/openJiuwen-ai/jiuwenswarm",
|
|
44
|
+
"tested": "2aa7fea7e55fbe1edaae0d468ef666b088c76fa9",
|
|
45
|
+
"tested_date": "2026-10-10",
|
|
46
|
+
"package": "workswarm",
|
|
47
|
+
"license": "Apache-2.0"
|
|
48
|
+
},
|
|
49
|
+
"install": {
|
|
50
|
+
"python": "3.12",
|
|
51
|
+
"pip": [
|
|
52
|
+
"workswarm==0.2.7"
|
|
53
|
+
]
|
|
54
|
+
},
|
|
55
|
+
"proven": {
|
|
56
|
+
"agent.task/v1": {
|
|
57
|
+
"passed": "3/3",
|
|
58
|
+
"on": "2026-11-02",
|
|
59
|
+
"model": "openai/gpt-5-mini",
|
|
60
|
+
"provider": "openrouter",
|
|
61
|
+
"version": "0.2.7",
|
|
62
|
+
"kit_version": 1,
|
|
63
|
+
"backend": "docker",
|
|
64
|
+
"job": "2026-11-02__13-28-04__kit__agent-task-v1__jiuwenswarm"
|
|
65
|
+
},
|
|
66
|
+
"agent.conversation/v1": {
|
|
67
|
+
"passed": "2/2",
|
|
68
|
+
"on": "2026-11-02",
|
|
69
|
+
"model": "openai/gpt-5-mini",
|
|
70
|
+
"provider": "openrouter",
|
|
71
|
+
"version": "0.2.7",
|
|
72
|
+
"kit_version": 1,
|
|
73
|
+
"backend": "docker",
|
|
74
|
+
"job": "2026-11-02__13-36-28__kit__agent-conversation-v1__jiuwenswarm"
|
|
75
|
+
},
|
|
76
|
+
"agent.workspace/v1": {
|
|
77
|
+
"passed": "2/2",
|
|
78
|
+
"on": "2026-11-02",
|
|
79
|
+
"model": "openai/gpt-5-mini",
|
|
80
|
+
"provider": "openrouter",
|
|
81
|
+
"version": "0.2.7",
|
|
82
|
+
"kit_version": 1,
|
|
83
|
+
"backend": "docker",
|
|
84
|
+
"job": "2026-11-02__13-41-01__kit__agent-workspace-v1__jiuwenswarm"
|
|
85
|
+
},
|
|
86
|
+
"agent.tools/v1": {
|
|
87
|
+
"passed": "2/2",
|
|
88
|
+
"on": "2026-11-02",
|
|
89
|
+
"model": "openai/gpt-5-mini",
|
|
90
|
+
"provider": "openrouter",
|
|
91
|
+
"version": "0.2.7",
|
|
92
|
+
"kit_version": 2,
|
|
93
|
+
"backend": "docker",
|
|
94
|
+
"job": "2026-11-02__13-33-45__kit__agent-tools-v1__jiuwenswarm",
|
|
95
|
+
"skills": {
|
|
96
|
+
"decoys": {
|
|
97
|
+
"passed": "1/1",
|
|
98
|
+
"ok": true
|
|
99
|
+
},
|
|
100
|
+
"chain": {
|
|
101
|
+
"passed": "1/1",
|
|
102
|
+
"ok": true
|
|
103
|
+
},
|
|
104
|
+
"retry": {
|
|
105
|
+
"passed": "1/1",
|
|
106
|
+
"ok": true
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
"0.2.8": {
|
|
113
|
+
"upstream": {
|
|
114
|
+
"repository": "https://github.com/openJiuwen-ai/jiuwenswarm",
|
|
115
|
+
"tag": "v0.2.8",
|
|
116
|
+
"tested": "068f148b59e140a179e31c9fe58eed8ea1700c1e",
|
|
117
|
+
"tested_date": "2026-11-12",
|
|
118
|
+
"package": "workswarm",
|
|
119
|
+
"license": "Apache-2.0",
|
|
120
|
+
"core": "openjiuwen 0.1.20 at 79f6ec656f513dadc412aaafa13f60ecf7f0fc04, pinned in install"
|
|
121
|
+
},
|
|
122
|
+
"install": {
|
|
123
|
+
"python": "3.12",
|
|
124
|
+
"pip": [
|
|
125
|
+
"workswarm==0.2.8",
|
|
126
|
+
"openjiuwen @ git+https://github.com/openJiuwen-ai/agent-core.git@79f6ec656f513dadc412aaafa13f60ecf7f0fc04"
|
|
127
|
+
]
|
|
128
|
+
},
|
|
129
|
+
"proven": {
|
|
130
|
+
"agent.task/v1": {
|
|
131
|
+
"passed": "3/3",
|
|
132
|
+
"on": "2026-11-12",
|
|
133
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
134
|
+
"provider": "openrouter",
|
|
135
|
+
"version": "0.2.8",
|
|
136
|
+
"kit_version": 1,
|
|
137
|
+
"backend": "docker",
|
|
138
|
+
"job": "2026-11-12__14-14-10__kit__agent-task-v1__jiuwenswarm"
|
|
139
|
+
},
|
|
140
|
+
"agent.workspace/v1": {
|
|
141
|
+
"passed": "2/2",
|
|
142
|
+
"on": "2026-11-12",
|
|
143
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
144
|
+
"provider": "openrouter",
|
|
145
|
+
"version": "0.2.8",
|
|
146
|
+
"kit_version": 1,
|
|
147
|
+
"backend": "docker",
|
|
148
|
+
"job": "2026-11-12__14-15-36__kit__agent-workspace-v1__jiuwenswarm"
|
|
149
|
+
},
|
|
150
|
+
"agent.tools/v1": {
|
|
151
|
+
"passed": "2/2",
|
|
152
|
+
"on": "2026-11-12",
|
|
153
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
154
|
+
"provider": "openrouter",
|
|
155
|
+
"version": "0.2.8",
|
|
156
|
+
"kit_version": 2,
|
|
157
|
+
"backend": "docker",
|
|
158
|
+
"job": "2026-11-12__14-16-36__kit__agent-tools-v1__jiuwenswarm",
|
|
159
|
+
"skills": {
|
|
160
|
+
"decoys": {
|
|
161
|
+
"passed": "1/1",
|
|
162
|
+
"ok": true
|
|
163
|
+
},
|
|
164
|
+
"chain": {
|
|
165
|
+
"passed": "1/1",
|
|
166
|
+
"ok": true
|
|
167
|
+
},
|
|
168
|
+
"retry": {
|
|
169
|
+
"passed": "1/1",
|
|
170
|
+
"ok": true
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
"agent.conversation/v1": {
|
|
175
|
+
"passed": "2/2",
|
|
176
|
+
"on": "2026-11-12",
|
|
177
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
178
|
+
"provider": "openrouter",
|
|
179
|
+
"version": "0.2.8",
|
|
180
|
+
"kit_version": 1,
|
|
181
|
+
"backend": "docker",
|
|
182
|
+
"job": "2026-11-12__14-19-05__kit__agent-conversation-v1__jiuwenswarm"
|
|
183
|
+
},
|
|
184
|
+
"agent.search/v1": {
|
|
185
|
+
"passed": "3/3",
|
|
186
|
+
"on": "2026-11-12",
|
|
187
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
188
|
+
"provider": "openrouter",
|
|
189
|
+
"version": "0.2.8",
|
|
190
|
+
"kit_version": 1,
|
|
191
|
+
"backend": "docker",
|
|
192
|
+
"job": "2026-11-12__14-20-04__kit__agent-search-v1__jiuwenswarm"
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
{nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/jiuwenswarm/agent.py
RENAMED
|
@@ -30,8 +30,11 @@ changes, each taken from the protocol itself:
|
|
|
30
30
|
that offers no files (Nullius on docker, where this wrapper runs in the
|
|
31
31
|
agent's container) gets none: ``initialize`` offers them to the bridge, and
|
|
32
32
|
this wrapper answers each request itself, inside the session's cwd.
|
|
33
|
+
Requests for the unit's own JiuwenSwarm home are served by this wrapper
|
|
34
|
+
on both backends, so its config and todo files remain accessible.
|
|
33
35
|
|
|
34
|
-
When the bridge ends,
|
|
36
|
+
The resident worker uses a short, private socket path. When the bridge ends,
|
|
37
|
+
the servers and worker are stopped.
|
|
35
38
|
"""
|
|
36
39
|
import json
|
|
37
40
|
import os
|
|
@@ -39,6 +42,7 @@ import signal
|
|
|
39
42
|
import socket
|
|
40
43
|
import subprocess
|
|
41
44
|
import sys
|
|
45
|
+
import tempfile
|
|
42
46
|
import threading
|
|
43
47
|
import time
|
|
44
48
|
|
|
@@ -168,11 +172,17 @@ class Sessions:
|
|
|
168
172
|
self.agent_in.flush()
|
|
169
173
|
|
|
170
174
|
|
|
175
|
+
def in_home(path):
|
|
176
|
+
home = os.path.realpath(JHOME)
|
|
177
|
+
path = os.path.realpath(path)
|
|
178
|
+
return path == home or path.startswith(home + os.sep)
|
|
179
|
+
|
|
180
|
+
|
|
171
181
|
def serve_fs(msg, cwd):
|
|
172
|
-
"""
|
|
182
|
+
"""Serve files in the session workspace or this unit's JiuwenSwarm home."""
|
|
173
183
|
params, root = msg["params"], os.path.realpath(cwd or "/nonexistent")
|
|
174
184
|
path = os.path.realpath(str(params.get("path") or ""))
|
|
175
|
-
if path != root and not path.startswith(root + os.sep):
|
|
185
|
+
if path != root and not path.startswith(root + os.sep) and not in_home(path):
|
|
176
186
|
return {"code": -32602, "message": f"{params.get('path')} is outside the session's directory"}, None
|
|
177
187
|
try:
|
|
178
188
|
if msg["method"] == "fs/write_text_file":
|
|
@@ -246,7 +256,8 @@ def to_client(src, dst, ses):
|
|
|
246
256
|
path, cwd = params.get("path"), ses.of(params.get("sessionId"))
|
|
247
257
|
if isinstance(path, str) and path and not os.path.isabs(path) and cwd:
|
|
248
258
|
params["path"] = os.path.normpath(os.path.join(cwd, path))
|
|
249
|
-
|
|
259
|
+
own_home = isinstance(params.get("path"), str) and in_home(params["path"])
|
|
260
|
+
if (ses.serve_fs or own_home) and "id" in msg:
|
|
250
261
|
err, res = serve_fs(msg, ses.of(params.get("sessionId")))
|
|
251
262
|
reply = {"jsonrpc": "2.0", "id": msg["id"], **({"error": err} if err else {"result": res})}
|
|
252
263
|
ses.to_bridge((json.dumps(reply) + "\n").encode())
|
|
@@ -256,7 +267,26 @@ def to_client(src, dst, ses):
|
|
|
256
267
|
dst.flush()
|
|
257
268
|
|
|
258
269
|
|
|
259
|
-
def
|
|
270
|
+
def stop_worker(env):
|
|
271
|
+
"""0.2.8's resident worker outlives the frontend; use its shutdown API."""
|
|
272
|
+
path = env["JIUWENSWARM_RUNTIME_SOCKET"]
|
|
273
|
+
if not os.path.exists(path):
|
|
274
|
+
return
|
|
275
|
+
code = ("import asyncio, os, sys\n"
|
|
276
|
+
"from jiuwenswarm.server.supervisor.worker_process import request_resident_worker_shutdown\n"
|
|
277
|
+
"async def stop():\n"
|
|
278
|
+
" await request_resident_worker_shutdown(sys.argv[1])\n"
|
|
279
|
+
" while os.path.exists(sys.argv[1]):\n"
|
|
280
|
+
" await asyncio.sleep(0.05)\n"
|
|
281
|
+
"asyncio.run(asyncio.wait_for(stop(), 25))\n")
|
|
282
|
+
try:
|
|
283
|
+
subprocess.run([PY, "-c", code, path], env=env, stdin=subprocess.DEVNULL,
|
|
284
|
+
stdout=sys.stderr, stderr=sys.stderr, timeout=30, check=True)
|
|
285
|
+
except (subprocess.SubprocessError, OSError) as e:
|
|
286
|
+
say("runtime worker shutdown failed:", e)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def run(runtime_socket):
|
|
260
290
|
os.makedirs(HOME, exist_ok=True)
|
|
261
291
|
if not os.path.isfile(os.path.join(CONFIG, "config.yaml")):
|
|
262
292
|
r = subprocess.run([PY, "-c", "from jiuwenswarm.common.utils import prepare_workspace\n"
|
|
@@ -269,7 +299,8 @@ def main():
|
|
|
269
299
|
reasoning_level()
|
|
270
300
|
ports = free_ports(4)
|
|
271
301
|
write_env(ports)
|
|
272
|
-
env = dict(os.environ, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1"
|
|
302
|
+
env = dict(os.environ, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1",
|
|
303
|
+
JIUWENSWARM_RUNTIME_SOCKET=runtime_socket)
|
|
273
304
|
server = subprocess.Popen([PY, "-m", "jiuwenswarm.app"], stdin=subprocess.DEVNULL,
|
|
274
305
|
stdout=sys.stderr, stderr=sys.stderr, cwd=HOME, env=env)
|
|
275
306
|
bridge = None
|
|
@@ -310,6 +341,14 @@ def main():
|
|
|
310
341
|
return code
|
|
311
342
|
finally:
|
|
312
343
|
stop()
|
|
344
|
+
stop_worker(env)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def main():
|
|
348
|
+
# A long job path exceeds the Unix socket limit. Each unit owns a short,
|
|
349
|
+
# private directory, including on the local backend where /tmp is shared.
|
|
350
|
+
with tempfile.TemporaryDirectory(prefix="nj-", dir="/tmp") as runtime:
|
|
351
|
+
return run(os.path.join(runtime, "worker.sock"))
|
|
313
352
|
|
|
314
353
|
|
|
315
354
|
if __name__ == "__main__":
|
{nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/opencode/agent.json
RENAMED
|
@@ -63,7 +63,8 @@
|
|
|
63
63
|
"agent.task/v1",
|
|
64
64
|
"agent.workspace/v1",
|
|
65
65
|
"agent.tools/v1",
|
|
66
|
-
"agent.conversation/v1"
|
|
66
|
+
"agent.conversation/v1",
|
|
67
|
+
"agent.search/v1"
|
|
67
68
|
],
|
|
68
69
|
"proven": {
|
|
69
70
|
"agent.task/v1": {
|
|
@@ -119,6 +120,16 @@
|
|
|
119
120
|
"kit_version": 1,
|
|
120
121
|
"backend": "docker",
|
|
121
122
|
"job": "2026-11-02__14-14-07__kit__agent-conversation-v1__opencode"
|
|
123
|
+
},
|
|
124
|
+
"agent.search/v1": {
|
|
125
|
+
"passed": "3/3",
|
|
126
|
+
"on": "2026-11-12",
|
|
127
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
128
|
+
"provider": "openrouter",
|
|
129
|
+
"version": "1.18.35",
|
|
130
|
+
"kit_version": 1,
|
|
131
|
+
"backend": "docker",
|
|
132
|
+
"job": "2026-11-12__11-10-20__kit__agent-search-v1__opencode"
|
|
122
133
|
}
|
|
123
134
|
}
|
|
124
135
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "gaia",
|
|
3
|
+
"title": "GAIA",
|
|
4
|
+
"category": "General assistance",
|
|
5
|
+
"domain": "Real-world questions needing reasoning, web research and file understanding.",
|
|
6
|
+
"scoring": "Answer accuracy",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard"
|
|
9
|
+
],
|
|
10
|
+
"origin": "official",
|
|
11
|
+
"good_to_know": [
|
|
12
|
+
"Agents need public internet access.",
|
|
13
|
+
"Accept GAIA's access conditions on Hugging Face, then add a token in Settings → Data providers. Do not redistribute its questions or answers in a crawlable form.",
|
|
14
|
+
"Levels 1, 2 and 3 (53, 86 and 26 questions) split the 165 validation questions.",
|
|
15
|
+
"The authors' scorer gets the text after the last FINAL ANSWER: marker, or the whole reply without one.",
|
|
16
|
+
"Original is the paper's no-tools API baseline; choose an agent for files and web research. The paper ran it 3 times: 3 attempts per question."
|
|
17
|
+
],
|
|
18
|
+
"upstream_limitations": [
|
|
19
|
+
"The 301 test questions have private answers and are excluded from local scoring.",
|
|
20
|
+
"GAIA uses the live web; some evidence may have changed or become unavailable."
|
|
21
|
+
],
|
|
22
|
+
"limitations": [],
|
|
23
|
+
"datasets": [
|
|
24
|
+
{
|
|
25
|
+
"key": "gaia-validation",
|
|
26
|
+
"title": "Validation · All levels",
|
|
27
|
+
"tasks": 165,
|
|
28
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
29
|
+
"source": "Hugging Face",
|
|
30
|
+
"registry": "",
|
|
31
|
+
"hub": "",
|
|
32
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
33
|
+
"notes": "Official 2023 validation split."
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"key": "gaia-validation-level1",
|
|
37
|
+
"title": "Validation · Level 1",
|
|
38
|
+
"tasks": 53,
|
|
39
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
40
|
+
"source": "Hugging Face",
|
|
41
|
+
"registry": "",
|
|
42
|
+
"hub": "",
|
|
43
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
44
|
+
"notes": "Official 2023 validation split."
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"key": "gaia-validation-level2",
|
|
48
|
+
"title": "Validation · Level 2",
|
|
49
|
+
"tasks": 86,
|
|
50
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
51
|
+
"source": "Hugging Face",
|
|
52
|
+
"registry": "",
|
|
53
|
+
"hub": "",
|
|
54
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
55
|
+
"notes": "Official 2023 validation split."
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"key": "gaia-validation-level3",
|
|
59
|
+
"title": "Validation · Level 3",
|
|
60
|
+
"tasks": 26,
|
|
61
|
+
"official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
|
|
62
|
+
"source": "Hugging Face",
|
|
63
|
+
"registry": "",
|
|
64
|
+
"hub": "",
|
|
65
|
+
"verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
|
|
66
|
+
"notes": "Official 2023 validation split."
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
"citation": "@misc{mialon2023gaia,\n title={GAIA: a benchmark for General AI Assistants},\n author={Grégoire Mialon and Clémentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom},\n year={2023},\n eprint={2311.12983},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}",
|
|
70
|
+
"citation_source": "https://huggingface.co/spaces/gaia-benchmark/leaderboard/blob/9f133d71362e77b3539f1514f31b9c101a545fec/content.py",
|
|
71
|
+
"task_count": 165
|
|
72
|
+
}
|