nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
4
|
+
"setting": "--domain airline",
|
|
5
|
+
"values": "a registered domain",
|
|
6
|
+
"default": "none (required)",
|
|
7
|
+
"nullius": "dataset:airline"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
11
|
+
"setting": "--domain retail",
|
|
12
|
+
"values": "a registered domain",
|
|
13
|
+
"default": "none (required)",
|
|
14
|
+
"nullius": "dataset:retail"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
18
|
+
"setting": "--domain telecom",
|
|
19
|
+
"values": "a registered domain",
|
|
20
|
+
"default": "none (required)",
|
|
21
|
+
"nullius": "dataset:telecom"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
25
|
+
"setting": "--domain telecom-workflow",
|
|
26
|
+
"values": "a registered domain (registry.py: the telecom tasks under the workflow policy)",
|
|
27
|
+
"default": "none (required)",
|
|
28
|
+
"nullius": "dataset:telecom-workflow"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
32
|
+
"setting": "--domain banking_knowledge",
|
|
33
|
+
"values": "a registered domain",
|
|
34
|
+
"default": "none (required)",
|
|
35
|
+
"nullius": "dataset:banking_knowledge"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
39
|
+
"setting": "--domain mock",
|
|
40
|
+
"values": "a registered domain",
|
|
41
|
+
"default": "none (required)",
|
|
42
|
+
"nullius": "not offered - the upstream's test fixture (10 tasks for its own tests), not part of the benchmark or the leaderboard"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
46
|
+
"setting": "--num-trials",
|
|
47
|
+
"values": "an integer from 1",
|
|
48
|
+
"default": "1 (DEFAULT_NUM_TRIALS)",
|
|
49
|
+
"nullius": "fixed:1 trial per attempt - a run's --repeat K (the builder's Attempts per unit) runs each unit K times, attempt N being the upstream's trial N-1 at the seed run_tasks draws for it (limits.attempts); the job's result.json carries pass^k"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
53
|
+
"setting": "--agent",
|
|
54
|
+
"values": "llm_agent, llm_agent_gt, llm_agent_solo, discrete_time_audio_native_agent",
|
|
55
|
+
"default": "llm_agent",
|
|
56
|
+
"nullius": "option:upstream_agent"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
60
|
+
"setting": "--agent discrete_time_audio_native_agent",
|
|
61
|
+
"values": "an agent",
|
|
62
|
+
"default": "llm_agent",
|
|
63
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
67
|
+
"setting": "--agent-llm",
|
|
68
|
+
"values": "any litellm model name",
|
|
69
|
+
"default": "gpt-4.1-2025-04-14 (DEFAULT_LLM_AGENT)",
|
|
70
|
+
"nullius": "setup:original"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
74
|
+
"setting": "--agent-llm-args reasoning_effort",
|
|
75
|
+
"values": "minimal, low, medium, high, xhigh (what the model accepts)",
|
|
76
|
+
"default": "not sent",
|
|
77
|
+
"nullius": "option:agent_reasoning_effort"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
81
|
+
"setting": "--agent-llm-args (temperature and any other key)",
|
|
82
|
+
"values": "a JSON object",
|
|
83
|
+
"default": "{\"temperature\": 0.0} (DEFAULT_LLM_ARGS_AGENT)",
|
|
84
|
+
"nullius": "fixed:{\"temperature\": 0.0} - the upstream's default, sent unless a reasoning effort is chosen (the JSON then replaces it, as the CLI's does); other keys are free-form and no leaderboard row of a standard submission documents one"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
88
|
+
"setting": "--user",
|
|
89
|
+
"values": "user_simulator, dummy_user",
|
|
90
|
+
"default": "user_simulator",
|
|
91
|
+
"nullius": "fixed:user_simulator - the upstream's default; dummy_user only pairs with llm_agent_solo, which the upstream_agent option sets with it"
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
95
|
+
"setting": "--user-llm",
|
|
96
|
+
"values": "any litellm model name",
|
|
97
|
+
"default": "gpt-4.1-2025-04-14 (DEFAULT_LLM_USER)",
|
|
98
|
+
"nullius": "fixed:gpt-4.1-2025-04-14 - the upstream's default, a run's --simulator-model (the builder's user simulator model) puts another in its place, as the leaderboard's gpt-5.2; the job records it"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
102
|
+
"setting": "--user-llm-args reasoning_effort",
|
|
103
|
+
"values": "minimal, low, medium, high, xhigh",
|
|
104
|
+
"default": "not sent",
|
|
105
|
+
"nullius": "option:user_reasoning_effort"
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
109
|
+
"setting": "--user-llm-args (temperature and any other key)",
|
|
110
|
+
"values": "a JSON object",
|
|
111
|
+
"default": "{\"temperature\": 0.0} (DEFAULT_LLM_ARGS_USER)",
|
|
112
|
+
"nullius": "fixed:{\"temperature\": 0.0} - the upstream's default, sent unless a reasoning effort is chosen"
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
116
|
+
"setting": "--task-set-name (a domain's own)",
|
|
117
|
+
"values": "a registered task set",
|
|
118
|
+
"default": "the domain's",
|
|
119
|
+
"nullius": "fixed:the dataset's domain - each dataset is its domain's task set, as tau2 run loads it when the flag is left out"
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
123
|
+
"setting": "--task-set-name telecom_full",
|
|
124
|
+
"values": "a registered task set",
|
|
125
|
+
"default": "the domain's",
|
|
126
|
+
"nullius": "not offered - at v1.0.1 the upstream's loader raises TypeError (get_tasks_full() got an unexpected keyword argument 'task_split_name'); the same 2285 tasks are telecom with task_split full"
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
130
|
+
"setting": "--task-set-name telecom_small",
|
|
131
|
+
"values": "a registered task set",
|
|
132
|
+
"default": "the domain's",
|
|
133
|
+
"nullius": "not offered - at v1.0.1 the upstream's loader raises TypeError (get_tasks_small() got an unexpected keyword argument 'task_split_name'); the same 20 tasks are telecom with task_split small"
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
137
|
+
"setting": "--task-split-name",
|
|
138
|
+
"values": "base, train, test (telecom also small, full)",
|
|
139
|
+
"default": "base",
|
|
140
|
+
"nullius": "option:task_split"
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
144
|
+
"setting": "--task-ids",
|
|
145
|
+
"values": "task ids",
|
|
146
|
+
"default": "every task",
|
|
147
|
+
"nullius": "fixed:every task of the split - a run's --limit and shards pick units by position; the leaderboard asks for every task"
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
151
|
+
"setting": "--num-tasks",
|
|
152
|
+
"values": "an integer",
|
|
153
|
+
"default": "every task",
|
|
154
|
+
"nullius": "fixed:every task of the split - a run's --limit N runs the first N, the same cut"
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
158
|
+
"setting": "--max-steps",
|
|
159
|
+
"values": "an integer",
|
|
160
|
+
"default": "200 (DEFAULT_MAX_STEPS)",
|
|
161
|
+
"nullius": "fixed:200 - the upstream's default, the leaderboard's; limits.turns (100) lets its limit be the one met"
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
165
|
+
"setting": "--max-errors",
|
|
166
|
+
"values": "an integer",
|
|
167
|
+
"default": "10 (DEFAULT_MAX_ERRORS)",
|
|
168
|
+
"nullius": "fixed:10 - the upstream's default"
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
172
|
+
"setting": "--timeout",
|
|
173
|
+
"values": "seconds",
|
|
174
|
+
"default": "none",
|
|
175
|
+
"nullius": "fixed:none - the upstream's default; a run's turn limit (600 s an agent turn, 7200 s an original conversation) ends a unit that hangs, as the unit's error"
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
179
|
+
"setting": "--save-to",
|
|
180
|
+
"values": "a path",
|
|
181
|
+
"default": "data/simulations/<name>",
|
|
182
|
+
"nullius": "fixed:none - the SimulationRun is written beside the trial's verdict (verifier/simulation.json)"
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
186
|
+
"setting": "--max-concurrency",
|
|
187
|
+
"values": "an integer",
|
|
188
|
+
"default": "3 (DEFAULT_MAX_CONCURRENCY)",
|
|
189
|
+
"nullius": "fixed:1 - each unit is its own batch run; a run's -j runs units side by side"
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
193
|
+
"setting": "--seed",
|
|
194
|
+
"values": "an integer",
|
|
195
|
+
"default": "300 (DEFAULT_SEED)",
|
|
196
|
+
"nullius": "fixed:300 - the upstream's default and nearly every leaderboard row's"
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
200
|
+
"setting": "--log-level",
|
|
201
|
+
"values": "a loguru level",
|
|
202
|
+
"default": "ERROR (DEFAULT_LOG_LEVEL)",
|
|
203
|
+
"nullius": "fixed:ERROR - the upstream's default, into the driver's log"
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
207
|
+
"setting": "--verbose-logs",
|
|
208
|
+
"values": "a flag",
|
|
209
|
+
"default": "off",
|
|
210
|
+
"nullius": "fixed:off - the relay records every model call of the unit"
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
214
|
+
"setting": "--llm-log-mode",
|
|
215
|
+
"values": "all, latest",
|
|
216
|
+
"default": "latest",
|
|
217
|
+
"nullius": "fixed:latest - acts only with --verbose-logs"
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
221
|
+
"setting": "--max-retries",
|
|
222
|
+
"values": "an integer",
|
|
223
|
+
"default": "3 (DEFAULT_RETRY_ATTEMPTS)",
|
|
224
|
+
"nullius": "fixed:3 - the upstream's default: an attempt that raises is made again, then INFRASTRUCTURE_ERROR is the unit's error"
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
228
|
+
"setting": "--retry-delay",
|
|
229
|
+
"values": "seconds",
|
|
230
|
+
"default": "1.0 (DEFAULT_RETRY_MIN_WAIT)",
|
|
231
|
+
"nullius": "fixed:1.0 - the upstream's default"
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
235
|
+
"setting": "--enforce-communication-protocol",
|
|
236
|
+
"values": "a flag",
|
|
237
|
+
"default": "off",
|
|
238
|
+
"nullius": "fixed:off - the upstream's default and the leaderboard's"
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
242
|
+
"setting": "--user-persona",
|
|
243
|
+
"values": "a JSON persona",
|
|
244
|
+
"default": "none",
|
|
245
|
+
"nullius": "fixed:none - at v1.0.1 tau2 run parses it and passes it to no text run (cli.py builds the PersonaConfig, noqa F841); personas act in voice mode only"
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
249
|
+
"setting": "--auto-resume",
|
|
250
|
+
"values": "a flag",
|
|
251
|
+
"default": "off",
|
|
252
|
+
"nullius": "fixed:off - each unit is a fresh run"
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
256
|
+
"setting": "--auto-review",
|
|
257
|
+
"values": "a flag",
|
|
258
|
+
"default": "off",
|
|
259
|
+
"nullius": "not offered - an LLM review written beside each simulation (claude-opus-4-5 by default); it does not change the reward"
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
263
|
+
"setting": "--review-mode",
|
|
264
|
+
"values": "full, user",
|
|
265
|
+
"default": "full",
|
|
266
|
+
"nullius": "not offered - acts only with --auto-review"
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
270
|
+
"setting": "--review-model",
|
|
271
|
+
"values": "a model",
|
|
272
|
+
"default": "claude-opus-4-5 (DEFAULT_LLM_EVAL_USER_SIMULATOR)",
|
|
273
|
+
"nullius": "not offered - acts only with --auto-review"
|
|
274
|
+
},
|
|
275
|
+
{
|
|
276
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
277
|
+
"setting": "--hallucination-retries",
|
|
278
|
+
"values": "an integer",
|
|
279
|
+
"default": "3",
|
|
280
|
+
"nullius": "fixed:3 - the upstream's default; it acts on full-duplex runs only"
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
284
|
+
"setting": "--retrieval-config",
|
|
285
|
+
"values": "the retrieval variants (retrieval.py)",
|
|
286
|
+
"default": "alltools (DEFAULT_RETRIEVAL_VARIANT) for banking_knowledge",
|
|
287
|
+
"nullius": "option:retrieval"
|
|
288
|
+
},
|
|
289
|
+
{
|
|
290
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
291
|
+
"setting": "--retrieval-config qwen_embeddings",
|
|
292
|
+
"values": "a retrieval variant",
|
|
293
|
+
"default": "alltools",
|
|
294
|
+
"nullius": "option:retrieval"
|
|
295
|
+
},
|
|
296
|
+
{
|
|
297
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
298
|
+
"setting": "--retrieval-config qwen_embeddings_grep",
|
|
299
|
+
"values": "a retrieval variant",
|
|
300
|
+
"default": "alltools",
|
|
301
|
+
"nullius": "option:retrieval"
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
305
|
+
"setting": "--retrieval-config qwen_embeddings_reranker",
|
|
306
|
+
"values": "a retrieval variant",
|
|
307
|
+
"default": "alltools",
|
|
308
|
+
"nullius": "option:retrieval"
|
|
309
|
+
},
|
|
310
|
+
{
|
|
311
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
312
|
+
"setting": "--retrieval-config qwen_embeddings_reranker_grep",
|
|
313
|
+
"values": "a retrieval variant",
|
|
314
|
+
"default": "alltools",
|
|
315
|
+
"nullius": "option:retrieval"
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
319
|
+
"setting": "--retrieval-config alltools-qwen",
|
|
320
|
+
"values": "a retrieval variant",
|
|
321
|
+
"default": "alltools",
|
|
322
|
+
"nullius": "option:retrieval"
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
326
|
+
"setting": "--retrieval-config openai_embeddings_reranker",
|
|
327
|
+
"values": "a retrieval variant",
|
|
328
|
+
"default": "alltools",
|
|
329
|
+
"nullius": "option:retrieval"
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
333
|
+
"setting": "--retrieval-config openai_embeddings_reranker_grep",
|
|
334
|
+
"values": "a retrieval variant",
|
|
335
|
+
"default": "alltools",
|
|
336
|
+
"nullius": "option:retrieval"
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
340
|
+
"setting": "--retrieval-config bm25_reranker",
|
|
341
|
+
"values": "a retrieval variant",
|
|
342
|
+
"default": "alltools",
|
|
343
|
+
"nullius": "option:retrieval"
|
|
344
|
+
},
|
|
345
|
+
{
|
|
346
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
347
|
+
"setting": "--retrieval-config bm25_reranker_grep",
|
|
348
|
+
"values": "a retrieval variant",
|
|
349
|
+
"default": "alltools",
|
|
350
|
+
"nullius": "option:retrieval"
|
|
351
|
+
},
|
|
352
|
+
{
|
|
353
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
354
|
+
"setting": "--retrieval-config-kwargs",
|
|
355
|
+
"values": "JSON: top_k, grep_top_k, case_sensitive, reranker_min_score",
|
|
356
|
+
"default": "none",
|
|
357
|
+
"nullius": "not offered - free-form overrides of a variant's constructor; the leaderboard's rows run every variant at its own values, which a score is compared at"
|
|
358
|
+
},
|
|
359
|
+
{
|
|
360
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
361
|
+
"setting": "--audio-native",
|
|
362
|
+
"values": "voice mode",
|
|
363
|
+
"default": "off / its default",
|
|
364
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
365
|
+
},
|
|
366
|
+
{
|
|
367
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
368
|
+
"setting": "--audio-native-provider",
|
|
369
|
+
"values": "voice mode",
|
|
370
|
+
"default": "off / its default",
|
|
371
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
372
|
+
},
|
|
373
|
+
{
|
|
374
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
375
|
+
"setting": "--cascaded-config",
|
|
376
|
+
"values": "voice mode",
|
|
377
|
+
"default": "off / its default",
|
|
378
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
382
|
+
"setting": "--audio-native-model",
|
|
383
|
+
"values": "voice mode",
|
|
384
|
+
"default": "off / its default",
|
|
385
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
386
|
+
},
|
|
387
|
+
{
|
|
388
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
389
|
+
"setting": "--reasoning-effort",
|
|
390
|
+
"values": "voice mode",
|
|
391
|
+
"default": "off / its default",
|
|
392
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
393
|
+
},
|
|
394
|
+
{
|
|
395
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
396
|
+
"setting": "--tick-duration",
|
|
397
|
+
"values": "voice mode",
|
|
398
|
+
"default": "off / its default",
|
|
399
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
400
|
+
},
|
|
401
|
+
{
|
|
402
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
403
|
+
"setting": "--max-steps-seconds",
|
|
404
|
+
"values": "voice mode",
|
|
405
|
+
"default": "off / its default",
|
|
406
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
410
|
+
"setting": "--speech-complexity",
|
|
411
|
+
"values": "voice mode",
|
|
412
|
+
"default": "off / its default",
|
|
413
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
417
|
+
"setting": "--pcm-sample-rate",
|
|
418
|
+
"values": "voice mode",
|
|
419
|
+
"default": "off / its default",
|
|
420
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
421
|
+
},
|
|
422
|
+
{
|
|
423
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
424
|
+
"setting": "--telephony-rate",
|
|
425
|
+
"values": "voice mode",
|
|
426
|
+
"default": "off / its default",
|
|
427
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
428
|
+
},
|
|
429
|
+
{
|
|
430
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
431
|
+
"setting": "--wait-to-respond-other",
|
|
432
|
+
"values": "voice mode",
|
|
433
|
+
"default": "off / its default",
|
|
434
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
435
|
+
},
|
|
436
|
+
{
|
|
437
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
438
|
+
"setting": "--wait-to-respond-self",
|
|
439
|
+
"values": "voice mode",
|
|
440
|
+
"default": "off / its default",
|
|
441
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
442
|
+
},
|
|
443
|
+
{
|
|
444
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
445
|
+
"setting": "--yield-when-interrupted",
|
|
446
|
+
"values": "voice mode",
|
|
447
|
+
"default": "off / its default",
|
|
448
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
449
|
+
},
|
|
450
|
+
{
|
|
451
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
452
|
+
"setting": "--yield-when-interrupting",
|
|
453
|
+
"values": "voice mode",
|
|
454
|
+
"default": "off / its default",
|
|
455
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
456
|
+
},
|
|
457
|
+
{
|
|
458
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
459
|
+
"setting": "--interruption-check-interval",
|
|
460
|
+
"values": "voice mode",
|
|
461
|
+
"default": "off / its default",
|
|
462
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
463
|
+
},
|
|
464
|
+
{
|
|
465
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
466
|
+
"setting": "--integration-duration",
|
|
467
|
+
"values": "voice mode",
|
|
468
|
+
"default": "off / its default",
|
|
469
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
470
|
+
},
|
|
471
|
+
{
|
|
472
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
473
|
+
"setting": "--silence-annotation-threshold",
|
|
474
|
+
"values": "voice mode",
|
|
475
|
+
"default": "off / its default",
|
|
476
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
477
|
+
},
|
|
478
|
+
{
|
|
479
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
480
|
+
"setting": "--xml-prompt",
|
|
481
|
+
"values": "voice mode",
|
|
482
|
+
"default": "off / its default",
|
|
483
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
484
|
+
},
|
|
485
|
+
{
|
|
486
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
487
|
+
"setting": "--no-xml-prompt",
|
|
488
|
+
"values": "voice mode",
|
|
489
|
+
"default": "off / its default",
|
|
490
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
491
|
+
},
|
|
492
|
+
{
|
|
493
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
494
|
+
"setting": "--audio-debug",
|
|
495
|
+
"values": "voice mode",
|
|
496
|
+
"default": "off / its default",
|
|
497
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
498
|
+
},
|
|
499
|
+
{
|
|
500
|
+
"entry": "tau2 run (src/tau2/cli.py)",
|
|
501
|
+
"setting": "--audio-taps",
|
|
502
|
+
"values": "voice mode",
|
|
503
|
+
"default": "off / its default",
|
|
504
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
505
|
+
},
|
|
506
|
+
{
|
|
507
|
+
"entry": "src/tau2/config.py",
|
|
508
|
+
"setting": "DEFAULT_LLM_NL_ASSERTIONS",
|
|
509
|
+
"values": "a model",
|
|
510
|
+
"default": "gpt-4.1-2025-04-14",
|
|
511
|
+
"nullius": "fixed:gpt-4.1-2025-04-14 - the upstream's NL-assertion judge (score.judge); a run's --judge-model puts another in its place, recorded in the job"
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
"entry": "src/tau2/config.py",
|
|
515
|
+
"setting": "DEFAULT_LLM_NL_ASSERTIONS_ARGS",
|
|
516
|
+
"values": "a JSON object",
|
|
517
|
+
"default": "{\"temperature\": 0.0}",
|
|
518
|
+
"nullius": "fixed:{\"temperature\": 0.0} - the upstream's"
|
|
519
|
+
},
|
|
520
|
+
{
|
|
521
|
+
"entry": "src/tau2/config.py",
|
|
522
|
+
"setting": "DEFAULT_LLM_ENV_INTERFACE (and its args)",
|
|
523
|
+
"values": "a model",
|
|
524
|
+
"default": "gpt-4.1-2025-04-14",
|
|
525
|
+
"nullius": "not offered - the upstream's interactive interface agent (tau2 play), not part of a run"
|
|
526
|
+
},
|
|
527
|
+
{
|
|
528
|
+
"entry": "src/tau2/config.py",
|
|
529
|
+
"setting": "DEFAULT_MAX_RETRIES (litellm num_retries)",
|
|
530
|
+
"values": "an integer",
|
|
531
|
+
"default": "3",
|
|
532
|
+
"nullius": "fixed:3 - the upstream's, each model call retried by litellm"
|
|
533
|
+
},
|
|
534
|
+
{
|
|
535
|
+
"entry": "src/tau2/config.py",
|
|
536
|
+
"setting": "DEFAULT_RETRY_ATTEMPTS / MIN_WAIT / MAX_WAIT / MULTIPLIER",
|
|
537
|
+
"values": "numbers",
|
|
538
|
+
"default": "3 / 1.0 / 10.0 / 1.0",
|
|
539
|
+
"nullius": "fixed:3 / 1.0 / 10.0 / 1.0 - the upstream's constants"
|
|
540
|
+
},
|
|
541
|
+
{
|
|
542
|
+
"entry": "src/tau2/config.py",
|
|
543
|
+
"setting": "LLM_CACHE_ENABLED (redis)",
|
|
544
|
+
"values": "a boolean",
|
|
545
|
+
"default": "False",
|
|
546
|
+
"nullius": "fixed:False - the upstream's default: every call is made"
|
|
547
|
+
},
|
|
548
|
+
{
|
|
549
|
+
"entry": "src/tau2/config.py",
|
|
550
|
+
"setting": "USE_LANGFUSE",
|
|
551
|
+
"values": "a boolean",
|
|
552
|
+
"default": "False",
|
|
553
|
+
"nullius": "fixed:False - the upstream's default"
|
|
554
|
+
},
|
|
555
|
+
{
|
|
556
|
+
"entry": "src/tau2/config.py",
|
|
557
|
+
"setting": "DEFAULT_VOICE_* and audio-native constants",
|
|
558
|
+
"values": "voice settings",
|
|
559
|
+
"default": "theirs",
|
|
560
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
561
|
+
},
|
|
562
|
+
{
|
|
563
|
+
"entry": "environment variables read by src/tau2",
|
|
564
|
+
"setting": "OPENAI_API_KEY",
|
|
565
|
+
"values": "a key",
|
|
566
|
+
"default": "unset",
|
|
567
|
+
"nullius": "fixed:the relay's placeholder - every model and embedding call goes through the run's relay, which holds the key"
|
|
568
|
+
},
|
|
569
|
+
{
|
|
570
|
+
"entry": "environment variables read by src/tau2",
|
|
571
|
+
"setting": "OPENROUTER_API_KEY",
|
|
572
|
+
"values": "a key",
|
|
573
|
+
"default": "unset",
|
|
574
|
+
"nullius": "fixed:unused - the driver and MCP server override only the pinned Qwen embedder's client URL and key with the search relay's URL and placeholder; the provider key stays in the relay"
|
|
575
|
+
},
|
|
576
|
+
{
|
|
577
|
+
"entry": "environment variables read by src/tau2",
|
|
578
|
+
"setting": "TAU2_DATA_DIR",
|
|
579
|
+
"values": "a path",
|
|
580
|
+
"default": "the upstream's data/",
|
|
581
|
+
"nullius": "fixed:the upstream's data/ at its pin - the datasets' sha256 are checked"
|
|
582
|
+
},
|
|
583
|
+
{
|
|
584
|
+
"entry": "environment variables read by src/tau2",
|
|
585
|
+
"setting": "BASETEN_API_KEY / BGE_RERANKER_API_KEY / QWEN_RERANKER_API_KEY",
|
|
586
|
+
"values": "keys",
|
|
587
|
+
"default": "unset",
|
|
588
|
+
"nullius": "not offered - rerankers on Sierra's own hosted endpoints, used by no retrieval variant tau2 run offers"
|
|
589
|
+
},
|
|
590
|
+
{
|
|
591
|
+
"entry": "environment variables read by src/tau2",
|
|
592
|
+
"setting": "ELEVENLABS / DEEPGRAM / GEMINI / XAI / DASHSCOPE / GOOGLE / AWS keys",
|
|
593
|
+
"values": "keys",
|
|
594
|
+
"default": "unset",
|
|
595
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
596
|
+
},
|
|
597
|
+
{
|
|
598
|
+
"entry": "docs/leaderboard-submission.md",
|
|
599
|
+
"setting": "submission_type standard",
|
|
600
|
+
"values": "standard, custom",
|
|
601
|
+
"default": "standard",
|
|
602
|
+
"nullius": "setup:original"
|
|
603
|
+
},
|
|
604
|
+
{
|
|
605
|
+
"entry": "docs/leaderboard-submission.md",
|
|
606
|
+
"setting": "trials (4 or more)",
|
|
607
|
+
"values": "an integer",
|
|
608
|
+
"default": "4",
|
|
609
|
+
"nullius": "fixed:1 a unit's attempt - --repeat 4 runs the leaderboard's four trials (limits.attempts)"
|
|
610
|
+
},
|
|
611
|
+
{
|
|
612
|
+
"entry": "docs/leaderboard-submission.md",
|
|
613
|
+
"setting": "user simulator gpt-5.2 at reasoning_effort low",
|
|
614
|
+
"values": "a model and its arguments",
|
|
615
|
+
"default": "recommended",
|
|
616
|
+
"nullius": "option:user_reasoning_effort"
|
|
617
|
+
},
|
|
618
|
+
{
|
|
619
|
+
"entry": "docs/leaderboard-submission.md",
|
|
620
|
+
"setting": "every task, base split",
|
|
621
|
+
"values": "the split",
|
|
622
|
+
"default": "base",
|
|
623
|
+
"nullius": "option:task_split"
|
|
624
|
+
},
|
|
625
|
+
{
|
|
626
|
+
"entry": "docs/leaderboard-submission.md",
|
|
627
|
+
"setting": "banking_knowledge retrieval_config",
|
|
628
|
+
"values": "a variant",
|
|
629
|
+
"default": "reported as a badge",
|
|
630
|
+
"nullius": "option:retrieval"
|
|
631
|
+
},
|
|
632
|
+
{
|
|
633
|
+
"entry": "docs/leaderboard-submission.md",
|
|
634
|
+
"setting": "voice: speech complexity regular",
|
|
635
|
+
"values": "control, regular",
|
|
636
|
+
"default": "regular",
|
|
637
|
+
"nullius": "not offered - voice (audio-native, full-duplex) mode only: Nullius runs the text, half-duplex benchmark"
|
|
638
|
+
}
|
|
639
|
+
]
|