nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,763 @@
1
+ /* Pure recipe compiler for Harbor recipes. It prints shell; it has no DOM, storage, or execution API.
2
+ This file also creates, loads and saves Nullius standard recipes; the server compiles those
3
+ (POST /new-run/runner, standard/recipe.py). Other harnesses are rejected. */
4
+ (function (root, factory) {
5
+ if (typeof module === 'object' && module.exports) module.exports = factory;
6
+ else root.NulliusRecipe = factory;
7
+ })(typeof globalThis !== 'undefined' ? globalThis : this, function (schema) {
8
+ 'use strict';
9
+ const own = (o, k) => Object.prototype.hasOwnProperty.call(o, k);
10
+ const copy = value => JSON.parse(JSON.stringify(value));
11
+ const q = value => "'" + String(value).replace(/'/g, "'\\''") + "'";
12
+ const path = value => String(value).startsWith('~/') ? '"$HOME"/' + q(String(value).slice(2)) : q(value);
13
+ const env = name => '"${' + name + ':?Set ' + name + ' in your shell}"';
14
+ const join = parts => parts.join(' \\\n ');
15
+ const textValue = value => typeof value === 'string' ? value : JSON.stringify(value);
16
+ // Keep string literals (e.g. 'false' or '12') distinct from typed values
17
+ // when the harness decodes key=value through JSON/YAML.
18
+ const kwargValue = value => typeof value === 'string' && /^[A-Za-z_][A-Za-z0-9_./-]*$/.test(value) && !/^(?:true|false|null|none|yes|no|on|off)$/i.test(value) ? value : JSON.stringify(value);
19
+ const parameters = () => schema.parameters;
20
+ // Error step numbers match nullius/providers.py.
21
+ // Setup is a reserved position in saved drafts; the Agent step selects setups, agents and checks.
22
+ // The benchmark first, then what runs it: a runner is only a choice when the benchmark has both.
23
+ const STEPS = ['Benchmark', 'Runner', 'Dataset', 'Setup', 'Agent', 'Model', 'Limits', 'Where', 'Review'];
24
+ const STEP = {benchmark: 0, runner: 1, dataset: 2, setup: 3, agent: 4, model: 5, limits: 6, where: 7, review: 8};
25
+ const runners = () => schema.runners || [{key: 'harbor', label: 'Harbor', builtin: true}];
26
+ // Unsupported runners use the server's rejection message.
27
+ const RUNNER_GONE = schema.runnerGone || 'Only Harbor and Nullius recipes are supported.';
28
+ // The Nullius standard (nullius/standard/recipe.py): a benchmark, an agent or a capability that proved the ports it
29
+ // needs, a model. Its choices and why a pair does not fit come with the schema; the server compiles the recipe.
30
+ const isStandard = r => !!(r && r.harness === 'standard');
31
+ const standard = () => schema.standard || {benchmarks: [], agents: [], capabilities: [], fits: {}, ports: {}};
32
+ // A recipe names a benchmark by its key (its default version) or key@version, one of the page's versions.
33
+ const standardBench = r => standard().benchmarks.find(b => b.key === r.benchmark || b.ref === r.benchmark
34
+ || (b.versions || []).some(v => v.ref === r.benchmark)) || null;
35
+ // Every agent (or capability) a benchmark of the standard could take, each with the reasons it does not fit: none, it fits.
36
+ // Then the controls it declares (nop, oracle): no agent, no model, always runnable - each with its source
37
+ // (official, adapter, nullius) and, per dataset, its proof.
38
+ function standardOthers(r) {
39
+ const b = standardBench(r);
40
+ if (!b) return [];
41
+ const why = standard().fits[r.benchmark] || standard().fits[b.key] || {};
42
+ const controls = (b.controls || []).map(c => ({key: c.key, label: c.label, summary: c.from, protocol: 'no agent', source: c.source,
43
+ version: '', offers: [], proven: {}, options: {}, control: true, why: []}));
44
+ // its setups (recipe.py, _bench_page): the benchmark as published, run by its own code - beside the agents or capabilities
45
+ const setups = (b.setups || []).map(x => ({...x, protocol: 'its own code', version: '', offers: [], proven: {}, options: {}, setup: true, why: []}));
46
+ return (b.family === 'agent' ? standard().agents : standard().capabilities).map(o => ({...o, why: why[o.key] || []})).concat(setups, controls);
47
+ }
48
+ // A control that calls no model at all - on a benchmark with no judge - needs no key to run.
49
+ // The agents, capabilities and controls a standard recipe runs, each its own job of the one run.
50
+ const chosenOf = r => Array.isArray(r.others) ? r.others : (r.other ? [r.other] : []);
51
+ // The datasets a standard recipe runs: each with each choice, a job each; dataset also accepts one key.
52
+ const datasetsOf = r => Array.isArray(r.datasets) ? r.datasets : (r.dataset ? [r.dataset] : []);
53
+ const standardKeyless = r => { const b = standardBench(r), c = chosenOf(r); return !!b && !b.judged && c.length > 0 && c.every(k => (b.controls || []).some(x => x.key === k)); };
54
+ // Where a standard recipe runs: a list of machines, 'local' or a host's entry; one machine is kept as a word.
55
+ const whereList = r => Array.isArray(r.where) ? r.where : [r.where || 'local'];
56
+ // A step this recipe shows: every one but Setup.
57
+ const stepShown = (r, i) => i !== STEP.setup;
58
+ // Every participant except an unjudged control uses the one selected model.
59
+ const paperModels = r => ((standardBench(r) || {}).setups || []).find(x => x.key === 'original') || {};
60
+ // Like the paper keeps what the paper fixed: its models when it lists them, its judge and its user simulator.
61
+ const paperListed = r => !!(paperModels(r).models || []).length;
62
+ const paperOffered = r => { const b = standardBench(r) || {}; return paperListed(r) || !!(b.judge || b.simulator); };
63
+ // Like the leaderboard: what the upstream's leaderboard fixes besides the ranked model, when the benchmark declares it.
64
+ const leaderboard = r => (standardBench(r) || {}).leaderboard || null;
65
+ // Choose how the model is chosen: paper or leaderboard keep their fixed models (the leaderboard also its options and
66
+ // fewest attempts); leaving the leaderboard gives its options back to their defaults.
67
+ function setModelChoice(r, choice) {
68
+ const lb = leaderboard(r), was = r.model_choice;
69
+ r.model_choice = choice;
70
+ if (paperListed(r)) r.model = '';
71
+ if (choice !== 'any') { delete r.judge_model; delete r.simulator_model; }
72
+ if (lb && was === 'leaderboard' && choice !== 'leaderboard') {
73
+ r.bench_options = {...(r.bench_options || {})};
74
+ for (const k of Object.keys(lb.bench_options || {})) delete r.bench_options[k];
75
+ }
76
+ if (lb && choice === 'leaderboard') {
77
+ r.bench_options = {...(r.bench_options || {}), ...(lb.bench_options || {})};
78
+ r.repeat = Math.max(Number(r.repeat) || 1, lb.repeat || 1);
79
+ }
80
+ return r;
81
+ }
82
+ function createStandard(benchmark = '', other = '') {
83
+ const st = standard(), b = st.benchmarks.find(x => x.key === benchmark) || st.benchmarks[0];
84
+ const r = {version: 1, harness: 'standard', name: 'standard-experiment', title: 'Standard experiment', benchmark: '', datasets: [], others: [],
85
+ model_choice: 'any', provider: firstAvailable(PROVIDERS, 'openai'), model: bareModel(schema.defaultModel || ''), limit: '', concurrency: '', unsandboxed: false, where: 'local',
86
+ backend: 'local', options: {}};
87
+ return b ? setStandardBenchmark(r, b.key, other) : r;
88
+ }
89
+ // Keep only options and values the benchmark accepts.
90
+ const ownOptions = (b, opts) => Object.fromEntries(Object.entries(opts || {}).filter(([k, v]) => {
91
+ const o = ((b || {}).options || {})[k]; return o && (o.values ? o.values.includes(v) : new RegExp('^(?:' + o.pattern + ')$').test(v)); }));
92
+ // key: a benchmark's key, or one of its versions' refs (key@version)
93
+ function setStandardBenchmark(r, key, other = '') {
94
+ const b = standardBench({benchmark: key});
95
+ if (!b) return r;
96
+ const was = standardBench(r);
97
+ if (was !== b) r.model_choice = 'any';
98
+ r.benchmark = key;
99
+ const kept = was === b ? datasetsOf(r).filter(id => b.datasets.some(d => d.id === id)) : [];
100
+ r.datasets = kept.length ? kept : [b.datasets[0].id];
101
+ delete r.dataset;
102
+ const fitting = standardOthers(r).filter(o => !o.why.length);
103
+ const keys = fitting.map(o => o.key), first = fitting.find(o => o.setup && o.key === 'original')
104
+ || fitting.find(o => !o.control && !o.setup);
105
+ // Keep explicit choices that still fit; otherwise prefer the benchmark's reference.
106
+ r.others = other && keys.includes(other) ? [other] : chosenOf(r).filter(k => keys.includes(k));
107
+ delete r.other;
108
+ if (!r.others.length && first) r.others = [first.key];
109
+ r.options = Object.fromEntries(Object.entries(r.options || {}).filter(([k, v]) => r.others.includes(k) && v && typeof v === 'object')); // options are each agent's own
110
+ if (r.bench_options) r.bench_options = ownOptions(b, r.bench_options);
111
+ if (was !== b || r.name === 'standard-experiment') { r.name = b.key + '-experiment'; r.title = b.label + ' experiment'; }
112
+ return r;
113
+ }
114
+ // A saved standard recipe read back: known fields only, each of its type.
115
+ function standardFromJSON(raw) {
116
+ const r = createStandard(typeof raw.benchmark === 'string' ? raw.benchmark : '');
117
+ for (const key of ['name', 'title', 'model', 'where', 'model_choice']) if (typeof raw[key] === 'string') r[key] = raw[key];
118
+ const sets = Array.isArray(raw.datasets) ? raw.datasets.filter(k => typeof k === 'string') : typeof raw.dataset === 'string' && raw.dataset ? [raw.dataset] : null;
119
+ if (sets && sets.length) r.datasets = [...new Set(sets)];
120
+ const picked = Array.isArray(raw.others) ? raw.others.filter(k => typeof k === 'string') : typeof raw.other === 'string' && raw.other ? [raw.other] : null;
121
+ if (picked) r.others = [...new Set(picked)];
122
+ if (Array.isArray(raw.where) && raw.where.length && raw.where.every(w => typeof w === 'string')) r.where = raw.where.length > 1 ? [...raw.where] : raw.where[0];
123
+ if (['local', 'container', 'docker'].includes(raw.backend)) r.backend = raw.backend === 'docker' ? 'container' : raw.backend; // docker: its earlier name
124
+ if (raw.bench_options && typeof raw.bench_options === 'object' && !Array.isArray(raw.bench_options)) // the benchmark's own options, a word each
125
+ r.bench_options = ownOptions(standardBench(r), Object.fromEntries(Object.entries(raw.bench_options).filter(([, v]) => typeof v === 'string')));
126
+ // Options per choice, {ref: {name: word}}; a single agent also accepts flat {name: word} options.
127
+ if (raw.options && typeof raw.options === 'object' && !Array.isArray(raw.options)) {
128
+ const words = o => Object.fromEntries(Object.entries(o).filter(([, v]) => typeof v === 'string'));
129
+ const flat = !Object.values(raw.options).some(v => v && typeof v === 'object');
130
+ r.options = flat ? (r.others.length === 1 ? {[r.others[0]]: words(raw.options)} : {})
131
+ : Object.fromEntries(Object.entries(raw.options).filter(([k, v]) => r.others.includes(k) && v && typeof v === 'object').map(([k, v]) => [k, words(v)]));
132
+ }
133
+ if (PROVIDERS.includes(raw.provider)) r.provider = raw.provider;
134
+ keepEndpoint(r, raw);
135
+ if ((Number.isInteger(raw.limit) && raw.limit >= 1) || raw.limit === '') r.limit = raw.limit;
136
+ if ((Number.isInteger(raw.concurrency) && raw.concurrency >= 1) || raw.concurrency === '') r.concurrency = raw.concurrency;
137
+ // attempts per unit and the judge's and user simulator's models: a reopened recipe runs as it was saved
138
+ if (Number.isInteger(raw.repeat) && raw.repeat >= 1 && raw.repeat <= 32) r.repeat = raw.repeat;
139
+ // the run's spending cap in USD, shared by its jobs (recipe.py): an amount above zero, or none
140
+ if (typeof raw.max_usd === 'number' && Number.isFinite(raw.max_usd) && raw.max_usd > 0) r.max_usd = raw.max_usd;
141
+ // and each unit's own: a unit that reaches it is cut off there, the run going on
142
+ if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit;
143
+ for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim();
144
+ r.unsandboxed = raw.unsandboxed === true;
145
+ if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id';
146
+ else r.model = bareModel(r.model);
147
+ if (typeof raw.title !== 'string' || !raw.title.trim()) r.title = r.name;
148
+ return r;
149
+ }
150
+ // Custom agents are import paths typed per recipe: `custom-<n>` names the
151
+ // nth entry of r.customAgents, and behaves as an agent under evaluation.
152
+ const customIndex = id => /^custom-\d+$/.test(id) ? Number(id.slice(7)) : -1;
153
+ const agentById = (id, r) => customIndex(id) >= 0
154
+ ? (r && r.customAgents[customIndex(id)] !== undefined ? {id, label: 'Custom · ' + (r.customAgents[customIndex(id)] || 'import path'), arg: r.customAgents[customIndex(id)], control: false, kind: 'evaluated', custom: true} : undefined)
155
+ : agents().find(a => a.id === id);
156
+ // The agents on offer: the vocabulary (launch.py), each marked whether the
157
+ // recipe's Harbor registers it, and - once that Harbor has been scanned
158
+ // (GET/POST /new-run/harbor/agents) - every other name it registers, as a
159
+ // Harbor built-in named as Harbor names it. Without a scan there are no
160
+ // built-ins to offer, and the page says where they come from.
161
+ let scannedAgents = null;
162
+ function mergeAgents(names, source) {
163
+ if (!Array.isArray(names)) return schema.agents.map(a => ({...a, scanned: false}));
164
+ const has = new Set(names), vocab = new Set(schema.agents.map(a => a.arg));
165
+ return [...schema.agents.map(a => ({...a, scanned: has.has(a.arg)})),
166
+ ...[...has].filter(n => !vocab.has(n)).sort().map(n => ({id: n, label: n, arg: n, control: false, kind: 'builtin', scanned: true,
167
+ summary: source ? 'Harbor built-in, as ' + source + ' registers it.' : 'Harbor built-in.'}))];
168
+ }
169
+ function setScanned(names, source) { scannedAgents = Array.isArray(names) ? mergeAgents(names, source) : null; return agents(); }
170
+ const agents = () => scannedAgents || schema.agents;
171
+ const secret = key => /(?:api[-_]?key|token|password|authorization|credential|secret)/i.test(key);
172
+ const defaults = () => Object.fromEntries(parameters().map(p => [p.key, p.default]));
173
+ // The provider is a recipe field and the model a bare name. Each consumer gets
174
+ // the prefix matching its credential bindings; a typed model prefix takes precedence.
175
+ const PROVIDERS = ['openrouter', 'openai', 'custom'];
176
+ const PROVIDER_PREFIX = /^(openrouter|openai)\//;
177
+ const bareModel = m => String(m || '').replace(PROVIDER_PREFIX, '');
178
+ const providerOf = (r, m) => { if(r.modelFormat === 'provider-id' && PROVIDERS.includes(r.provider))return r.provider; const x = String(m || '').match(PROVIDER_PREFIX); return x ? x[1] : (PROVIDERS.includes(r.provider) ? r.provider : 'openrouter'); };
179
+ // A Harbor recipe's provider and model settled once, the model as its provider names it (modelFormat
180
+ // 'provider-id'): the page then reads the provider /run uses, the recipe's own. A leading openrouter/ or
181
+ // openai/ says the provider only where the recipe names none; on OpenRouter openai/ is the model's own
182
+ // name there (openai/gpt-4o), never another provider.
183
+ function settleModel(provider, model) {
184
+ const m = String(model || ''), x = m.match(PROVIDER_PREFIX), p = PROVIDERS.includes(provider) ? provider : '';
185
+ if (!x) return [p || 'openrouter', m];
186
+ if (x[1] === 'openai' && p === 'openrouter') return [p, m];
187
+ const q = p || x[1], rest = m.slice(x[0].length);
188
+ return [q, q === 'openrouter' ? rest : rest.replace(PROVIDER_PREFIX, '')];
189
+ }
190
+ // Into a recipe: a bare name as before, but an OpenRouter model named openai/... (which a bare name cannot
191
+ // say, its prefix reading as the OpenAI provider) as the provider names it.
192
+ function adoptModel(r, provider, model, key = 'model') {
193
+ const [p, m] = settleModel(provider, model);
194
+ r.provider = p;
195
+ if (r.modelFormat === 'provider-id' || (p === 'openrouter' && PROVIDER_PREFIX.test(m))) {
196
+ if (r.modelFormat !== 'provider-id') for (const k of ['model', 'userModel', 'evalModel']) if (k !== key && typeof r[k] === 'string') r[k] = settleModel(p, r[k])[1];
197
+ r.modelFormat = 'provider-id'; r[key] = m;
198
+ } else r[key] = bareModel(m);
199
+ }
200
+ // Where a run started here can go: the providers whose key - for an OpenAI-compatible endpoint, its URL, the key
201
+ // being optional - is set in Settings (schema.credentials, runners.available), in the order the Model step prefers
202
+ // them. With none, every provider stays visible. A schema that does not say (a test's) is
203
+ // checked by the server alone.
204
+ const PROVIDER_LABEL = {openrouter: 'OpenRouter', openai: 'OpenAI', custom: 'OpenAI-compatible'};
205
+ const PROVIDER_NEEDS = {openrouter: 'OPENROUTER_API_KEY', openai: 'OPENAI_API_KEY', custom: 'MODEL_API_URL'};
206
+ // A named OpenAI-compatible endpoint: the recipe keeps provider custom and the endpoint's id, which names its
207
+ // settings (MODEL_<ID>_API_URL); the Model step offers it as the choice custom:<id> (nullius/providers.py).
208
+ const ENDPOINT_ID = /^(?=.{1,32}$)[A-Z0-9]+(?:_[A-Z0-9]+)*$/;
209
+ function keepEndpoint(r, raw) { if (r.provider === 'custom' && ENDPOINT_ID.test(raw.endpoint || '')) r.endpoint = raw.endpoint; else delete r.endpoint; }
210
+ const kindOf = c => String(c).split(':')[0];
211
+ const endpointOf = c => String(c).split(':')[1] || '';
212
+ const labelOf = c => endpointOf(c) ? ((schema.credentials || {}).endpoints || {})[endpointOf(c)] || endpointOf(c) : PROVIDER_LABEL[c] || c;
213
+ const needsOf = c => endpointOf(c) ? 'MODEL_' + endpointOf(c) + '_API_URL' : PROVIDER_NEEDS[c] || PROVIDER_NEEDS.custom;
214
+ const available = () => ((schema.credentials || {}).available || []).filter(p => PROVIDERS.includes(kindOf(p)));
215
+ const firstAvailable = (offered, otherwise) => available().find(p => offered.includes(p)) || otherwise;
216
+ const providersOf = () => PROVIDERS;
217
+ const recipeProvider = r => isStandard(r) ? r.provider : providerOf(r, r.model || r.userModel);
218
+ const choiceOf = r => recipeProvider(r) === 'custom' && r.endpoint ? 'custom:' + r.endpoint : recipeProvider(r);
219
+ // The Model step's choice into the recipe: its provider, and for a named endpoint its id.
220
+ function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); }
221
+ // Why Run cannot start this recipe here, or '': its provider is not one of those. A recipe naming one keeps it,
222
+ // said, until another is chosen - never switched behind the person's back. The server refuses the same
223
+ // (web/execute.py, provider_problem).
224
+ function providerBlocker(r) {
225
+ if (!schema.credentials || (!isStandard(r) && !needsCredentials(r))) return '';
226
+ if (isStandard(r) && standardKeyless(r)) return ''; // a control of an unjudged benchmark is handed no key (execution.Plan.keyless)
227
+ const offered = providersOf(r), have = available().filter(p => offered.includes(kindOf(p))), p = choiceOf(r);
228
+ const or = xs => xs.slice(0, -1).join(', ') + (xs.length > 1 ? ' or ' : '') + xs[xs.length - 1];
229
+ if (have.includes(p)) return '';
230
+ if (!have.length) return 'Run needs a model key, and none is set in Settings: set ' + or(offered.map(x => PROVIDER_NEEDS[x])) + ' there.';
231
+ if (((schema.credentials || {}).broken || []).includes(p))
232
+ return 'This recipe runs on ' + labelOf(p) + ', and its URL in Settings holds a credential or is not an http(s) URL. Put the key in the Key field in Settings.';
233
+ return 'This recipe runs on ' + labelOf(p) + ', and ' + needsOf(p) + ' is not set in Settings. Choose '
234
+ + or(have.map(labelOf)) + ' on the Model step to run it, or set it in Settings.';
235
+ }
236
+ // Harbor's registry agents: routed by prefix; an OpenAI-compatible endpoint is spelled openai/.
237
+ const agentModel = (r, m = r.model) => (providerOf(r, m) === 'openrouter' ? 'openrouter' : 'openai') + '/' + (r.modelFormat === 'provider-id' ? String(m || '') : bareModel(m));
238
+ // The τ² simulated user and judge: litellm with OPENAI_API_KEY + OPENAI_BASE_URL, whatever the provider.
239
+ const tau2Model = (m, r) => 'openai/' + (r.modelFormat === 'provider-id' ? String(m || '') : bareModel(m));
240
+ // The τ³ reference agent: litellm-native, with the provider's own key name bound beside it (credentialBindings).
241
+ // The τ³ runtime container reads that same name too: τ²'s OpenRouter embedder (dense
242
+ // search in the alltools-qwen retrieval setup) takes OPENROUTER_API_KEY and nothing
243
+ // else, so every part of a τ³ run on OpenRouter carries it, including the reference agent.
244
+ const litellmModel = (r, m = r.model) => agentModel(r, m);
245
+ const benchmarkCallsModel = r => r.benchmark === 'tau3-bench';
246
+ const needsCredentials = r => benchmarkCallsModel(r) || r.agents.some(id => !agentById(id, r)?.control);
247
+ function credentialBindings(r, agentId) {
248
+ if (!needsCredentials(r)) return [];
249
+ const agentCallsModel = agentId ? !agentById(agentId, r)?.control : r.agents.some(id => !agentById(id, r)?.control);
250
+ if (!agentCallsModel && !benchmarkCallsModel(r)) return [];
251
+ const keyTargets = [...(agentCallsModel ? ['LLM_API_KEY'] : []), 'OPENAI_API_KEY', ...((agentId === 'tau3-llm-agent' || benchmarkCallsModel(r)) && providerOf(r, r.model) === 'openrouter' ? ['OPENROUTER_API_KEY'] : [])];
252
+ const urlTargets = [...(agentCallsModel ? ['LLM_BASE_URL'] : []), 'OPENAI_BASE_URL'];
253
+ return [{source: r.keyEnv, targets: keyTargets}, ...(r.urlEnv ? [{source: r.urlEnv, targets: urlTargets}] : [])];
254
+ }
255
+ function benchmarks() {
256
+ return schema.benchmarks.filter(b => b.harness.includes('harbor'));
257
+ }
258
+ // Why a benchmark cannot be launched on Harbor, or '' when it can.
259
+ function routeBlocker(key) {
260
+ if (key === 'custom') return '';
261
+ if (benchmarks().some(b => b.key === key)) return '';
262
+ const b = schema.benchmarks.find(b => b.key === key);
263
+ if (!b) return 'This benchmark has no supported launch adapter.';
264
+ return b.title + ' has no Harbor route; the harness it runs on is not available here.';
265
+ }
266
+ function dataset(r) { return schema.datasets.find(d => d.benchmark === r.benchmark) || null; }
267
+ // How many tasks an unlimited recipe would run: the catalogue's count for the benchmark's dataset, 0 when it has none.
268
+ const taskCount = r => Number(dataset(r)?.tasks) || 0;
269
+ const datasetLabel = r => ['local','generate'].includes(r.source) ? r.path : r.dataset;
270
+ // Where a reference agent's adapter checkout is: the recipe's own path for
271
+ // it, else the vocabulary's default. `~` is the run target's home.
272
+ const adapterPath = (r, id) => (r.adapterPaths && r.adapterPaths[id]) || agentById(id, r)?.pythonpath || '';
273
+ function sources(r) {
274
+ const d = dataset(r), options = [];
275
+ if (d?.hub) options.push(['hub', d.hub.includes('(private)') ? 'Private hub' : 'Harbor hub']);
276
+ else if (d?.registry) options.push(['registry', 'Versioned registry']);
277
+ else options.push(['registry', 'Registry or hub']);
278
+ options.push(['local', 'Local folder']);
279
+ if (d?.generate) options.push(['generate', 'Generate tasks']);
280
+ return options;
281
+ }
282
+ function setBenchmark(r, key) {
283
+ r.model_choice = 'any';
284
+ r.benchmark = key;
285
+ const d = dataset(r);
286
+ r.source = sources(r)[0][0];
287
+ r.dataset = d?.registry || '';
288
+ r.path = ''; // a Harbor folder is typed, never pre-filled: the usual one is the placeholder
289
+ return r;
290
+ }
291
+ function create() {
292
+ const r = {version: 1, harness: 'harbor', name: 'harbor-experiment', title: 'Harbor experiment', benchmark: '', source: '', dataset: '', path: '',
293
+ generatorRoot: '', agents: [], harbor: {kind: 'release', version: '', extras: []},
294
+ customAgents: [], model_choice: 'any', provider: firstAvailable(PROVIDERS, providerOf({}, schema.defaultModel || '')), model: '', userModel: '', evalModel: '', keyEnv: (schema.credentialVars && schema.credentialVars.key) || 'API_TOKEN', urlEnv: schema.credentialVars ? (schema.credentialVars.url || '') : 'API_URL',
295
+ tasks: '', concurrency: 1, retries: 3,
296
+ timeoutMultiplier: 1, agentMinutes: '', attempts: '', setupTimeoutMultiplier: '', verifierTimeoutMultiplier: '', buildTimeoutMultiplier: '', harborDev: '', include: '', output: 'jobs',
297
+ where: 'local', adapterPaths: {}, params: defaults(), custom: []};
298
+ adoptModel(r, r.provider, schema.defaultModel || '');
299
+ const b = benchmarks().find(b => b.key === 'tau3-bench') || benchmarks()[0];
300
+ return setBenchmark(r, b?.key || 'custom');
301
+ }
302
+ // Harbor's --agent-timeout-multiplier scales every per-phase timeout the
303
+ // selected tasks declare. The catalogue records those declarations where a
304
+ // real run has shown them (phase timeout_sec on the feed ÷ the run's
305
+ // multiplier); a benchmark without a recording gets no numbers, only the rule.
306
+ const PHASES = [['env_setup', 'Environment setup'], ['agent_setup', 'Agent setup'], ['agent_run', 'Agent'], ['verify', 'Verifier']];
307
+ function declaredTimeouts(r) {
308
+ const t = schema.benchmarks.find(b => b.key === r.benchmark)?.timeouts;
309
+ return t && PHASES.every(([k]) => Number.isFinite(Number(t[k])) && Number(t[k]) > 0) ? t : null;
310
+ }
311
+ // Each phase has its own Harbor multiplier; the recipe keeps the multipliers
312
+ // (that is what Harbor takes), and the page shows minutes where the
313
+ // benchmark's declared timeouts are recorded.
314
+ const PHASE_FIELD = {env_setup: 'buildTimeoutMultiplier', agent_setup: 'setupTimeoutMultiplier', agent_run: 'timeoutMultiplier', verify: 'verifierTimeoutMultiplier'};
315
+ const phaseMultiplier = (r, key) => { const v = r[PHASE_FIELD[key]]; return v === '' || v == null ? 1 : Number(v); };
316
+ function effectiveTimeouts(r) {
317
+ const t = declaredTimeouts(r);
318
+ if (!t) return null;
319
+ return {phases: PHASES.map(([key, label]) => {
320
+ const m = phaseMultiplier(r, key), ok = Number.isFinite(m) && m > 0;
321
+ return {key, label, field: PHASE_FIELD[key], declared: Number(t[key]), multiplier: m, effective: ok ? Math.round(Number(t[key]) * m) : null,
322
+ minutes: ok ? Math.round(Number(t[key]) * m / 60 * 100) / 100 : null};
323
+ })};
324
+ }
325
+ // The absolute form of the same knob: minutes a phase may take per trial,
326
+ // turned into that phase's multiplier against its declared timeout. The
327
+ // agent phase keeps `agentMinutes` as typed; the others are derived.
328
+ function setPhaseMinutes(r, key, minutes) {
329
+ const t = declaredTimeouts(r), v = minutes === '' || minutes == null ? '' : Number(minutes);
330
+ if (key === 'agent_run') r.agentMinutes = v;
331
+ if (!t) return r;
332
+ if (v === '') { if (key !== 'agent_run') r[PHASE_FIELD[key]] = ''; return r; }
333
+ if (Number(v) > 0) r[PHASE_FIELD[key]] = Math.round(Number(v) * 60 / Number(t[key]) * 1000) / 1000;
334
+ return r;
335
+ }
336
+ // Which Harbor runs a recipe.
337
+ //
338
+ // A recipe pins its Harbor (`r.harbor`, chosen on the Runner step): a
339
+ // release of the official package on PyPI, `{kind: 'release', version,
340
+ // extras}`, or a repository - a private fork included - at a commit,
341
+ // `{kind: 'repo', repo, revisionKind, revision, commit, extras}`, the
342
+ // commit being the pin a branch or a tag resolved to. Run builds a repository
343
+ // on this machine and runs it here or on a host (nullius/harbor/source.py).
344
+ //
345
+ const HARBOR_VERSION = /^[0-9][0-9A-Za-z.+!-]{0,63}$/, HARBOR_EXTRA = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/, COMMIT = /^[0-9a-f]{40}$/;
346
+ // A repository as git is given it: owner/name is GitHub over https; a URL or user@host:path as typed (nullius/gitrefs.py url_of).
347
+ function harborRepoUrl(text) {
348
+ const t = String(text || '').trim(), m = t.match(/^([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/);
349
+ if (!t || t.startsWith('-') || /[\s\x00-\x1f]/.test(t)) return '';
350
+ if (m && !t.startsWith('.')) return 'https://github.com/' + m[1] + '/' + m[2].replace(/\.git$/, '') + '.git';
351
+ return /^(?:https?|ssh|git|file):\/\/\S+$/.test(t) || /^[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+:\S+$/.test(t) ? t : '';
352
+ }
353
+ const repoLabel = url => (String(url).match(/^(?:https:\/\/github\.com\/|ssh:\/\/git@github\.com\/|git@github\.com:)([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+?)(?:\.git)?\/?$/) || [])[1] || url;
354
+ // The recipe's pin when it names one completely, else null.
355
+ function harborPin(r) {
356
+ const h = r.harbor || {}, extras = [...new Set((Array.isArray(h.extras) ? h.extras : []).filter(x => HARBOR_EXTRA.test(x)))].sort();
357
+ if (h.kind === 'release') return HARBOR_VERSION.test(h.version || '') ? {kind: 'release', version: h.version, extras} : null;
358
+ if (h.kind !== 'repo' || !harborRepoUrl(h.repo) || !COMMIT.test(h.commit || '')) return null;
359
+ const revisionKind = ['branch', 'tag', 'commit'].includes(h.revisionKind) ? h.revisionKind : 'commit';
360
+ return {kind: 'repo', repo: String(h.repo).trim(), url: harborRepoUrl(h.repo), revisionKind, revision: h.revision || h.commit, commit: h.commit, extras};
361
+ }
362
+ // What is wrong with a pin half-chosen; '' for a complete pin and for none chosen at all (a release not picked yet).
363
+ function harborProblem(r) {
364
+ const h = r.harbor || {};
365
+ if (h.kind === 'release') return !h.version || HARBOR_VERSION.test(h.version) ? '' : h.version + ' is not a Harbor release version.';
366
+ if (h.kind !== 'repo') return '';
367
+ if (!harborRepoUrl(h.repo)) return 'Enter the repository that holds the Harbor: owner/name on GitHub, or a git URL.';
368
+ return COMMIT.test(h.commit || '') ? '' : 'Choose a branch, a tag or a full commit of ' + repoLabel(harborRepoUrl(h.repo)) + ': the pin is a commit.';
369
+ }
370
+ // The directory the server keeps a pin in (source.key_of): what its scan is asked by.
371
+ const harborKey = pin => pin.kind === 'release' ? 'release-' + pin.version : 'git-' + pin.commit;
372
+ // What `uvx --from` installs: the release from PyPI, or the repository at the pinned commit (a URL uv clones; scp-like becomes ssh://).
373
+ function harborSpec(pin) {
374
+ const ex = pin.extras.length ? '[' + pin.extras.join(',') + ']' : '';
375
+ if (pin.kind === 'release') return 'harbor' + ex + '==' + pin.version;
376
+ const scp = pin.url.match(/^([A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+):(.*)$/);
377
+ return 'harbor' + ex + ' @ git+' + (scp ? 'ssh://' + scp[1] + '/' + scp[2].replace(/^\//, '') : pin.url) + '@' + pin.commit;
378
+ }
379
+ function harborLabel(pin) {
380
+ const ex = pin.extras.length ? ' · extras ' + pin.extras.join(', ') : '';
381
+ if (pin.kind === 'release') return 'harbor ' + pin.version + ' · PyPI' + ex;
382
+ const at = pin.commit.slice(0, 12);
383
+ return 'harbor from ' + repoLabel(pin.url) + ' · ' + (pin.revisionKind === 'commit' ? 'commit ' + at : pin.revisionKind + ' ' + pin.revision + ' at ' + at) + ex;
384
+ }
385
+ function harborExecutable(r) {
386
+ if (r.harborDev) return {kind: 'development', executable: r.harborDev, command: path(r.harborDev), label: 'development · ' + r.harborDev + ' — not a release'};
387
+ const pin = harborPin(r);
388
+ if (pin) return {kind: pin.kind, executable: 'harbor', command: 'uvx --from ' + q(harborSpec(pin)) + ' harbor', label: harborLabel(pin), pin};
389
+ return {kind: 'path', executable: 'harbor', command: 'harbor', label: 'harbor from PATH'};
390
+ }
391
+ function scopes() {
392
+ return [['agent', 'Agent · --ak'], ['harness', 'Harness option'], ['environment', 'Environment · --ek'], ['verifier', 'Verifier kwarg']];
393
+ }
394
+ // `harbor run` options the named fields already write; a custom one of these would be a duplicate flag.
395
+ const RESERVED = ['agent','a','model','m','dataset','d','path','p','n-tasks','l','n-concurrent','n','max-retries','r','env','e','job-name','jobs-dir','o','agent-timeout-multiplier','include-task-name','i','ak','agent-kwarg','ek','environment-kwarg','verifier-kwarg','agent-env','ae'];
396
+ function validate(r) {
397
+ const errors = [], notes = [];
398
+ const add = (step, message, group) => errors.push({step, message, ...(group ? {group} : {})});
399
+ if (isStandard(r)) return {errors, notes}; // compiled and checked by the server, from its definition files
400
+ if (r.harness !== 'harbor') { add(STEP.runner, RUNNER_GONE); return {errors, notes}; }
401
+ const blocker = routeBlocker(r.benchmark);
402
+ if (blocker) add(STEP.benchmark, blocker);
403
+ if (!sources(r).some(([s]) => s === r.source)) add(STEP.dataset, 'Choose a dataset source supported by this benchmark.');
404
+ const local = r.source === 'local';
405
+ if (local && !r.path.trim()) add(STEP.dataset, 'Enter the local task folder.');
406
+ if (!local && r.source !== 'generate' && !r.dataset.trim()) add(STEP.dataset, 'Enter a dataset reference.');
407
+ if (r.dataset.includes('<') || r.dataset.includes('>')) add(STEP.dataset, 'Replace the placeholder with a real dataset or task.');
408
+ if (r.source === 'generate') {
409
+ if (!r.generatorRoot.trim() || !r.path.trim()) add(STEP.dataset, 'Set the generator checkout and the folder the tasks are written to.');
410
+ }
411
+ if (!Array.isArray(r.agents) || !r.agents.length) add(STEP.agent, 'Choose at least one agent.');
412
+ const hp = harborProblem(r);
413
+ if (hp) add(STEP.runner, hp);
414
+ for (const id of r.agents) if (!agentById(id, r)) add(STEP.agent, customIndex(id) >= 0 ? 'Custom agent ' + id + ' has no import path.' : id + ' is not an agent this builder offers' + (scannedAgents ? '' : ' until the recipe\'s Harbor is scanned (Runner step)') + '; or name it as a custom agent (module:Class) instead.');
415
+ for (const id of r.agents) { const a = agentById(id, r); if (a?.benchmarks && !a.benchmarks.includes(r.benchmark)) add(STEP.agent, a.label + ' runs only on ' + a.benchmarks.join(', ') + '.'); }
416
+ r.customAgents.forEach((p, i) => { if (r.agents.includes('custom-' + i) && !/^[\w.]+:[\w.]+$/.test(p)) add(STEP.agent, 'Custom agent ' + (i + 1) + ' must use module:Class syntax.'); });
417
+ const callsModel = needsCredentials(r);
418
+ // τ³ with control agents only: the customer model is the one that matters, and the form asks for that one.
419
+ const named = r.model.trim() || (benchmarkCallsModel(r) && r.agents.every(id => agentById(id, r)?.control) ? (r.userModel || '').trim() : '');
420
+ if (callsModel && r.model_choice === 'paper') {
421
+ const allowed = Object.values(paperModels(r).sent?.[r.provider] || {});
422
+ if (!allowed.includes(r.model)) add(STEP.model, 'Choose one of the paper models on this provider.');
423
+ }
424
+ if (callsModel && !named) add(STEP.model, benchmarkCallsModel(r) ? 'Enter a model for the agent and/or simulated user and evaluator.' : 'Enter a model.');
425
+ if (callsModel && !PROVIDERS.includes(r.provider)) add(STEP.model, 'Choose a provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.');
426
+ for (const [key,label] of [['keyEnv','API key source variable'],['urlEnv','Endpoint source variable']]) {
427
+ if (callsModel && (key !== 'urlEnv' || r[key]) && !/^[A-Z_][A-Z0-9_]*$/.test(r[key])) add(STEP.model, label + ' must be an environment variable name.');
428
+ }
429
+ if (callsModel && benchmarkCallsModel(r) && providerOf(r, r.model || r.userModel) !== 'openai' && !r.urlEnv) add(STEP.model, 'A non-OpenAI model needs an endpoint source variable.');
430
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(r.name)) add(STEP.limits, 'Recipe name may contain letters, numbers, dots, underscores and hyphens.');
431
+ if (!r.output.trim()) add(STEP.limits, 'Enter the output directory.');
432
+ for (const key of ['tasks','concurrency','retries']) {
433
+ const min = key === 'retries' ? 0 : 1;
434
+ if (key === 'tasks' && r.tasks === '') continue; // empty: every task, no limit written
435
+ if (!Number.isInteger(Number(r[key])) || Number(r[key]) < min) add(STEP.limits, key + ' must be an integer of at least ' + min + ', or empty for every task.');
436
+ }
437
+ if (!Number.isFinite(Number(r.timeoutMultiplier)) || !(Number(r.timeoutMultiplier) > 0)) add(STEP.agent, 'Timeout multiplier must be positive.', 'Common');
438
+ // Common trial settings (Harbor's own per-trial flags): empty means the flag is not written.
439
+ for (const [key, label] of COMMON) if (r[key] !== '' && (!Number.isFinite(Number(r[key])) || !(Number(r[key]) > 0))) add(STEP.agent, label + ' must be a positive number, or empty.', 'Common');
440
+ if (r.attempts !== '' && (!Number.isInteger(Number(r.attempts)) || Number(r.attempts) < 1)) add(STEP.agent, 'Repeats per task must be a whole number of at least 1, or empty.', 'Common');
441
+ if (r.agentMinutes !== '' && r.agentMinutes != null) {
442
+ if (!Number.isFinite(Number(r.agentMinutes)) || !(Number(r.agentMinutes) > 0)) add(STEP.limits, 'Max agent time per trial must be a positive number of minutes.');
443
+ else if (!declaredTimeouts(r)) add(STEP.limits, 'No run has recorded this benchmark\'s declared timeouts, so minutes cannot be turned into a multiplier here; set the multiplier directly.');
444
+ }
445
+ if (r.harborDev && !/^(\/|~\/|\.{1,2}\/)/.test(r.harborDev)) add(STEP.limits, 'A development Harbor executable must be a path (absolute, ~/ or ./), not a program name.');
446
+ // The shared agent parameters (the MCP call timeout): each a valid value of its type, or left to its default.
447
+ for (const p of parameters()) {
448
+ const v = r.params[p.key];
449
+ if (v == null || v === '') continue;
450
+ if ((p.type === 'integer' || p.type === 'number') && (!Number.isFinite(Number(v)) || (p.type === 'integer' && !Number.isInteger(Number(v))) || (p.min != null && Number(v) < p.min) || (p.max != null && Number(v) > p.max)))
451
+ add(STEP.agent, 'Enter a valid ' + p.label.toLowerCase() + '.', p.group);
452
+ }
453
+ const seen = new Set(), parameterNames = new Set(parameters().map(p => p.key));
454
+ for (const p of r.custom) {
455
+ const customError = message => add(STEP.agent, message, 'Extra parameters');
456
+ if (!scopes().some(([s]) => s === p.scope)) { customError('Choose a valid custom-parameter scope.'); continue; }
457
+ if (!/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(p.key)) customError('Parameter names must contain letters, numbers, underscores or hyphens.');
458
+ if (secret(p.key)) customError('Use environment references for credentials; do not put them in custom parameters.');
459
+ const id = p.scope + ':' + (p.agent || '') + ':' + p.key;
460
+ if (seen.has(id)) customError('Duplicate parameter: ' + p.key);
461
+ seen.add(id);
462
+ if (p.scope === 'agent' && (parameterNames.has(p.key) || ['model','api_key','base_url','agent_timeout_sec','mcp_servers'].includes(p.key))) customError('Use the named field or task definition for ' + p.key + '.');
463
+ if (p.scope === 'harness' && RESERVED.includes(p.key)) customError('Use the named field for ' + p.key + '.');
464
+ if (!['string','number','boolean','JSON','flag'].includes(p.type)) customError('Choose a parameter type.');
465
+ if (p.type === 'number' && (String(p.value).trim() === '' || !Number.isFinite(Number(p.value)))) customError(p.key + ' requires a number.');
466
+ if (p.type === 'boolean' && !['true','false'].includes(String(p.value))) customError(p.key + ' requires true or false.');
467
+ if (p.type === 'flag' && p.scope !== 'harness') customError('Flag without value is only supported for harness CLI options.');
468
+ if (p.type === 'JSON') { try { JSON.parse(p.value); } catch { customError(p.key + ' requires valid JSON.'); } }
469
+ notes.push('Custom ' + p.scope + ' parameter ' + (p.key || '(unnamed)') + ': check support in your installed version.');
470
+ }
471
+ if (harborExecutable(r).kind === 'development') notes.push('Development Harbor executable: ' + r.harborDev + ' is a checkout, not an immutable release; runs from it are development evidence.');
472
+ else if (harborExecutable(r).kind === 'path' && !hp) notes.push('Choose Harbor on the Runner step.');
473
+ // an agent that calls its own vendor (Cursor's CLI agent) needs that vendor's key: Settings, Custom keys
474
+ const held = (schema.credentials || {}).agent_keys || [];
475
+ for (const id of r.agents) {
476
+ const want = (schema.agentNeeds || {})[agentById(id, r)?.arg || id];
477
+ if (want && !want.some(k => held.includes(k))) notes.push((agentById(id, r)?.label || id) + ' needs ' + want.join(' or ') + ' of its own: add it in Settings, Custom keys, or its trials fail at the start.');
478
+ }
479
+ return {errors, notes: [...new Set(notes)]};
480
+ }
481
+ const customValue = p => p.type === 'number' ? Number(p.value) : p.type === 'boolean' ? String(p.value) === 'true' : p.type === 'JSON' ? JSON.parse(p.value) : p.value;
482
+ // Harbor's per-trial settings every selected agent shares; `timeoutMultiplier`
483
+ // (the agent phase), retries, concurrency and the task limit are on Limits.
484
+ const COMMON = [['setupTimeoutMultiplier', 'Agent setup timeout multiplier'], ['verifierTimeoutMultiplier', 'Verifier timeout multiplier'], ['buildTimeoutMultiplier', 'Environment build timeout multiplier']];
485
+ const COMMON_FLAGS = {setupTimeoutMultiplier: '--agent-setup-timeout-multiplier', verifierTimeoutMultiplier: '--verifier-timeout-multiplier', buildTimeoutMultiplier: '--environment-build-timeout-multiplier'};
486
+ const commonFlags = r => [...(r.attempts !== '' ? ['--n-attempts ' + Number(r.attempts)] : []), ...COMMON.filter(([k]) => r[k] !== '').map(([k]) => COMMON_FLAGS[k] + ' ' + Number(r[k]))];
487
+ const agentCustom = (r, id) => r.custom.filter(p => p.scope === 'agent' && (!p.agent || p.agent === id));
488
+ const cliExtras = r => r.custom.filter(p => p.scope === 'harness').map(p => '--' + p.key + (p.type === 'flag' ? '' : '=' + q(textValue(customValue(p)))));
489
+ function compile(r, {runner = false} = {}) {
490
+ const checks = validate(r);
491
+ if (checks.errors.length) return {...checks, artifacts: []};
492
+ const artifacts = [], add = (id, label, filename, language, cwd, text) => artifacts.push({id,label,filename,language,cwd,text});
493
+ let run = '';
494
+ {
495
+ let taskPath = r.path;
496
+ for (const id of r.agents) {
497
+ const a = agentById(id, r), model = !a.control;
498
+ const prefix = credentialBindings(r, id).flatMap(b => b.targets.map(target => target+'='+env(b.source)));
499
+ // A custom module:Class agent is imported from the directory the script runs in.
500
+ if (a.custom) prefix.push('PYTHONPATH=$PWD');
501
+ const adapter = adapterPath(r, id);
502
+ if (adapter) prefix.push('PYTHONPATH='+path(adapter));
503
+ // τ³: the simulated user and the evaluator are model calls too; they follow the agent's model and endpoint.
504
+ if (benchmarkCallsModel(r)) prefix.push('TAU2_USER_MODEL='+q(tau2Model(r.userModel || r.model, r)),'TAU2_NL_ASSERTIONS_MODEL='+q(tau2Model(r.evalModel || r.userModel || r.model, r)),"TAU2_USER_REASONING_EFFORT=''");
505
+ const exe = harborExecutable(r);
506
+ // A runner (Run from the Review step, here or on a host) sets the
507
+ // credentials and the three paths as environment; the script names
508
+ // them and nothing else about where it runs.
509
+ const parts = [...prefix, runner ? '"$NULLIUS_HARBOR" run' : exe.command+' run',
510
+ (['local','generate'].includes(r.source)?'-p '+(runner && r.source === 'local' ? '"$NULLIUS_TASKS"' : path(taskPath)):'-d '+q(r.dataset)),
511
+ '--env docker','--agent '+q(a.arg),'--yes']; // the tasks' environment variables were reviewed here; the script runs unattended
512
+ if (model) parts.push('--model '+q(id === 'tau3-llm-agent' ? litellmModel(r) : agentModel(r)));
513
+ parts.push(...agentCustom(r, id).map(p=>'--ak '+q(p.key+'='+kwargValue(customValue(p)))));
514
+ // The reference agent's tool timeout follows the shared MCP timeout, so both agents wait the same.
515
+ if (id === 'tau3-llm-agent' && r.params.mcp_timeout_s !== '' && r.params.mcp_timeout_s != null) parts.push('--ak '+q('tool_timeout_sec='+Number(r.params.mcp_timeout_s)));
516
+ parts.push(...(r.tasks === '' ? [] : ['--n-tasks '+r.tasks]),'--n-concurrent '+r.concurrency,'--max-retries '+r.retries,
517
+ '--agent-timeout-multiplier '+r.timeoutMultiplier,'--job-name '+q(r.name+(r.agents.length>1?'-'+id:'')), '--jobs-dir '+(runner ? '"$NULLIUS_JOBS"' : path(r.output)));
518
+ if (r.include) parts.push('--include-task-name '+q(r.include));
519
+ parts.push(...commonFlags(r));
520
+ for (const p of r.custom.filter(p=>['environment','verifier'].includes(p.scope))) parts.push((p.scope==='environment'?'--ek ':'--verifier-kwarg ')+q(p.key+'='+kwargValue(customValue(p))));
521
+ parts.push(...cliExtras(r));
522
+ run = join(parts);
523
+ // A runner part runs from the run directory on whatever machine was chosen: the
524
+ // adapter directory is checked there before Harbor starts, and a missing one is
525
+ // said by name instead of an import error from inside Harbor.
526
+ if (runner && adapter) run = 'nullius_adapter='+path(adapter)+'\n[ -d "$nullius_adapter" ] || { printf \'%s\\n\' "'+a.label.replace(/"/g, '')+' needs its adapter directory on this machine: $nullius_adapter is missing" >&2; exit 2; }\n'+run;
527
+ add('run-'+id,a.label+' run',r.name+'-'+id+'.sh','shell','Working directory',run);
528
+ }
529
+ }
530
+ return {...checks, artifacts};
531
+ }
532
+ // What Run on the Review step posts: the run commands with the paths as
533
+ // environment names (the harbor on the target's PATH among them) and the
534
+ // task folder they read. Says plainly what cannot run from here yet.
535
+ function runnable(r) {
536
+ const errors = [];
537
+ if (r.source === 'generate') errors.push('Generated tasks: run the generator first, then point the recipe at its output folder.');
538
+ if (r.agents.some(id => agentById(id, r)?.custom)) errors.push('A custom agent is imported from its configured module.');
539
+ if (harborExecutable(r).kind === 'development') errors.push('A development Harbor cannot run here. Choose a release or repository on the Runner step.');
540
+ else if (!harborPin(r)) errors.push('Choose the Harbor that runs this recipe on the Runner step: a release, or a repository at a branch, tag or commit.');
541
+ if (providerBlocker(r)) errors.push(providerBlocker(r));
542
+ if (errors.length) return {errors, script: '', parts: [], tasks: '', model: '', jobs: []};
543
+ const compiled = compile(r, {runner: true});
544
+ if (compiled.errors.length) return {errors: compiled.errors.map(e => e.message), script: '', parts: [], tasks: '', model: '', jobs: []};
545
+ // Controls first - nop, then oracle - so the dataset and the environment are proven before an agent under
546
+ // evaluation spends anything; the runner stops between agents on a failure and asks.
547
+ const rank = id => id === 'nop' ? 0 : id === 'oracle' ? 1 : agentById(id, r)?.control ? 2 : 3;
548
+ const order = [...r.agents].sort((a, b) => rank(a) - rank(b));
549
+ const runs = order.map(id => compiled.artifacts.find(a => a.id === 'run-' + id)).filter(Boolean);
550
+ const script = ['#!/usr/bin/env bash', '# ' + r.name + ' — generated by Nullius; NULLIUS_HARBOR, NULLIUS_TASKS, NULLIUS_JOBS and the credentials arrive as environment.',
551
+ 'set -uo pipefail', 'status=0', ...runs.map(a => '# ' + a.label + '\n' + a.text + ' || status=1'), 'exit "$status"'].join('\n\n') + '\n';
552
+ const parts = runs.map(a => ({id: a.id.slice(4), label: a.label, script: '#!/usr/bin/env bash\nset -uo pipefail\n' + a.text + '\n'}));
553
+ const jobs = order.map(id => ({name: r.name + (r.agents.length > 1 ? '-' + id : ''), agent: agentById(id, r)?.arg || id, dataset: r.source === 'local' ? r.path.replace(/\/$/, '').split('/').pop() : r.dataset}));
554
+ // No task limit means the whole dataset - 375 tasks and five hours on τ³ - so Run asks first, with the count when the catalogue knows it.
555
+ // A task-name pattern narrows it to the tasks that match, a number only Harbor knows, so the question names the pattern instead.
556
+ const confirm = r.tasks !== '' ? '' : taskFilter(r)
557
+ ? 'Run the tasks of ' + datasetLabel(r) + ' ' + taskFilter(r) + ' (no limit)?'
558
+ : 'Run every task of ' + datasetLabel(r) + (taskCount(r) ? ' (' + taskCount(r) + ' tasks, no limit)?' : ' (no limit)?');
559
+ return {errors: [], script, parts, tasks: r.source === 'local' ? r.path : '', model: needsCredentials(r) ? agentModel(r, r.model || r.userModel) : '', jobs, confirm};
560
+ }
561
+ // The task-name patterns a run is narrowed by: the Include field, and an exclude written as a custom harness option.
562
+ // Harbor matches them as globs; how many tasks match is known only once it resolves the dataset.
563
+ function taskFilter(r) {
564
+ const exclude = r.custom.filter(p => p.scope === 'harness' && ['exclude-task-name', 'x'].includes(p.key) && p.value).map(p => p.value);
565
+ return [r.include ? 'matching ' + r.include : '', exclude.length ? 'not matching ' + exclude.join(', ') : ''].filter(Boolean).join(' and ');
566
+ }
567
+ function scopeText(r) {
568
+ const filter = taskFilter(r), count = taskCount(r);
569
+ if (r.tasks !== '') return filter ? 'up to ' + r.tasks + ' tasks ' + filter : r.tasks + ' tasks';
570
+ if (filter) return 'tasks ' + filter + (count ? ' (of ' + count + ' in this dataset)' : '');
571
+ return 'every task' + (count ? ' (' + count + ' in this dataset)' : '');
572
+ }
573
+ // What the Review step shows: one row per thing a person is asked to
574
+ // confirm - the harness and benchmark, the dataset or source, the agents,
575
+ // the model, the task scope and the execution location (and, on Harbor,
576
+ // which runtime and executable) - each with the step that edits it. The
577
+ // page renders these rows as they are; the route suite reads them here.
578
+ function review(r) {
579
+ const b = benchmarks(r.harness).find(b => b.key === r.benchmark) || {title: 'Custom benchmark', category: 'Custom tasks'};
580
+ const label = id => agentById(id, r)?.label || id;
581
+ const exe = harborExecutable(r);
582
+ return [[STEP.benchmark, 'Benchmark', 'Harbor / ' + b.title],
583
+ [STEP.runner, 'Runner', 'Harbor'],
584
+ [exe.kind === 'development' ? STEP.limits : STEP.runner, 'Harbor', exe.kind === 'path' ? 'not chosen' : exe.label],
585
+ [STEP.dataset, 'Dataset', r.source + ' · ' + (r.source === 'generate' ? r.path : r.source === 'local' ? r.path : r.dataset)],
586
+ [STEP.agent, 'Agents', r.agents.map(label).join(', ')],
587
+ [STEP.model, 'Model', needsCredentials(r) ? (r.benchmark === 'tau3-bench' && r.userModel && r.userModel !== r.model ? (r.model ? agentModel(r) + ' · customer ' : 'customer ') + bareModel(r.userModel) : agentModel(r, r.model || r.userModel)) : 'No model required'],
588
+ [STEP.limits, 'Scope', scopeText(r) + ' · ' + r.concurrency + ' concurrent'],
589
+ [STEP.where, 'Where', whereLabel(r)]];
590
+ }
591
+ const machineLabel = w => w === 'local' ? 'This machine' : (schema.hosts || []).find(x => x.entry === w)?.label || w;
592
+ const whereLabel = r => whereList(r).map(machineLabel).join(' + ');
593
+ function fromJSON(input) {
594
+ const raw = typeof input === 'string' ? JSON.parse(input) : input;
595
+ if (raw && raw.version === 1 && isStandard(raw)) return standardFromJSON(raw);
596
+ if (raw && raw.version === 1 && typeof raw.harness === 'string' && raw.harness && raw.harness !== 'harbor') throw Error(RUNNER_GONE);
597
+ if (!raw || raw.version !== 1 || raw.harness !== 'harbor') throw Error('Unsupported recipe format.');
598
+ const r=create();
599
+ for (const key of Object.keys(r)) {
600
+ if (['params','custom','agents'].includes(key)) continue;
601
+ if (typeof r[key] === 'string' && typeof raw[key] === 'string') r[key]=raw[key];
602
+ if (typeof r[key] === 'number' && typeof raw[key] === 'number') r[key]=raw[key];
603
+ if (['agentMinutes','attempts','setupTimeoutMultiplier','verifierTimeoutMultiplier','buildTimeoutMultiplier'].includes(key) && typeof raw[key] === 'number') r[key]=raw[key]; // '' by default, a number once set
604
+ if (key === 'tasks' && typeof raw[key] === 'number') r[key]=raw[key]; // '' by default (every task), a number once set
605
+ }
606
+ if (Array.isArray(raw.customAgents)) r.customAgents=raw.customAgents.slice(0,20).map(p=>String(p||''));
607
+ if (raw.adapterPaths && typeof raw.adapterPaths === 'object') for (const [k,v] of Object.entries(raw.adapterPaths)) if (typeof v === 'string' && v) r.adapterPaths[k]=v;
608
+ if (Array.isArray(raw.agents)) r.agents=raw.agents.filter(id=>typeof id==='string');
609
+ // Normalize the single customAgent field into the customAgents list.
610
+ if (r.agents.includes('custom')) { r.customAgents.push(typeof raw.customAgent==='string'?raw.customAgent:''); r.agents=r.agents.map(id=>id==='custom'?'custom-'+(r.customAgents.length-1):id); }
611
+ if (raw.params && typeof raw.params === 'object') for(const key of Object.keys(r.params)) if(own(raw.params,key)) r.params[key]=raw.params[key];
612
+ if (Array.isArray(raw.custom)) r.custom=raw.custom.slice(0,100).map(p=>({scope:String(p.scope||''),key:String(p.key||''),type:String(p.type||'string'),value:String(p.value??''),...(p.agent?{agent:String(p.agent)}:{})}));
613
+ // Infer an omitted provider from the model prefix, then keep the bare model name.
614
+ r.provider = PROVIDERS.includes(raw.provider) ? raw.provider : providerOf({}, raw.model || raw.userModel || '');
615
+ keepEndpoint(r, raw);
616
+ if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id';
617
+ else { const typed = {model: r.model, userModel: r.userModel, evalModel: r.evalModel};
618
+ for (const key of ['model', 'userModel', 'evalModel']) r[key] = bareModel(typed[key]);
619
+ if (typed.model) adoptModel(r, r.provider, typed.model); }
620
+ if (typeof raw.title !== 'string' || !raw.title.trim()) r.title = typeof raw.name === 'string' ? raw.name : r.name;
621
+ // Read pin strings as written; validate checks them. An omitted pin selects no release.
622
+ const h = raw.harbor && typeof raw.harbor === 'object' ? raw.harbor : {}, str = v => typeof v === 'string' ? v : '';
623
+ const extras = Array.isArray(h.extras) ? h.extras.filter(x => typeof x === 'string').slice(0, 40) : [];
624
+ if (h.kind === 'repo') r.harbor = {kind: 'repo', repo: str(h.repo), revisionKind: ['branch', 'tag', 'commit'].includes(h.revisionKind) ? h.revisionKind : 'commit', revision: str(h.revision), commit: str(h.commit), extras};
625
+ else if (h.kind === 'release') r.harbor = {kind: 'release', version: str(h.version), extras};
626
+ return r;
627
+ }
628
+ // "New run like this" for a run that kept no recipe: the recipe its job
629
+ // recorded - parse.launch_of over Harbor's config.json, Harbor's defaults
630
+ // filled in - with the Harbor its lock names. Whatever the record holds that a recipe
631
+ // cannot say comes back as a note naming it; nothing is dropped silently.
632
+ const KWARG_TYPE = v => typeof v === 'number' ? 'number' : typeof v === 'boolean' ? 'boolean' : typeof v === 'string' ? 'string' : 'JSON';
633
+ const NAMED_KWARGS = ['model', 'api_key', 'base_url', 'agent_timeout_sec', 'mcp_servers']; // validate's "use the named field" list
634
+ function fromJob(like) {
635
+ const L = (like && like.launch) || {}, r = create(), lost = [], notes = [];
636
+ const recorded = (L.agents || []).filter(a => a && a.name);
637
+ if (!recorded.length && like.agent) recorded.push({name: like.agent, model: '', kwargs: {}, env: {}, secret: []});
638
+ // Agents by Harbor's own name for them - the registry name or the import path - never the display label.
639
+ const ran = [];
640
+ r.agents = [];
641
+ for (const a of recorded) {
642
+ const known = agents().find(x => [x.arg, x.id].includes(a.name));
643
+ let id = known && known.id;
644
+ if (!id && a.name.includes(':')) { r.customAgents.push(a.name); id = 'custom-' + (r.customAgents.length - 1); }
645
+ if (!id) { lost.push('agent ' + a.name + ' (not offered here)'); continue; }
646
+ if (!r.agents.includes(id)) r.agents.push(id);
647
+ ran.push([id, a]);
648
+ }
649
+ const m = (recorded.find(a => a.model) || {}).model;
650
+ if (m) adoptModel(r, PROVIDER_PREFIX.test(m) ? '' : r.provider, m); // the recorded prefix names the provider it ran on
651
+ // The benchmark: the catalogue dataset the job read, else the one its agent exists for.
652
+ const ds = L.dataset || {}, ref = ds.path || (ds.name ? ds.name + (ds.version ? '@' + ds.version : '') : '');
653
+ const match = ref ? schema.datasets.find(d => [d.registry, d.local, d.key].includes(ref) || (ds.name && [d.registry, d.key].includes(ds.name))) : null;
654
+ const offered = key => benchmarks().some(b => b.key === key);
655
+ const declared = [...new Set(r.agents.flatMap(id => agentById(id, r)?.benchmarks || []))].filter(offered);
656
+ setBenchmark(r, match?.benchmark || (declared.length === 1 ? declared[0] : '') || 'custom');
657
+ if (ds.path) { r.source = 'local'; r.path = ds.path; }
658
+ else if (ref) { r.source = match?.hub ? 'hub' : 'registry'; r.dataset = ref; }
659
+ else if (L.tasks) lost.push('its ' + L.tasks + ' tasks named one by one');
660
+ const names = L.task_names || [], excluded = L.exclude_task_names || [];
661
+ if (names.length === 1) r.include = names[0];
662
+ else if (names.length > 1) lost.push('task names ' + names.join(', ') + ' (a recipe takes one include pattern)');
663
+ if (excluded.length === 1) r.custom.push({scope: 'harness', key: 'exclude-task-name', type: 'string', value: excluded[0]});
664
+ else if (excluded.length > 1) lost.push('excluded task names ' + excluded.join(', ') + ' (a recipe takes one)');
665
+ // Limits and timeouts, as Harbor ran them: a key the config left out held Harbor's default, and launch_of filled it in.
666
+ if (Object.keys(L).length) {
667
+ r.tasks = Number(L.n_tasks) > 0 ? Number(L.n_tasks) : '';
668
+ if (Number.isInteger(L.n_concurrent_trials)) r.concurrency = L.n_concurrent_trials;
669
+ if (Number.isInteger(L.max_retries)) r.retries = L.max_retries;
670
+ r.attempts = Number(L.n_attempts) > 1 ? Number(L.n_attempts) : '';
671
+ const all = Number(L.timeout_multiplier ?? 1);
672
+ r.timeoutMultiplier = L.agent_timeout_multiplier ?? all;
673
+ for (const [k, f] of [['agent_setup_timeout_multiplier', 'setupTimeoutMultiplier'], ['verifier_timeout_multiplier', 'verifierTimeoutMultiplier'], ['environment_build_timeout_multiplier', 'buildTimeoutMultiplier']]) if (L[k] != null) r[f] = L[k];
674
+ if (all !== 1) r.custom.push({scope: 'harness', key: 'timeout-multiplier', type: 'number', value: String(all)});
675
+ } else notes.push('This job recorded no config.json: only its agent is carried.');
676
+ // Each agent's parameters: the reference agent's tool timeout, every other one as an extra agent parameter.
677
+ const fields = new Set(parameters().map(p => p.key));
678
+ for (const [id, a] of ran) {
679
+ for (const k of a.secret || []) lost.push(k + ' (a credential; Settings hands the run its key)');
680
+ for (const [k, v] of Object.entries(a.kwargs || {})) {
681
+ if (id === 'tau3-llm-agent' && k === 'tool_timeout_sec') r.params.mcp_timeout_s = v;
682
+ else if (fields.has(k) || NAMED_KWARGS.includes(k) || !/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(k)) lost.push(k + '=' + textValue(v) + ' for ' + a.name);
683
+ else r.custom.push({scope: 'agent', key: k, type: KWARG_TYPE(v), value: typeof v === 'string' ? v : JSON.stringify(v), agent: id});
684
+ }
685
+ for (const [k, v] of Object.entries(a.env || {})) {
686
+ if (r.benchmark === 'tau3-bench' && k === 'TAU2_USER_MODEL') { if (bareModel(v) !== r.model) r.userModel = bareModel(v); }
687
+ else if (r.benchmark === 'tau3-bench' && k === 'TAU2_NL_ASSERTIONS_MODEL') { if (bareModel(v) !== bareModel(r.userModel || r.model)) r.evalModel = bareModel(v); }
688
+ else if (!(r.benchmark === 'tau3-bench' && k === 'TAU2_USER_REASONING_EFFORT')) lost.push('agent environment ' + k);
689
+ }
690
+ if (adapterPath(r, id)) notes.push('A job does not record ' + (agentById(id, r)?.label || id) + '’s adapter directory: ' + adapterPath(r, id) + ' is this builder’s.');
691
+ if (a.model && !agentById(id, r)?.control) {
692
+ const sent = id === 'tau3-llm-agent' ? litellmModel(r) : agentModel(r);
693
+ if (sent !== a.model) notes.push((agentById(id, r)?.label || id) + ' is sent the model as ' + sent + '; the run recorded ' + a.model + '.');
694
+ }
695
+ }
696
+ const env = like.env || L.environment;
697
+ if (env && env !== 'docker') lost.push('environment ' + env + ' (a recipe runs docker)');
698
+ const hv = like.harbor && like.harbor.version;
699
+ // A plain version is what PyPI publishes, so it is pinned as that release and said; a local version (+label) is a fork's build, chosen by its repository.
700
+ const released = hv && !like.harbor.editable && /^\d+(?:\.\d+)*$/.test(hv);
701
+ if (released) { r.harbor = {kind: 'release', version: hv, extras: []}; notes.push('Harbor ' + hv + ' ran this job: this recipe pins harbor ' + hv + ' from PyPI. If that was a fork\'s build, choose its repository on the Runner step.'); }
702
+ else if (hv) notes.push('Harbor ' + hv + ' ran this job, a build PyPI does not publish: choose its repository and commit on the Runner step.');
703
+ if (hv && like.harbor.editable) notes.push('That Harbor was an editable checkout, not a release.');
704
+ lost.push(...(L.unexpressed || []));
705
+ return {recipe: r, notes: ['Opened from job “' + like.job + '”, as it recorded itself. Run gives it a new name.',
706
+ ...(lost.length ? ['Not carried: ' + lost.join('; ') + '.'] : []), ...notes]};
707
+ }
708
+ function prefill(query) {
709
+ // ?standard=<benchmark>[&other=<agent or capability>]: the standard's recipe for that pair (the Agents and Capabilities pages link here).
710
+ if(query.standard)return createStandard(query.standard,query.other||'');
711
+ // nothing asked for: the Nullius runner, chosen first; a Harbor recipe comes from its own card
712
+ if(!query.agent&&!query.dataset&&!query.model&&!query.n_tasks&&standard().benchmarks.length)return createStandard();
713
+ const r=create(), known=schema.agents.find(a=>[a.id,a.arg,a.label].includes(query.agent))||(query.agent&&schema.agents.find(a=>String(a.arg||'').includes(':')&&String(a.arg).split(':').pop()===query.agent));
714
+ if(query.model)adoptModel(r,PROVIDER_PREFIX.test(query.model)?'':r.provider,query.model);
715
+ if(known)r.agents=[known.id];
716
+ if(query.n_tasks&&Number(query.n_tasks)>0)r.tasks=Number(query.n_tasks);
717
+ if(query.dataset){
718
+ const d=schema.datasets.find(d=>[d.registry,d.local,d.key].includes(query.dataset));
719
+ // a dataset Harbor does not serve (no hub, no registry), of a benchmark the standard runs, opens the standard's recipe
720
+ const b=d&&schema.benchmarks.find(x=>x.key===d.benchmark);
721
+ if(b&&!(d.hub||d.registry)&&standard().benchmarks.some(x=>x.key===b.key))return createStandard(b.key);
722
+ setBenchmark(r,d?.benchmark||'custom');
723
+ if(query.dataset.startsWith('/')||query.dataset.startsWith('.')||query.dataset.startsWith('~')){r.source='local';r.path=query.dataset;}
724
+ // a catalogue dataset named by its key runs under the name Harbor knows it by (its registry), not the key
725
+ else{r.source=d?.hub?'hub':'registry';r.dataset=d&&d.key===query.dataset&&d.registry?d.registry:query.dataset;}
726
+ }
727
+ return r;
728
+ }
729
+ // A link that already names the benchmark (Build a run on its page) opens past it: on the runner when the
730
+ // benchmark runs on both, so the choice is seen, else on its dataset. Anything else starts at the benchmark.
731
+ const openingStep = (query, r) => !(query && (query.standard || query.dataset)) ? STEP.benchmark
732
+ : r && runnersOf(benchKey(r)).length > 1 ? STEP.runner : STEP.dataset;
733
+ // The runners a benchmark has here, Nullius first: its own Harbor folder ('custom') is Harbor's alone.
734
+ const runnersOf = key => [...(standardBench({benchmark: key}) ? ['standard'] : []),
735
+ ...(key === 'custom' || benchmarks().some(b => b.key === key) ? ['harbor'] : [])];
736
+ // The runner a benchmark picked opens: the recipe's own when it has that benchmark, else Nullius before Harbor.
737
+ const runnerFor = (r, key) => { const has = runnersOf(key); return has.includes(r.harness) ? r.harness : has[0] || r.harness; };
738
+ // The benchmark a recipe names, by its key whichever runner it is for (a version of the standard's is its benchmark).
739
+ const benchKey = r => isStandard(r) ? (standardBench(r) || {}).key || r.benchmark : r.benchmark;
740
+ // The page keeps what is on screen as a draft in the browser, so leaving for
741
+ // another page and coming back finds the recipe as it was left. Pure halves:
742
+ // the page owns the storage. A draft does not come back when the page was
743
+ // opened for something else - a prefill from a run, or a run's own recipe.
744
+ const OPENED_FOR = ['run','like','dataset','model','agent','n_tasks','standard','other'];
745
+ function packDraft(state) {
746
+ return JSON.stringify({version: 1, at: new Date().toISOString(), recipe: state.recipe, step: state.step, steps: STEPS.length, first: 'benchmark', tab: state.tab});
747
+ }
748
+ function unpackDraft(text, query) {
749
+ if (!text || Object.keys(query || {}).some(k => OPENED_FOR.includes(k))) return null;
750
+ try {
751
+ const d = JSON.parse(text);
752
+ if (!d || d.version !== 1) return null;
753
+ // Translate saved step indexes using the draft's recorded step count.
754
+ const had = Number.isInteger(d.steps) ? d.steps : STEPS.length - 2;
755
+ let step = Number.isInteger(d.step) ? d.step + (had < STEPS.length - 1 ? 1 : 0) : 0;
756
+ if (Number.isInteger(d.step) && had < STEPS.length && step >= STEP.setup) step += 1;
757
+ if (d.first !== 'benchmark' && (step === 0 || step === 1)) step = 1 - step; // a draft from when the runner came first
758
+ return {recipe: fromJSON(d.recipe), step: Math.min(step, STEP.review), tab: typeof d.tab === 'string' ? d.tab : 'select',
759
+ at: String(d.at || '')};
760
+ } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; }
761
+ }
762
+ return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById};
763
+ });