nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,370 @@
1
+ """The benchmarks pages: the list, a card per benchmark, and one page per
2
+ benchmark folder - its harnesses, its datasets, its limitations, its citation."""
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import html
7
+ import json
8
+ import os
9
+ import re
10
+ import time
11
+
12
+ from flask import abort, flash, jsonify, redirect, render_template, request, url_for
13
+ from nullius import execution, paths, providers, secrets, settings
14
+ from nullius.catalogue import catalog, citation, harbor_entry, sources, storage
15
+ from nullius.standard import definitions as standard_definitions
16
+ from nullius.standard import howto, prove
17
+ from nullius.standard import recipe as standard_recipe
18
+
19
+ from .core import app
20
+ from . import execute
21
+ from . import settings as settings_web
22
+ from . import shared
23
+
24
+
25
+ @app.route("/benchmarks")
26
+ def benchmarks_page():
27
+ # the benchmarks of the standard some of whose options need a machine's setup (``host``): a bubble on their card
28
+ setup = {b["key"]: sorted(b["host"]) for b in standard_definitions.load("benchmark")[0] if b.get("host")}
29
+ return render_template("catalogue/benchmarks.html", cat=shared._catalog_cached(), origins=catalog.ORIGINS,
30
+ prompts=howto.prompts(), catalogue_dir=paths.catalogue_dir(), setup=setup)
31
+
32
+
33
+ @app.route("/benchmarks/<key>")
34
+ def benchmark_page(key):
35
+ """One listed benchmark; an entry the list does not show has no page either."""
36
+ cat = shared._catalog_cached()
37
+ b = next((x for x in cat.listed_benchmarks if x.key == key), None)
38
+ if b is None:
39
+ abort(404)
40
+ entry = harbor_entry.entry_of(key)
41
+ # the standard's definition of it, when it has one: its proofs per dataset, and the form that runs cli prove
42
+ std = standard_recipe.page_of(key) if "standard" in b.harness else None
43
+ # where each dataset's data comes from, one row per way it runs - each fetched by its harness, never by hand
44
+ std_def = standard_definitions.find(standard_definitions.load("benchmark")[0], key) if std else None
45
+ ds = cat.datasets_of(b)
46
+ data = {d.key: sources.of(d, b.harness, std_def, only=len(ds) == 1) for d in ds}
47
+ jobs = {j.job_id for j in shared._jobs_for_request()} if std else set()
48
+ return render_template("catalogue/benchmark.html", cat=cat, b=b, csrf=shared._csrf_token(), papers=citation.parse(b.citation), data=data,
49
+ editable=entry is not None and not harbor_entry.not_editable(entry), std=std, std_jobs=jobs,
50
+ deletable=os.path.isfile(os.path.join(paths.catalogue_dir(), catalog.BENCHMARKS, key, catalog.ENTRY)),
51
+ restore_bundle=storage.can_restore("benchmarks", key, catalog.ROOT),
52
+ local=settings_web._local_client(),
53
+ providers=[(p, providers.label(p)) for p in providers.available(os.environ)])
54
+
55
+
56
+ @app.route("/catalogue/<group>/<key>/use-bundled", methods=["POST"])
57
+ def catalogue_use_bundled(group, key):
58
+ shared._require_csrf()
59
+ if not settings_web._local_client():
60
+ abort(403, description="Catalogue entries are changed from a browser on the application host only.")
61
+ endpoints = {"benchmarks": "benchmark_page", "agents": "agent_page", "capabilities": "feature_page"}
62
+ if group not in endpoints:
63
+ abort(404)
64
+ bundled = catalog.ROOT if group == "benchmarks" else standard_definitions.CATALOGUE
65
+ try:
66
+ storage.use_bundled(group, key, bundled)
67
+ except (ValueError, OSError) as error:
68
+ abort(409, description=str(error))
69
+ flash("Using the bundled entry. Saved runs and results are unchanged.", "ok")
70
+ return redirect(url_for(endpoints[group], key=key))
71
+
72
+
73
+ @app.route("/benchmarks/<key>/prove", methods=["POST"])
74
+ def benchmark_prove(key):
75
+ """Prove a benchmark of the standard, here, as a run - its units, the leak
76
+ check and its controls (``cli prove``), as an agent's page proves its
77
+ ports; then the first control's job page, which follows the run. The
78
+ controls call no model, so an unjudged benchmark's run is handed no key
79
+ at all (``execution.Plan.keyless``)."""
80
+ shared._require_csrf()
81
+ if not settings_web._local_client():
82
+ abort(403, description="Proofs are started from a browser on the application host only.")
83
+ back = url_for("benchmark_page", key=key) + "#standard"
84
+ limit = request.form.get("limit", "").strip()
85
+ provider, endpoint = providers.split(request.form.get("provider", "openai"))
86
+ stamp = time.strftime(prove.STAMP)
87
+ got = standard_recipe.proving(key, dataset=request.form.get("dataset", ""), limit=int(limit) if limit.isdigit() else (limit or None),
88
+ provider=provider, judge_model=request.form.get("judge_model", "").strip(), stamp=stamp)
89
+ problem = " ".join(got["errors"]) or ("" if got["keyless"] else execute.provider_problem(provider, endpoint))
90
+ if problem:
91
+ flash(problem, "error")
92
+ return redirect(back)
93
+ plan = execution.Plan(name=f"prove-{key}-{time.strftime('%m%d-%H%M%S')}", script=got["script"], runner=standard_recipe.HARNESS,
94
+ **execute.credentials_for(provider, endpoint), keyless=got["keyless"], jobs=got["jobs"],
95
+ parts=[{"id": key, "label": got["label"] + "'s proofs", "script": got["script"]}])
96
+ problem = plan.check() or (execution.name_taken(plan.name) if os.path.exists(os.path.join(execution.runs_root(), plan.name)) else None)
97
+ if problem:
98
+ flash(problem, "error")
99
+ return redirect(back)
100
+ ex = execution.start(plan)
101
+ shared.adopt_roots([ex.jobs])
102
+ flash(f"Proving {got['label']}: its units and the leak check, then " + (f"{len(got['jobs'])} control runs." if got["jobs"] else "no control: it declares none."), "ok")
103
+ return redirect(url_for("job", job_id=got["jobs"][0]["name"]) if got["jobs"] else back)
104
+
105
+
106
+ @app.route("/benchmarks/<key>/licences", methods=["POST"])
107
+ def benchmark_licence(key):
108
+ """Set, replace, remove or move one of a benchmark's credentials, kept in its
109
+ own store (``nullius.secrets``), managed in Settings: a key or a server's
110
+ address from the text field, a licence file from the upload. Only a name
111
+ the benchmark's environments declare, as the kind they declare it, never
112
+ a data provider's token (Settings → Data providers). A
113
+ browser on this machine only, with the form token, as Settings. *Move
114
+ from saved settings* takes a value saved settings still hold into the
115
+ store and out of the file - on this click, never by itself. Nothing is
116
+ ever sent back of a value."""
117
+ shared._require_csrf()
118
+ if not settings_web._local_client():
119
+ abort(403, description="Credentials are set from a browser on the application host only.")
120
+ back = settings_web._settings_return() + "#access-" + key
121
+ name, action = request.form.get("name", ""), request.form.get("action", "set")
122
+ declared = {s["name"]: s for e in standard_recipe.licences_of(key) for s in e["needs"]}
123
+ s = declared.get(name)
124
+ if s is None or s.get("provider"):
125
+ flash(f"{name} is set in Settings → Data providers." if s else f"{name or 'That'} is not a credential this benchmark needs.", "error")
126
+ return redirect(back)
127
+ try:
128
+ if action == "delete":
129
+ gone = secrets.delete(key, name)
130
+ flash(f"{name} removed." if gone else f"{name} was not set.", "ok" if gone else "error")
131
+ elif action == "move":
132
+ value = settings.in_file(shared.ENV_FILE).get(name)
133
+ if not value:
134
+ flash(f"Saved settings hold no {name}.", "error")
135
+ return redirect(back)
136
+ secrets.set(key, name, value, kind=s["kind"])
137
+ settings.take(shared.ENV_FILE, name)
138
+ flash(f"{name} moved out of saved settings.", "ok")
139
+ else:
140
+ expires = request.form.get("expires", "").strip()
141
+ if s["kind"] == "file":
142
+ f = request.files.get("file")
143
+ data = f.read(secrets.MAX_BYTES + 1) if f else b""
144
+ if not data:
145
+ flash(f"{name}: choose the licence file.", "error")
146
+ return redirect(back)
147
+ secrets.set(key, name, data, kind="file", filename=f.filename or "", expires=expires)
148
+ else:
149
+ secrets.set(key, name, request.form.get("value", ""), kind=s["kind"], expires=expires)
150
+ flash(f"{name} set.", "ok")
151
+ except ValueError as e:
152
+ flash(str(e), "error")
153
+ return redirect(back)
154
+
155
+
156
+ # --- adding a Harbor benchmark by hand ----------------------------------------------
157
+
158
+ MARKS = os.path.join(shared._APP_DIR, "static", "catalogue", "marks.svg")
159
+ LOGO_TYPES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".svg": "image/svg+xml"}
160
+ LOGO_MAX = 200_000
161
+ # the mark of a benchmark added without a logo: a box of tasks, in the current colour
162
+ PLAIN_MARK = ('<symbol id="bm-{key}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" '
163
+ 'stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="4" width="18" height="16" rx="2"/>'
164
+ '<path d="M7 9h10M7 13h10M7 17h6"/></symbol>')
165
+
166
+
167
+ def _guard():
168
+ """Every write here: form token, and a browser on this machine - adding a
169
+ benchmark writes into the user's catalogue."""
170
+ shared._require_csrf()
171
+ if not settings_web._local_client():
172
+ abort(403, description="Benchmarks are added from a browser on the application host only.")
173
+
174
+
175
+ def _set_mark(key: str, logo) -> None:
176
+ """Keep an uploaded logo beside the user entry; no logo keeps its mark."""
177
+ if logo is not None:
178
+ harbor_entry._write(os.path.join(catalog.writable_dir(key), "logo.json"), logo)
179
+
180
+
181
+ @app.route("/static/catalogue/marks.svg")
182
+ def benchmark_marks():
183
+ """The bundled sprite with user logos and a fallback for custom entries.
184
+
185
+ The exact route precedes Flask's generic static-file route, so the
186
+ builder and every catalogue page keep their existing sprite URL.
187
+ """
188
+ with open(MARKS, encoding="utf-8") as fh:
189
+ text = fh.read()
190
+ cat = shared._catalog_cached()
191
+ for b in cat.benchmarks:
192
+ symbol = None
193
+ try:
194
+ with open(os.path.join(cat.entry_dir(b.key), "logo.json"), encoding="utf-8") as fh:
195
+ logo = json.load(fh)
196
+ if (isinstance(logo, dict) and logo.get("kind") in LOGO_TYPES.values()
197
+ and isinstance(logo.get("data"), str) and len(logo["data"]) * 3 // 4 <= LOGO_MAX
198
+ and re.fullmatch(r"[A-Za-z0-9+/=]+", logo["data"])):
199
+ symbol = (f'<symbol id="bm-{html.escape(b.key, quote=True)}" viewBox="0 0 24 24">'
200
+ f'<image width="24" height="24" href="data:{logo["kind"]};base64,{logo["data"]}"/></symbol>')
201
+ except (OSError, ValueError):
202
+ pass
203
+ have = re.search(rf'<symbol id="bm-{re.escape(b.key)}"[\s\S]*?</symbol>', text)
204
+ if have:
205
+ if symbol:
206
+ text = text[:have.start()] + symbol + text[have.end():]
207
+ else:
208
+ at = text.rindex("</svg>")
209
+ text = text[:at] + " " + (symbol or PLAIN_MARK.format(key=html.escape(b.key, quote=True))) + "\n" + text[at:]
210
+ return app.response_class(text, mimetype="image/svg+xml", headers={"Cache-Control": "no-cache"})
211
+
212
+
213
+ def _logo(values: dict):
214
+ """``({kind, data, name} or None, problem or None)``: the file chosen now,
215
+ else the one a save that came back kept in the form (``logo_kept``)."""
216
+ up = request.files.get("logo")
217
+ if up is not None and up.filename:
218
+ kind = LOGO_TYPES.get(os.path.splitext(up.filename)[1].lower())
219
+ if kind is None:
220
+ return None, "logo: a PNG, JPEG or SVG file"
221
+ raw = up.read()
222
+ if len(raw) > LOGO_MAX:
223
+ return None, f"logo: {len(raw) // 1000} KB; keep it under {LOGO_MAX // 1000} KB (a 64 px image is plenty)"
224
+ return {"kind": kind, "data": base64.b64encode(raw).decode(), "name": os.path.basename(up.filename)}, None
225
+ kept = re.match(r"^data:(image/png|image/jpeg|image/svg\+xml);base64,([A-Za-z0-9+/=]+)$", values.get("logo_kept") or "")
226
+ if kept and len(kept.group(2)) * 3 // 4 <= LOGO_MAX:
227
+ return {"kind": kept.group(1), "data": kept.group(2), "name": values.get("logo_kept_name") or "the logo chosen"}, None
228
+ return None, None
229
+
230
+
231
+ def _form(values, errors=(), status=200, editing=None):
232
+ """The add form, or the same form editing ``editing`` (an entry's key)."""
233
+ cat = shared._catalog_cached()
234
+ return render_template("catalogue/new.html", cat=cat, values=values, errors=list(errors), editing=editing, routable=_routable(),
235
+ origins=catalog.ORIGINS, catalogue_dir=paths.catalogue_dir(),
236
+ writable=settings_web._local_client(), csrf=shared._csrf_token()), status
237
+
238
+
239
+ def _back(values, errors, logo, editing=None):
240
+ """The form again, with what to fix and everything typed - the logo too,
241
+ which a browser never puts back in a file field."""
242
+ if logo is not None:
243
+ values = dict(values, logo_kept=f"data:{logo['kind']};base64,{logo['data']}", logo_kept_name=logo["name"])
244
+ return _form(values, errors, 400, editing)
245
+
246
+
247
+ def _opened(key: str):
248
+ listed = any(b.key == key for b in shared._catalog_cached().listed_benchmarks)
249
+ return redirect(url_for("benchmark_page", key=key) if listed else url_for("benchmarks_page"))
250
+
251
+
252
+ @app.route("/benchmarks/new", methods=["GET"])
253
+ def benchmark_new():
254
+ """The form that adds a Harbor benchmark to the catalogue."""
255
+ return _form({"source": "hub", "origin": "official"})
256
+
257
+
258
+ @app.route("/benchmarks/new/read", methods=["POST"])
259
+ def benchmark_new_read():
260
+ """What Harbor says of a dataset, to fill the form: its task count and
261
+ the limits its tasks declare. Asked on the button, never on a page load."""
262
+ _guard()
263
+ body = request.get_json(silent=True) or {}
264
+ try:
265
+ return jsonify(ok=True, **harbor_entry.read(str(body.get("source") or ""), str(body.get("ref") or "")))
266
+ except harbor_entry.ReadError as e:
267
+ return jsonify(ok=False, error=str(e)), 400
268
+
269
+
270
+ @app.route("/benchmarks/new", methods=["POST"])
271
+ def benchmark_new_save():
272
+ """Write the entry, and its mark, then open its page. Nothing is written
273
+ while anything is wrong: the form comes back with what to fix."""
274
+ _guard()
275
+ values = request.form.to_dict()
276
+ route_for = (values.get("route_for") or "").strip()
277
+ if route_for:
278
+ return _add_route(route_for, values)
279
+ logo, bad_logo = _logo(values)
280
+ entry = harbor_entry.build(values)
281
+ errors = harbor_entry.problems(entry)
282
+ if entry["key"] == "new":
283
+ errors.append("key: new is the address of this form")
284
+ if bad_logo:
285
+ errors.append(bad_logo)
286
+ if errors:
287
+ return _back(values, errors, logo)
288
+ harbor_entry.save(entry)
289
+ _set_mark(entry["key"], logo)
290
+ return _opened(entry["key"])
291
+
292
+
293
+ def _routable() -> list:
294
+ """The benchmarks a Harbor route can be added to: listed, and run so far only on the Nullius standard."""
295
+ out = []
296
+ for b in shared._catalog_cached().benchmarks:
297
+ if "harbor" not in b.harness:
298
+ out.append((b.key, b.title))
299
+ return out
300
+
301
+
302
+ def _add_route(key: str, values: dict):
303
+ """The form's dataset as a Harbor route of an existing benchmark: one
304
+ entry, one card, run with Harbor or on the Nullius standard."""
305
+ old = harbor_entry.entry_of(key)
306
+ if old is None:
307
+ abort(404)
308
+ if "harbor" in (old.get("harness") or []):
309
+ return _form(values, [f"{old.get('title', key)} already runs with Harbor: edit it from its page"], 400)
310
+ entry = harbor_entry.harbor_route(old, values)
311
+ errors = harbor_entry.problems(entry, replacing=key, dataset=-1)
312
+ if errors:
313
+ return _form(values, errors, 400)
314
+ harbor_entry.update(entry, was=key, dataset=-1)
315
+ flash(f"{old.get('title', key)} now runs with Harbor too.", "ok")
316
+ return _opened(key)
317
+
318
+
319
+ def _editable(key: str) -> dict:
320
+ entry = harbor_entry.entry_of(key)
321
+ if entry is None:
322
+ abort(404)
323
+ why = harbor_entry.not_editable(entry)
324
+ if why:
325
+ abort(409, description=f"This entry is not edited here: {why}.")
326
+ return entry
327
+
328
+
329
+ @app.route("/benchmarks/<key>/edit", methods=["GET"])
330
+ def benchmark_edit(key):
331
+ """The add form, filled with a Harbor entry, to change it."""
332
+ return _form(harbor_entry.values_of(_editable(key)), editing=key)
333
+
334
+
335
+ @app.route("/benchmarks/<key>/edit", methods=["POST"])
336
+ def benchmark_edit_save(key):
337
+ """Write the changes over the entry, keeping all it holds that the form
338
+ does not show; a logo chosen replaces its mark. Nothing is written while
339
+ anything is wrong."""
340
+ _guard()
341
+ old = _editable(key)
342
+ values = request.form.to_dict()
343
+ logo, bad_logo = _logo(values)
344
+ entry = dict(harbor_entry.merged(old, harbor_entry.build(values)), key=values.get("key", "").strip() or key)
345
+ errors = harbor_entry.problems(entry, replacing=key)
346
+ if entry["key"] == "new":
347
+ errors.append("key: new is the address of the add form")
348
+ if bad_logo:
349
+ errors.append(bad_logo)
350
+ if errors:
351
+ return _back(values, errors, logo, editing=key)
352
+ harbor_entry.update(entry, was=key)
353
+ _set_mark(entry["key"], logo)
354
+ return _opened(entry["key"])
355
+
356
+
357
+ @app.route("/benchmarks/<key>/delete", methods=["POST"])
358
+ def benchmark_delete(key):
359
+ """Remove a Harbor entry: its folder and its mark. The page asks first,
360
+ in place; runs already made from it are not touched."""
361
+ _guard()
362
+ entry = _editable(key)
363
+ try:
364
+ harbor_entry.delete(key)
365
+ except ValueError as e:
366
+ abort(409, description=str(e))
367
+ restored = harbor_entry.entry_of(key) is not None
368
+ flash(f"Deleted {entry.get('title') or key} from your catalogue; "
369
+ + ("the bundled entry is shown again. " if restored else "") + "Runs made from it are untouched.", "ok")
370
+ return redirect(url_for("benchmarks_page"))