nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/settings.py ADDED
@@ -0,0 +1,544 @@
1
+ """The application's settings: every environment variable it reads, in one
2
+ table, and the one file it writes them to.
3
+
4
+ The user configuration ``settings.json`` is loaded at startup (``load``) with the rule the application has
5
+ always had — a variable already in the process environment wins over the
6
+ file — and read again (``refresh``) whenever the file has changed on disk
7
+ (``sync``, which the application calls before each request), so a file
8
+ edited by hand while the application runs counts from the next request,
9
+ without a restart; the Settings page edits
10
+ the same file (``save``) and names the keys in it that nothing reads (``unread``). What the page
11
+ shows for each variable is its *state*, never a secret's value: ``set`` from
12
+ the environment or from the file, or ``not set``, with a fixed mask and nothing else. Values are validated per variable before a byte is written;
13
+ the file is written whole, to a temporary name first, with mode 0600; a
14
+ variable the process environment holds is shown read-only, because the file
15
+ could not override it anyway.
16
+
17
+ ``example`` renders ``app/.env.example`` from the same table, so the two
18
+ cannot drift (a test compares them).
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import os
23
+ import json
24
+ import re
25
+ import tempfile
26
+ import urllib.parse
27
+ from dataclasses import dataclass
28
+ from typing import Callable, Dict, List, Optional, Tuple
29
+
30
+ _PATH_OK = re.compile(r"^[^\x00-\x1f]{1,400}$")
31
+ _URL_OK = re.compile(r"^https?://[^\s/?#]+(?:[/?#]\S*)?$")
32
+ _REPO_OK = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
33
+ _NAME_OK = re.compile(r"^[A-Za-z0-9._-]{1,64}$")
34
+ _SECRET_OK = re.compile(r"^[!-~]+$") # printable ASCII, no spaces
35
+ _LOCAL_KEY_OK = re.compile(r"^[!-~]+$") # the same, any length: a local server's key is often a word ("ollama", "EMPTY")
36
+ _TEXT_OK = re.compile(r"^[^\x00-\x1f]{1,400}$")
37
+ _MODEL_OK = re.compile(r"^~?[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,159}$") # an OpenRouter model id: [~]vendor/model[:variant]
38
+ _S1_OK = re.compile(r"^(typesafe|openrouter)$")
39
+ _LLM_JUDGE_OK = re.compile(r"^(openrouter|openai|local(:[A-Z0-9]+(_[A-Z0-9]+)*)?)$")
40
+ _ENVNAME_OK = re.compile(r"^[A-Z_][A-Z0-9_]*$")
41
+ _BOOL = {"0", "1", "true", "false", "yes", "no"}
42
+
43
+
44
+ def _check(pattern: re.Pattern, what: str) -> Callable[[str], Optional[str]]:
45
+ return lambda v: None if pattern.match(v) else what
46
+
47
+
48
+ def _paths(v: str) -> Optional[str]:
49
+ return None if v and all(_PATH_OK.match(p) for p in v.split(os.pathsep)) else \
50
+ f"one or more paths separated by {os.pathsep!r}, without control characters"
51
+
52
+
53
+ # Names an agent key may not take: the run's own, the shell's, and the ones that steer a process
54
+ _NOT_AGENT_KEYS = {"API_TOKEN", "OPENAI_API_KEY", "LLM_API_KEY", "API_URL", "OPENAI_BASE_URL", "LLM_BASE_URL", "API_MODEL_NAME",
55
+ "PATH", "HOME", "USER", "SHELL", "PWD", "LANG", "TERM", "TMPDIR", "PYTHONPATH", "PYTHONHOME",
56
+ "LD_PRELOAD", "LD_LIBRARY_PATH", "SSH_AUTH_SOCK"}
57
+
58
+
59
+ def reserved_name(n: str) -> bool:
60
+ """A name that is already something else's: a setting, a start-up option,
61
+ another spelling of one, the run's own or the shell's. An agent key or a
62
+ benchmark's secret may not take it."""
63
+ return n in BY_NAME or n in STARTUP or n in INSTEAD or n in _NOT_AGENT_KEYS or n.startswith("NULLIUS_") or bool(_ENDPOINT_VAR.match(n))
64
+
65
+
66
+ def _agent_key_names(v: str) -> Optional[str]:
67
+ names = [n.strip() for n in v.split(",") if n.strip()]
68
+ for n in names:
69
+ if not _ENVNAME_OK.match(n):
70
+ return f"names in capitals, digits and underscores ({n!r} is not)"
71
+ if reserved_name(n):
72
+ return f"names of their own ({n} is already the name of something else)"
73
+ if len(set(names)) != len(names):
74
+ return "names, each once"
75
+ return None
76
+
77
+
78
+ # Query parameters that carry a credential, whatever the service calls them
79
+ _CREDENTIAL_PARAM = re.compile(r"^(key|api[_-]?key|token|access[_-]?token|secret|password|auth|authorization|sig|signature)$", re.I)
80
+ _PUT_KEY = "put the key in the Key field"
81
+
82
+
83
+ def endpoint_url_problem(v: str) -> Optional[str]:
84
+ """Why ``v`` cannot be an endpoint's URL, or None: an http(s) URL with a host, and no
85
+ credential in it - no user or password, no query parameter named like a key - since
86
+ the page shows it."""
87
+ try:
88
+ p = urllib.parse.urlsplit(v)
89
+ p.port
90
+ except ValueError:
91
+ return "an http(s) URL with a host"
92
+ if p.scheme not in ("http", "https") or not p.hostname or any(c.isspace() or ord(c) < 32 for c in v):
93
+ return "an http(s) URL with a host"
94
+ if "@" in p.netloc:
95
+ return f"a URL without a user or password in it: {_PUT_KEY}"
96
+ if any(_CREDENTIAL_PARAM.match(k) for k, _ in urllib.parse.parse_qsl(p.query, keep_blank_values=True)):
97
+ return f"a URL without a key in its query: {_PUT_KEY}"
98
+ return None
99
+
100
+
101
+ def endpoint_url_shown(v: str) -> str:
102
+ """An endpoint's URL as the page shows it: as it is, or, when it holds a credential,
103
+ with the user, the password and each key-like query value hidden."""
104
+ if not endpoint_url_problem(v):
105
+ return v
106
+ try:
107
+ p = urllib.parse.urlsplit(v)
108
+ except ValueError:
109
+ return SECRET_SHOWN
110
+ netloc = ("[REDACTED]@" if "@" in p.netloc else "") + p.netloc.rpartition("@")[2]
111
+ query = "&".join(f"{k}=[REDACTED]" if _CREDENTIAL_PARAM.match(k) else f"{k}={w}"
112
+ for k, w in urllib.parse.parse_qsl(p.query, keep_blank_values=True))
113
+ return urllib.parse.urlunsplit((p.scheme, netloc, p.path, query, p.fragment))
114
+
115
+
116
+ def _hosts(v: str) -> Optional[str]:
117
+ from nullius import hosts
118
+ return hosts.validate_list(v)
119
+
120
+
121
+ def _bool(v: str) -> Optional[str]:
122
+ return None if v.lower() in _BOOL else "0, 1, true, false, yes or no"
123
+
124
+
125
+ @dataclass(frozen=True)
126
+ class Variable:
127
+ name: str
128
+ group: str
129
+ purpose: str
130
+ kind: str # secret | localkey | endpoint | path | paths | url | repo | name | bool | text | envname | hosts | model | llmprovider | s1provider | envnames
131
+ example: str = "" # shown as a placeholder, written to .env.example as a comment
132
+
133
+ @property
134
+ def secret(self) -> bool:
135
+ return self.kind in ("secret", "localkey")
136
+
137
+ def validate(self, value: str) -> Optional[str]:
138
+ """None when acceptable, else what the value should have been."""
139
+ if "\n" in value or "\r" in value:
140
+ return "a single line"
141
+ if value != value.strip():
142
+ return "no leading or trailing whitespace"
143
+ return {
144
+ "secret": _check(_SECRET_OK, "a nonempty single-line value without spaces"),
145
+ "localkey": _check(_LOCAL_KEY_OK, "a nonempty single-line value without spaces"),
146
+ "path": _check(_PATH_OK, "a path without control characters"),
147
+ "paths": _paths,
148
+ "url": _check(_URL_OK, "an http(s) URL"),
149
+ "endpoint": endpoint_url_problem,
150
+ "repo": _check(_REPO_OK, "owner/name"),
151
+ "name": _check(_NAME_OK, "letters, digits, dots, dashes or underscores (64 at most)"),
152
+ "bool": _bool,
153
+ "text": _check(_TEXT_OK, "a single line without control characters"),
154
+ "envname": _check(_ENVNAME_OK, "an environment variable name (capitals, digits, underscores)"),
155
+ "hosts": _hosts,
156
+ "model": _check(_MODEL_OK, "a model id as OpenRouter lists it, vendor/model"),
157
+ "llmprovider": _check(_LLM_JUDGE_OK, "openrouter, openai, local or local:<endpoint id>"),
158
+ "s1provider": _check(_S1_OK, "typesafe or openrouter"),
159
+ "envnames": _agent_key_names,
160
+ }[self.kind](value)
161
+
162
+
163
+ GROUPS = ["Model credentials", "Run hosts", "Custom keys"]
164
+ # The key a run on an OpenAI-compatible endpoint is sent with when the
165
+ # endpoint has no key: many such servers check none, but an
166
+ # OpenAI-compatible client refuses to start without one.
167
+ MODEL_KEY_PLACEHOLDER = "local"
168
+ # The OpenAI-compatible endpoint MODEL_API_URL and MODEL_API_KEY hold, by this name.
169
+ DEFAULT_ENDPOINT_NAME = "OpenAI-compatible"
170
+ # More OpenAI-compatible endpoints, each by an id: MODEL_<ID>_NAME, MODEL_<ID>_API_URL and
171
+ # MODEL_<ID>_API_KEY. A recipe records the id, not the name, so a rename keeps its recipes.
172
+ ENDPOINT_ID = re.compile(r"^(?=.{1,32}$)[A-Z0-9]+(?:_[A-Z0-9]+)*$")
173
+ _ENDPOINT_VAR = re.compile(r"^MODEL_([A-Z0-9]+(?:_[A-Z0-9]+)*)_(NAME|API_URL|API_KEY)$")
174
+ ENDPOINT_PARTS = ("NAME", "API_URL", "API_KEY")
175
+
176
+ # What a person sets. Everything else the application reads is a start-up
177
+ # option with a command-line form or a sensible default (STARTUP below) and
178
+ # is not on the page.
179
+ VARIABLES: List[Variable] = [
180
+ Variable("MODEL_API_KEY", GROUPS[0], "The key of the OpenAI-compatible endpoint at MODEL_API_URL: a recipe on that endpoint is sent there with this key. Optional, for a server that checks none: without it the run is sent the word " + repr(MODEL_KEY_PLACEHOLDER) + " as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log.", "localkey", "sk-…"),
181
+ Variable("MODEL_API_URL", GROUPS[0], "The URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More endpoints, each with a name, are MODEL_<ID>_NAME, MODEL_<ID>_API_URL and MODEL_<ID>_API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.", "endpoint", "https://…/v1"),
182
+ Variable("OPENAI_API_KEY", GROUPS[0], "An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own endpoint, https://api.openai.com/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"),
183
+ Variable("OPENROUTER_API_KEY", GROUPS[0], "An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own endpoint, https://openrouter.ai/api/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"),
184
+ Variable("ANTHROPIC_API_KEY", GROUPS[0], "An Anthropic key, for a run of the Nullius standard whose provider is Anthropic: its relay sends the calls to Anthropic's own endpoint, https://api.anthropic.com/v1, with this key. Only the relay holds it - never an agent, a recipe or a log.", "secret"),
185
+ Variable("TYPESAFE_API_KEY", GROUPS[0], "TypeSafe.ai key for the System One judgement on TypeSafe's own endpoint, POST https://api.typesafe.ai/v1/systemone — sent as a bearer header there and nowhere else.", "secret", "ts-…"),
186
+ Variable("SYSTEM_ONE_PROVIDER", GROUPS[0], "Where the System One judgement is asked: typesafe (its own endpoint, with TYPESAFE_API_KEY) or openrouter (https://openrouter.ai/api/v1/systemone, the same contract, with OPENROUTER_API_KEY). Unset: TypeSafe when its key is set, else OpenRouter when its key is.", "s1provider"),
187
+ Variable("SYSTEM_ONE_MODEL", GROUPS[0], "The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest, typesafe/jev-router, jaredpalmer/kev-4b... Unset: the provider's Jev.", "model"),
188
+ Variable("LLM_JUDGE_PROVIDER", GROUPS[0], "The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, local (the endpoint at MODEL_API_URL) or local:<ID> (the endpoint MODEL_<ID>_API_URL), asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.", "llmprovider"),
189
+ Variable("LLM_JUDGE_MODEL", GROUPS[0], "The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.", "model"),
190
+ Variable("AGENT_KEYS", GROUPS[2], "The names of the agent keys this file holds, comma-separated: the keys an agent needs of its own vendor, beside the model's (Cursor's CLI agent a CURSOR_API_KEY, Copilot's a GH_TOKEN). Each is a line of its own in this file. A Harbor run gets every one of them in its environment - over stdin to a host - and a Nullius standard run's never; none is ever written into a recipe or a log.", "envnames"),
191
+ Variable("HOSTS", GROUPS[1], "Machines a recipe can run on, over ssh: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]. Only the name is stored; keys stay where they are.", "hosts", "user@host[:port][:/path/to/key]"),
192
+ ]
193
+ # Read by the application but not offered on the page: each has a
194
+ # command-line form, a test-only use, or a default nobody changes. The
195
+ # server's address and port are read once, at start: the running server's
196
+ # own page could not change them.
197
+ STARTUP: Dict[str, str] = {
198
+ "NULLIUS_HOME": "a separate profile with config/, data/ and cache/ directories",
199
+ "NULLIUS_CATALOGUE": "the user catalogue directory (default catalogue/ in the user data directory)",
200
+ "NULLIUS_PRICES": "the model price list file (default prices.json in the user configuration directory); set for a run on a host",
201
+ "APP_HOST": "the address the server binds (default 127.0.0.1, or --host)",
202
+ "APP_PORT": "the port the server listens on (default 5057, or --port)",
203
+ "JOB_ROOTS": "job roots to open at start (or the command-line argument)",
204
+ "WORKSPACES_DIR": "where workspaces are kept (default workspaces/ in the user data directory)",
205
+ "WORKSPACE": "the name the start-up roots open as (or the picker's name field)",
206
+ "DATASET_ROOTS": "local dataset checkouts to look in for task files",
207
+ "DEBUG": "the Werkzeug debugger (or --no-debug)",
208
+ "RUNS_DIR": "where runs started from the builder keep their script, log and job (default runs/ in the user data directory)",
209
+ "HARBOR_PINS": "where the Harbor a recipe pins is built and kept on this machine (default harbor-pins/ beside the runs)",
210
+ "STANDARD_DIR": "where the Nullius standard installs agents, capabilities and benchmark upstreams (default standard/ in the user data directory)",
211
+ "STANDARD_DATA_DIR": "where the Nullius standard keeps the datasets it downloads, which can be gigabytes (default datasets/ in STANDARD_DIR)",
212
+ "SECRETS_DIR": "where each benchmark's licences (Settings → Custom keys) and the Hugging Face token (Settings → Data providers) are kept (default secrets/ in the user configuration directory)",
213
+ }
214
+ BY_NAME: Dict[str, Variable] = {v.name: v for v in VARIABLES}
215
+ LOADED_KEYS: set = set() # variables ``load`` set from the file (so: not held by the process environment)
216
+ _FROM: Dict[str, str] = {} # which file each of them came from, so a refresh of one file never drops another's
217
+ # Names other tools give the model settings, and names this application
218
+ # used before: a file written with one is read by nobody here, and the page
219
+ # says which of its own names was meant - for HOST and PORT, a start-up
220
+ # option, read at start. OPENAI_API_KEY and OPENROUTER_API_KEY are its own:
221
+ # the keys a recipe on OpenAI or on OpenRouter is sent with.
222
+ INSTEAD: Dict[str, str] = {"API_TOKEN": "MODEL_API_KEY", "API_URL": "MODEL_API_URL", "OPENAI_BASE_URL": "MODEL_API_URL",
223
+ "API_KEY_TYPESAFE": "TYPESAFE_API_KEY", "JUDGE_MODEL": "LLM_JUDGE_MODEL", "HOST": "APP_HOST", "PORT": "APP_PORT"}
224
+
225
+
226
+ def parse(path: str) -> List[Tuple[str, Optional[str], Optional[str]]]:
227
+ """The file as a list of (line, key, value): key and value are None for
228
+ a comment or blank line. Quotes around a value are dropped, as ``load``
229
+ always has."""
230
+ out = []
231
+ if not os.path.isfile(path):
232
+ return out
233
+ with open(path, encoding="utf-8") as fh:
234
+ if str(path).endswith(".json"):
235
+ try:
236
+ values = json.load(fh)
237
+ except (ValueError, UnicodeError) as e:
238
+ raise ValueError("Settings file is not valid JSON; it has not been changed.") from e
239
+ if not isinstance(values, dict) or any(
240
+ not isinstance(k, str) or not _ENVNAME_OK.fullmatch(k) or
241
+ not isinstance(v, str) or any(c in v for c in "\0\r\n")
242
+ for k, v in values.items()
243
+ ):
244
+ raise ValueError("Settings must be an object of variable names and single-line strings.")
245
+ return [(f"{key}={value}", key, value) for key, value in values.items()]
246
+ for raw in fh:
247
+ line = raw.rstrip("\n")
248
+ s = line.strip()
249
+ if not s or s.startswith("#") or "=" not in s:
250
+ out.append((line, None, None))
251
+ continue
252
+ key, _, value = s.partition("=")
253
+ out.append((line, key.strip(), value.strip().strip('"').strip("'")))
254
+ return out
255
+
256
+
257
+ def load(path: str) -> set:
258
+ """Apply the file to the process environment: existing variables win.
259
+ Returns the names the file supplied."""
260
+ applied = set()
261
+ for _, key, value in parse(path):
262
+ if key and key not in os.environ:
263
+ os.environ[key] = value or ""
264
+ applied.add(key)
265
+ LOADED_KEYS.update(applied)
266
+ _FROM.update((k, os.path.abspath(path)) for k in applied)
267
+ return applied
268
+
269
+
270
+ def refresh(path: str) -> Dict[str, List[str]]:
271
+ """The file as it is now, under ``load``'s rule, for a file written or
272
+ edited while the application runs (``sync`` calls it when the file has
273
+ changed): a key the process environment lacks is applied; a key the file
274
+ supplied follows the file - a new value, or gone when its line is; a
275
+ variable the process was started with still wins. A key taken out of
276
+ the environment (a removal saved from the page, a test) is not put back.
277
+ Returns what changed, by name and never by value: the keys ``loaded``,
278
+ ``updated`` and ``removed``, each sorted."""
279
+ here = os.path.abspath(path)
280
+ values = in_file(path)
281
+ done: Dict[str, List[str]] = {"loaded": [], "updated": [], "removed": []}
282
+ for key in [k for k in LOADED_KEYS if _FROM.get(k) == here and k not in values]:
283
+ if os.environ.pop(key, None) is not None:
284
+ done["removed"].append(key)
285
+ LOADED_KEYS.discard(key)
286
+ for key, value in values.items():
287
+ if key in LOADED_KEYS:
288
+ if key in os.environ and os.environ[key] != value:
289
+ os.environ[key] = value
290
+ done["updated"].append(key)
291
+ elif key not in os.environ:
292
+ os.environ[key] = value
293
+ LOADED_KEYS.add(key)
294
+ _FROM[key] = here
295
+ done["loaded"].append(key)
296
+ return {k: sorted(v) for k, v in done.items()}
297
+
298
+
299
+ _SEEN: Dict[str, Optional[tuple]] = {} # the file as last read by ``sync``: its stat, or None when it was absent
300
+
301
+
302
+ def sync(path: str) -> Dict[str, List[str]]:
303
+ """``refresh`` when the file is not as it was last read - another
304
+ size, time or inode, or come or gone - and nothing otherwise, so a
305
+ request costs one stat. Returns what ``refresh`` returns, every list
306
+ empty when the file was as read."""
307
+ here = os.path.abspath(path)
308
+ try:
309
+ st = os.stat(here)
310
+ now: Optional[tuple] = (st.st_mtime_ns, st.st_size, st.st_ino)
311
+ except OSError:
312
+ now = None
313
+ if here in _SEEN and _SEEN[here] == now:
314
+ return {"loaded": [], "updated": [], "removed": []}
315
+ done = refresh(path)
316
+ _SEEN[here] = now
317
+ return done
318
+
319
+
320
+ def in_file(path: str) -> Dict[str, str]:
321
+ return {k: v or "" for _, k, v in parse(path) if k}
322
+
323
+
324
+ def agent_key_names(env=None) -> List[str]:
325
+ """The agent keys' names, as AGENT_KEYS lists them; a list that would
326
+ not validate names none."""
327
+ raw = (os.environ if env is None else env).get("AGENT_KEYS", "")
328
+ if _agent_key_names(raw):
329
+ return []
330
+ return [n.strip() for n in raw.split(",") if n.strip()]
331
+
332
+
333
+ def agent_key_values(env=None) -> Dict[str, str]:
334
+ """Each agent key that holds a value: what a Harbor run is sent."""
335
+ env = os.environ if env is None else env
336
+ return {n: env[n] for n in agent_key_names(env) if env.get(n)}
337
+
338
+
339
+ def endpoint_ids(env=None) -> List[str]:
340
+ """The ids of the named OpenAI-compatible endpoints: every id one of whose settings is set."""
341
+ env = os.environ if env is None else env
342
+ return sorted({m.group(1) for m in map(_ENDPOINT_VAR.match, env) if m and ENDPOINT_ID.match(m.group(1))})
343
+
344
+
345
+ def endpoint_settings(i: str) -> List[str]:
346
+ """The settings of the endpoint ``i``: its name, URL and key."""
347
+ return [f"MODEL_{i}_{part}" for part in ENDPOINT_PARTS]
348
+
349
+
350
+ def endpoint_names(env=None) -> List[str]:
351
+ """Every setting of a named endpoint that ``env`` holds: what a run never inherits."""
352
+ return sorted(n for n in (os.environ if env is None else env) if _ENDPOINT_VAR.match(n))
353
+
354
+
355
+ def endpoint_variable(name: str) -> Optional[Variable]:
356
+ """A named endpoint's setting as the page and ``save_many`` treat it, or None for another name."""
357
+ m = _ENDPOINT_VAR.match(name)
358
+ if not m or not ENDPOINT_ID.match(m.group(1)):
359
+ return None
360
+ what, kind = {"NAME": ("name", "text"), "API_URL": ("URL", "endpoint"), "API_KEY": ("key, optional", "localkey")}[m.group(2)]
361
+ return Variable(name, GROUPS[0], f"The {what} of the OpenAI-compatible endpoint {m.group(1)}.", kind)
362
+
363
+
364
+ def new_endpoint_id(name: str, taken) -> str:
365
+ """An id for a new endpoint, from its name: capitals, digits and underscores, not one of ``taken``."""
366
+ base = re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")[:28].strip("_") or "ENDPOINT"
367
+ n, out = 1, base
368
+ while out in taken:
369
+ n += 1
370
+ out = f"{base}_{n}"
371
+ return out
372
+
373
+
374
+ def agent_variable(name: str) -> Variable:
375
+ """An agent key as the page and ``save_many`` treat it: a secret of its own."""
376
+ return Variable(name, GROUPS[2], f"An agent key, {name}: sent to Harbor runs, to the agent that reads it.", "secret")
377
+
378
+
379
+ def _write_lines(path: str, lines: List[str]) -> None:
380
+ """Atomically replace one owner-only settings file; never print values."""
381
+ directory = os.path.dirname(os.path.abspath(path))
382
+ os.makedirs(directory, mode=0o700, exist_ok=True)
383
+ fd, tmp = tempfile.mkstemp(prefix=".settings-", dir=directory)
384
+ try:
385
+ os.fchmod(fd, 0o600)
386
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
387
+ if str(path).endswith(".json"):
388
+ values = {line.partition("=")[0]: line.partition("=")[2]
389
+ for line in lines if "=" in line and not line.lstrip().startswith("#")}
390
+ json.dump(values, fh, indent=2, ensure_ascii=False)
391
+ fh.write("\n")
392
+ else:
393
+ fh.write("\n".join(lines) + ("\n" if lines else ""))
394
+ os.replace(tmp, path)
395
+ except BaseException:
396
+ try:
397
+ os.unlink(tmp)
398
+ except OSError:
399
+ pass
400
+ raise
401
+
402
+
403
+ def take(path: str, name: str) -> Optional[str]:
404
+ """Move a line out of the file: ``name``'s value, its line removed and,
405
+ when the file put it there, its variable gone from the process
406
+ environment; None when the file has no such line. Only for a name no
407
+ setting reads (``unread``) - a benchmark's licence from before licences
408
+ were kept per benchmark (``nullius.secrets``), which its page moves
409
+ there on the user's word."""
410
+ if name in BY_NAME or name in STARTUP or name in set(agent_key_names()):
411
+ raise ValueError(f"{name} is a setting of its own; it is not moved")
412
+ value = in_file(path).get(name)
413
+ if value is None:
414
+ return None
415
+ lines = [line for line, key, _ in parse(path) if key != name]
416
+ while lines and not lines[-1].strip():
417
+ lines.pop()
418
+ _write_lines(path, lines)
419
+ if name in LOADED_KEYS:
420
+ os.environ.pop(name, None)
421
+ LOADED_KEYS.discard(name)
422
+ return value
423
+
424
+
425
+ def unread(path: str) -> List[dict]:
426
+ """Keys in the file that no setting or start-up option reads, each with
427
+ the page's own name when it is a known other spelling (``API_TOKEN``):
428
+ the file applies them to the environment, but nothing here reads them.
429
+ A run inherits those its launcher does not withhold
430
+ (``execution.withheld``: every other spelling of a key among them)."""
431
+ keys = set(agent_key_names()) | set(agent_key_names(in_file(path)))
432
+ return [{"name": k, "instead": INSTEAD.get(k, "")} for k in in_file(path)
433
+ if k not in BY_NAME and k not in STARTUP and k not in keys and not endpoint_variable(k)]
434
+
435
+
436
+ # What a page shows of a secret that is set: the same eight stars whatever it is, so not even its length
437
+ # says anything about it.
438
+ SECRET_SHOWN = "********"
439
+
440
+ # Said beside an endpoint set without a key: allowed, the user's choice, but advised.
441
+ LOCAL_KEY_ADVICE = "No key: the run is sent the word " + repr(MODEL_KEY_PLACEHOLDER) + ". Fine for a server that checks none, but a key is advised."
442
+
443
+
444
+ def state(var: Variable, file_values: Dict[str, str]) -> dict:
445
+ """What the page shows: where the value comes from, never a secret."""
446
+ value = os.environ.get(var.name)
447
+ source = "unset" if value is None else ("file" if var.name in LOADED_KEYS else "environment")
448
+ shown = "" if value is None else (SECRET_SHOWN if var.secret else endpoint_url_shown(value) if var.kind == "endpoint" else value)
449
+ # an endpoint's URL saved or set before it was shown: said, and not saved again until fixed
450
+ problem = endpoint_url_problem(value) if var.kind == "endpoint" and value else None
451
+ return {"name": var.name, "group": var.group, "purpose": var.purpose, "kind": var.kind, "secret": var.secret,
452
+ "example": var.example, "source": source, "shown": shown,
453
+ "shadowed": source == "environment" and var.name in file_values and file_values[var.name] != value,
454
+ "advice": LOCAL_KEY_ADVICE if var.kind == "localkey" and value is None and os.environ.get(var.name[:-3] + "URL") else "",
455
+ "problem": f"This URL must be {problem}." if problem else ""}
456
+
457
+
458
+ def states(path: str) -> List[dict]:
459
+ """Every variable's state as the process environment holds it, the file
460
+ read only to say which values it shadows: a key written by hand after
461
+ start reads as before until ``refresh`` (``sync``, before a request) applies it."""
462
+ fv = in_file(path)
463
+ return ([state(v, fv) for v in VARIABLES] + [state(agent_variable(n), fv) for n in agent_key_names()]
464
+ + [state(endpoint_variable(n), fv) for i in endpoint_ids() for n in endpoint_settings(i)])
465
+
466
+
467
+ def save(path: str, name: str, value: Optional[str]) -> None:
468
+ """One variable: see ``save_many``."""
469
+ save_many(path, {name: value})
470
+
471
+
472
+ def save_many(path: str, changes: Dict[str, Optional[str]]) -> None:
473
+ """Set (or, with None, remove) variables in the file, keeping every
474
+ other line as it was, and apply each change to the process environment
475
+ unless the environment already held that variable. Every value is
476
+ validated before any is written - one bad value and nothing changes;
477
+ the file is written whole to a temporary name beside the target, then
478
+ renamed."""
479
+ problems = []
480
+ listed = changes.get("AGENT_KEYS") if "AGENT_KEYS" in changes else os.environ.get("AGENT_KEYS", "")
481
+ agents = set() if listed is None or _agent_key_names(listed) else {n.strip() for n in listed.split(",") if n.strip()}
482
+ agents |= set(agent_key_names()) # a key being removed with its name is still one this file may hold
483
+ for name, value in changes.items():
484
+ var = BY_NAME.get(name) or endpoint_variable(name) or (agent_variable(name) if name in agents else None)
485
+ if var is None:
486
+ problems.append(f"{name} is not a setting this application reads.")
487
+ elif value is not None:
488
+ why = var.validate(value)
489
+ if why:
490
+ problems.append(f"{name} must be {why}.")
491
+ if problems:
492
+ raise ValueError(" ".join(problems))
493
+ lines, done = [], set()
494
+ for line, key, _ in parse(path):
495
+ if key in changes:
496
+ value = changes[key]
497
+ if value is not None and key not in done:
498
+ lines.append(f"{key}={value}")
499
+ done.add(key) # a duplicate line is dropped rather than kept
500
+ continue
501
+ lines.append(line)
502
+ for name, value in changes.items():
503
+ if value is not None and name not in done:
504
+ lines.append(f"{name}={value}")
505
+ while lines and not lines[-1].strip():
506
+ lines.pop()
507
+ _write_lines(path, lines)
508
+ for name, value in changes.items():
509
+ if name in os.environ and name not in LOADED_KEYS:
510
+ continue # the process environment holds it; the file is for the next start
511
+ if value is None:
512
+ os.environ.pop(name, None)
513
+ LOADED_KEYS.discard(name)
514
+ else:
515
+ os.environ[name] = value
516
+ LOADED_KEYS.add(name)
517
+ _FROM[name] = os.path.abspath(path)
518
+
519
+
520
+ def example() -> str:
521
+ """``app/.env.example``: every variable, its purpose and an example,
522
+ none of them set."""
523
+ out = ["# Environment variable reference. Configure Nullius through Settings,",
524
+ "# which saves to your user settings.json, or export variables for automation.",
525
+ "# Existing app/.env files can be imported with nullius migrate --from CHECKOUT.",
526
+ "# Process environment variables win over saved settings. Never commit secrets.",
527
+ ""]
528
+ out.append("# Start-up options, not on the Settings page:")
529
+ for name, what in STARTUP.items():
530
+ out.append(f"# {name}: {what}")
531
+ out.append("")
532
+ for g in GROUPS:
533
+ if not any(v.group == g for v in VARIABLES):
534
+ continue # a group of names read from elsewhere (the agent keys): none to list here
535
+ out.append(f"# --- {g} ---")
536
+ for v in VARIABLES:
537
+ if v.group != g:
538
+ continue
539
+ out.append(f"# {v.purpose}")
540
+ if v.example:
541
+ out.append(f"# e.g. {v.example}")
542
+ out.append(f"{v.name}=")
543
+ out.append("")
544
+ return "\n".join(out)
@@ -0,0 +1,15 @@
1
+ """The Nullius standard: benchmarks, agents and capabilities, matched by ports.
2
+
3
+ A benchmark *needs* ports, an agent or a capability *offers* them, and a pair fits
4
+ when every port the benchmark needs is offered and proven (``match``). A port
5
+ is a versioned contract (``ports``) with a conformance kit, a tiny benchmark
6
+ run against the agent or capability before its claim is written down
7
+ (``conform``). The definitions are files in the catalogue (``definitions``).
8
+
9
+ A run (``run``) sets each unit up with the benchmark's driver (``jsonl``),
10
+ drives the agent over ACP (``acp``) with the benchmark's environment as MCP
11
+ servers, sends every model call through a relay that holds the key and records
12
+ the call (``relay``), scores the unit with the driver, and writes a job in
13
+ Harbor's shape (``job``) that the rest of the application reads unchanged.
14
+ ``docs/standard.md`` is the design.
15
+ """