nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/settings.py
ADDED
|
@@ -0,0 +1,544 @@
|
|
|
1
|
+
"""The application's settings: every environment variable it reads, in one
|
|
2
|
+
table, and the one file it writes them to.
|
|
3
|
+
|
|
4
|
+
The user configuration ``settings.json`` is loaded at startup (``load``) with the rule the application has
|
|
5
|
+
always had — a variable already in the process environment wins over the
|
|
6
|
+
file — and read again (``refresh``) whenever the file has changed on disk
|
|
7
|
+
(``sync``, which the application calls before each request), so a file
|
|
8
|
+
edited by hand while the application runs counts from the next request,
|
|
9
|
+
without a restart; the Settings page edits
|
|
10
|
+
the same file (``save``) and names the keys in it that nothing reads (``unread``). What the page
|
|
11
|
+
shows for each variable is its *state*, never a secret's value: ``set`` from
|
|
12
|
+
the environment or from the file, or ``not set``, with a fixed mask and nothing else. Values are validated per variable before a byte is written;
|
|
13
|
+
the file is written whole, to a temporary name first, with mode 0600; a
|
|
14
|
+
variable the process environment holds is shown read-only, because the file
|
|
15
|
+
could not override it anyway.
|
|
16
|
+
|
|
17
|
+
``example`` renders ``app/.env.example`` from the same table, so the two
|
|
18
|
+
cannot drift (a test compares them).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
import json
|
|
24
|
+
import re
|
|
25
|
+
import tempfile
|
|
26
|
+
import urllib.parse
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
from typing import Callable, Dict, List, Optional, Tuple
|
|
29
|
+
|
|
30
|
+
_PATH_OK = re.compile(r"^[^\x00-\x1f]{1,400}$")
|
|
31
|
+
_URL_OK = re.compile(r"^https?://[^\s/?#]+(?:[/?#]\S*)?$")
|
|
32
|
+
_REPO_OK = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
|
|
33
|
+
_NAME_OK = re.compile(r"^[A-Za-z0-9._-]{1,64}$")
|
|
34
|
+
_SECRET_OK = re.compile(r"^[!-~]+$") # printable ASCII, no spaces
|
|
35
|
+
_LOCAL_KEY_OK = re.compile(r"^[!-~]+$") # the same, any length: a local server's key is often a word ("ollama", "EMPTY")
|
|
36
|
+
_TEXT_OK = re.compile(r"^[^\x00-\x1f]{1,400}$")
|
|
37
|
+
_MODEL_OK = re.compile(r"^~?[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,159}$") # an OpenRouter model id: [~]vendor/model[:variant]
|
|
38
|
+
_S1_OK = re.compile(r"^(typesafe|openrouter)$")
|
|
39
|
+
_LLM_JUDGE_OK = re.compile(r"^(openrouter|openai|local(:[A-Z0-9]+(_[A-Z0-9]+)*)?)$")
|
|
40
|
+
_ENVNAME_OK = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
|
41
|
+
_BOOL = {"0", "1", "true", "false", "yes", "no"}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _check(pattern: re.Pattern, what: str) -> Callable[[str], Optional[str]]:
|
|
45
|
+
return lambda v: None if pattern.match(v) else what
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _paths(v: str) -> Optional[str]:
|
|
49
|
+
return None if v and all(_PATH_OK.match(p) for p in v.split(os.pathsep)) else \
|
|
50
|
+
f"one or more paths separated by {os.pathsep!r}, without control characters"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# Names an agent key may not take: the run's own, the shell's, and the ones that steer a process
|
|
54
|
+
_NOT_AGENT_KEYS = {"API_TOKEN", "OPENAI_API_KEY", "LLM_API_KEY", "API_URL", "OPENAI_BASE_URL", "LLM_BASE_URL", "API_MODEL_NAME",
|
|
55
|
+
"PATH", "HOME", "USER", "SHELL", "PWD", "LANG", "TERM", "TMPDIR", "PYTHONPATH", "PYTHONHOME",
|
|
56
|
+
"LD_PRELOAD", "LD_LIBRARY_PATH", "SSH_AUTH_SOCK"}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def reserved_name(n: str) -> bool:
|
|
60
|
+
"""A name that is already something else's: a setting, a start-up option,
|
|
61
|
+
another spelling of one, the run's own or the shell's. An agent key or a
|
|
62
|
+
benchmark's secret may not take it."""
|
|
63
|
+
return n in BY_NAME or n in STARTUP or n in INSTEAD or n in _NOT_AGENT_KEYS or n.startswith("NULLIUS_") or bool(_ENDPOINT_VAR.match(n))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _agent_key_names(v: str) -> Optional[str]:
|
|
67
|
+
names = [n.strip() for n in v.split(",") if n.strip()]
|
|
68
|
+
for n in names:
|
|
69
|
+
if not _ENVNAME_OK.match(n):
|
|
70
|
+
return f"names in capitals, digits and underscores ({n!r} is not)"
|
|
71
|
+
if reserved_name(n):
|
|
72
|
+
return f"names of their own ({n} is already the name of something else)"
|
|
73
|
+
if len(set(names)) != len(names):
|
|
74
|
+
return "names, each once"
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# Query parameters that carry a credential, whatever the service calls them
|
|
79
|
+
_CREDENTIAL_PARAM = re.compile(r"^(key|api[_-]?key|token|access[_-]?token|secret|password|auth|authorization|sig|signature)$", re.I)
|
|
80
|
+
_PUT_KEY = "put the key in the Key field"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def endpoint_url_problem(v: str) -> Optional[str]:
|
|
84
|
+
"""Why ``v`` cannot be an endpoint's URL, or None: an http(s) URL with a host, and no
|
|
85
|
+
credential in it - no user or password, no query parameter named like a key - since
|
|
86
|
+
the page shows it."""
|
|
87
|
+
try:
|
|
88
|
+
p = urllib.parse.urlsplit(v)
|
|
89
|
+
p.port
|
|
90
|
+
except ValueError:
|
|
91
|
+
return "an http(s) URL with a host"
|
|
92
|
+
if p.scheme not in ("http", "https") or not p.hostname or any(c.isspace() or ord(c) < 32 for c in v):
|
|
93
|
+
return "an http(s) URL with a host"
|
|
94
|
+
if "@" in p.netloc:
|
|
95
|
+
return f"a URL without a user or password in it: {_PUT_KEY}"
|
|
96
|
+
if any(_CREDENTIAL_PARAM.match(k) for k, _ in urllib.parse.parse_qsl(p.query, keep_blank_values=True)):
|
|
97
|
+
return f"a URL without a key in its query: {_PUT_KEY}"
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def endpoint_url_shown(v: str) -> str:
|
|
102
|
+
"""An endpoint's URL as the page shows it: as it is, or, when it holds a credential,
|
|
103
|
+
with the user, the password and each key-like query value hidden."""
|
|
104
|
+
if not endpoint_url_problem(v):
|
|
105
|
+
return v
|
|
106
|
+
try:
|
|
107
|
+
p = urllib.parse.urlsplit(v)
|
|
108
|
+
except ValueError:
|
|
109
|
+
return SECRET_SHOWN
|
|
110
|
+
netloc = ("[REDACTED]@" if "@" in p.netloc else "") + p.netloc.rpartition("@")[2]
|
|
111
|
+
query = "&".join(f"{k}=[REDACTED]" if _CREDENTIAL_PARAM.match(k) else f"{k}={w}"
|
|
112
|
+
for k, w in urllib.parse.parse_qsl(p.query, keep_blank_values=True))
|
|
113
|
+
return urllib.parse.urlunsplit((p.scheme, netloc, p.path, query, p.fragment))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _hosts(v: str) -> Optional[str]:
|
|
117
|
+
from nullius import hosts
|
|
118
|
+
return hosts.validate_list(v)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _bool(v: str) -> Optional[str]:
|
|
122
|
+
return None if v.lower() in _BOOL else "0, 1, true, false, yes or no"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass(frozen=True)
|
|
126
|
+
class Variable:
|
|
127
|
+
name: str
|
|
128
|
+
group: str
|
|
129
|
+
purpose: str
|
|
130
|
+
kind: str # secret | localkey | endpoint | path | paths | url | repo | name | bool | text | envname | hosts | model | llmprovider | s1provider | envnames
|
|
131
|
+
example: str = "" # shown as a placeholder, written to .env.example as a comment
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def secret(self) -> bool:
|
|
135
|
+
return self.kind in ("secret", "localkey")
|
|
136
|
+
|
|
137
|
+
def validate(self, value: str) -> Optional[str]:
|
|
138
|
+
"""None when acceptable, else what the value should have been."""
|
|
139
|
+
if "\n" in value or "\r" in value:
|
|
140
|
+
return "a single line"
|
|
141
|
+
if value != value.strip():
|
|
142
|
+
return "no leading or trailing whitespace"
|
|
143
|
+
return {
|
|
144
|
+
"secret": _check(_SECRET_OK, "a nonempty single-line value without spaces"),
|
|
145
|
+
"localkey": _check(_LOCAL_KEY_OK, "a nonempty single-line value without spaces"),
|
|
146
|
+
"path": _check(_PATH_OK, "a path without control characters"),
|
|
147
|
+
"paths": _paths,
|
|
148
|
+
"url": _check(_URL_OK, "an http(s) URL"),
|
|
149
|
+
"endpoint": endpoint_url_problem,
|
|
150
|
+
"repo": _check(_REPO_OK, "owner/name"),
|
|
151
|
+
"name": _check(_NAME_OK, "letters, digits, dots, dashes or underscores (64 at most)"),
|
|
152
|
+
"bool": _bool,
|
|
153
|
+
"text": _check(_TEXT_OK, "a single line without control characters"),
|
|
154
|
+
"envname": _check(_ENVNAME_OK, "an environment variable name (capitals, digits, underscores)"),
|
|
155
|
+
"hosts": _hosts,
|
|
156
|
+
"model": _check(_MODEL_OK, "a model id as OpenRouter lists it, vendor/model"),
|
|
157
|
+
"llmprovider": _check(_LLM_JUDGE_OK, "openrouter, openai, local or local:<endpoint id>"),
|
|
158
|
+
"s1provider": _check(_S1_OK, "typesafe or openrouter"),
|
|
159
|
+
"envnames": _agent_key_names,
|
|
160
|
+
}[self.kind](value)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
GROUPS = ["Model credentials", "Run hosts", "Custom keys"]
|
|
164
|
+
# The key a run on an OpenAI-compatible endpoint is sent with when the
|
|
165
|
+
# endpoint has no key: many such servers check none, but an
|
|
166
|
+
# OpenAI-compatible client refuses to start without one.
|
|
167
|
+
MODEL_KEY_PLACEHOLDER = "local"
|
|
168
|
+
# The OpenAI-compatible endpoint MODEL_API_URL and MODEL_API_KEY hold, by this name.
|
|
169
|
+
DEFAULT_ENDPOINT_NAME = "OpenAI-compatible"
|
|
170
|
+
# More OpenAI-compatible endpoints, each by an id: MODEL_<ID>_NAME, MODEL_<ID>_API_URL and
|
|
171
|
+
# MODEL_<ID>_API_KEY. A recipe records the id, not the name, so a rename keeps its recipes.
|
|
172
|
+
ENDPOINT_ID = re.compile(r"^(?=.{1,32}$)[A-Z0-9]+(?:_[A-Z0-9]+)*$")
|
|
173
|
+
_ENDPOINT_VAR = re.compile(r"^MODEL_([A-Z0-9]+(?:_[A-Z0-9]+)*)_(NAME|API_URL|API_KEY)$")
|
|
174
|
+
ENDPOINT_PARTS = ("NAME", "API_URL", "API_KEY")
|
|
175
|
+
|
|
176
|
+
# What a person sets. Everything else the application reads is a start-up
|
|
177
|
+
# option with a command-line form or a sensible default (STARTUP below) and
|
|
178
|
+
# is not on the page.
|
|
179
|
+
VARIABLES: List[Variable] = [
|
|
180
|
+
Variable("MODEL_API_KEY", GROUPS[0], "The key of the OpenAI-compatible endpoint at MODEL_API_URL: a recipe on that endpoint is sent there with this key. Optional, for a server that checks none: without it the run is sent the word " + repr(MODEL_KEY_PLACEHOLDER) + " as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log.", "localkey", "sk-…"),
|
|
181
|
+
Variable("MODEL_API_URL", GROUPS[0], "The URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More endpoints, each with a name, are MODEL_<ID>_NAME, MODEL_<ID>_API_URL and MODEL_<ID>_API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.", "endpoint", "https://…/v1"),
|
|
182
|
+
Variable("OPENAI_API_KEY", GROUPS[0], "An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own endpoint, https://api.openai.com/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"),
|
|
183
|
+
Variable("OPENROUTER_API_KEY", GROUPS[0], "An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own endpoint, https://openrouter.ai/api/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"),
|
|
184
|
+
Variable("ANTHROPIC_API_KEY", GROUPS[0], "An Anthropic key, for a run of the Nullius standard whose provider is Anthropic: its relay sends the calls to Anthropic's own endpoint, https://api.anthropic.com/v1, with this key. Only the relay holds it - never an agent, a recipe or a log.", "secret"),
|
|
185
|
+
Variable("TYPESAFE_API_KEY", GROUPS[0], "TypeSafe.ai key for the System One judgement on TypeSafe's own endpoint, POST https://api.typesafe.ai/v1/systemone — sent as a bearer header there and nowhere else.", "secret", "ts-…"),
|
|
186
|
+
Variable("SYSTEM_ONE_PROVIDER", GROUPS[0], "Where the System One judgement is asked: typesafe (its own endpoint, with TYPESAFE_API_KEY) or openrouter (https://openrouter.ai/api/v1/systemone, the same contract, with OPENROUTER_API_KEY). Unset: TypeSafe when its key is set, else OpenRouter when its key is.", "s1provider"),
|
|
187
|
+
Variable("SYSTEM_ONE_MODEL", GROUPS[0], "The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest, typesafe/jev-router, jaredpalmer/kev-4b... Unset: the provider's Jev.", "model"),
|
|
188
|
+
Variable("LLM_JUDGE_PROVIDER", GROUPS[0], "The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, local (the endpoint at MODEL_API_URL) or local:<ID> (the endpoint MODEL_<ID>_API_URL), asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.", "llmprovider"),
|
|
189
|
+
Variable("LLM_JUDGE_MODEL", GROUPS[0], "The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.", "model"),
|
|
190
|
+
Variable("AGENT_KEYS", GROUPS[2], "The names of the agent keys this file holds, comma-separated: the keys an agent needs of its own vendor, beside the model's (Cursor's CLI agent a CURSOR_API_KEY, Copilot's a GH_TOKEN). Each is a line of its own in this file. A Harbor run gets every one of them in its environment - over stdin to a host - and a Nullius standard run's never; none is ever written into a recipe or a log.", "envnames"),
|
|
191
|
+
Variable("HOSTS", GROUPS[1], "Machines a recipe can run on, over ssh: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]. Only the name is stored; keys stay where they are.", "hosts", "user@host[:port][:/path/to/key]"),
|
|
192
|
+
]
|
|
193
|
+
# Read by the application but not offered on the page: each has a
|
|
194
|
+
# command-line form, a test-only use, or a default nobody changes. The
|
|
195
|
+
# server's address and port are read once, at start: the running server's
|
|
196
|
+
# own page could not change them.
|
|
197
|
+
STARTUP: Dict[str, str] = {
|
|
198
|
+
"NULLIUS_HOME": "a separate profile with config/, data/ and cache/ directories",
|
|
199
|
+
"NULLIUS_CATALOGUE": "the user catalogue directory (default catalogue/ in the user data directory)",
|
|
200
|
+
"NULLIUS_PRICES": "the model price list file (default prices.json in the user configuration directory); set for a run on a host",
|
|
201
|
+
"APP_HOST": "the address the server binds (default 127.0.0.1, or --host)",
|
|
202
|
+
"APP_PORT": "the port the server listens on (default 5057, or --port)",
|
|
203
|
+
"JOB_ROOTS": "job roots to open at start (or the command-line argument)",
|
|
204
|
+
"WORKSPACES_DIR": "where workspaces are kept (default workspaces/ in the user data directory)",
|
|
205
|
+
"WORKSPACE": "the name the start-up roots open as (or the picker's name field)",
|
|
206
|
+
"DATASET_ROOTS": "local dataset checkouts to look in for task files",
|
|
207
|
+
"DEBUG": "the Werkzeug debugger (or --no-debug)",
|
|
208
|
+
"RUNS_DIR": "where runs started from the builder keep their script, log and job (default runs/ in the user data directory)",
|
|
209
|
+
"HARBOR_PINS": "where the Harbor a recipe pins is built and kept on this machine (default harbor-pins/ beside the runs)",
|
|
210
|
+
"STANDARD_DIR": "where the Nullius standard installs agents, capabilities and benchmark upstreams (default standard/ in the user data directory)",
|
|
211
|
+
"STANDARD_DATA_DIR": "where the Nullius standard keeps the datasets it downloads, which can be gigabytes (default datasets/ in STANDARD_DIR)",
|
|
212
|
+
"SECRETS_DIR": "where each benchmark's licences (Settings → Custom keys) and the Hugging Face token (Settings → Data providers) are kept (default secrets/ in the user configuration directory)",
|
|
213
|
+
}
|
|
214
|
+
BY_NAME: Dict[str, Variable] = {v.name: v for v in VARIABLES}
|
|
215
|
+
LOADED_KEYS: set = set() # variables ``load`` set from the file (so: not held by the process environment)
|
|
216
|
+
_FROM: Dict[str, str] = {} # which file each of them came from, so a refresh of one file never drops another's
|
|
217
|
+
# Names other tools give the model settings, and names this application
|
|
218
|
+
# used before: a file written with one is read by nobody here, and the page
|
|
219
|
+
# says which of its own names was meant - for HOST and PORT, a start-up
|
|
220
|
+
# option, read at start. OPENAI_API_KEY and OPENROUTER_API_KEY are its own:
|
|
221
|
+
# the keys a recipe on OpenAI or on OpenRouter is sent with.
|
|
222
|
+
INSTEAD: Dict[str, str] = {"API_TOKEN": "MODEL_API_KEY", "API_URL": "MODEL_API_URL", "OPENAI_BASE_URL": "MODEL_API_URL",
|
|
223
|
+
"API_KEY_TYPESAFE": "TYPESAFE_API_KEY", "JUDGE_MODEL": "LLM_JUDGE_MODEL", "HOST": "APP_HOST", "PORT": "APP_PORT"}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def parse(path: str) -> List[Tuple[str, Optional[str], Optional[str]]]:
|
|
227
|
+
"""The file as a list of (line, key, value): key and value are None for
|
|
228
|
+
a comment or blank line. Quotes around a value are dropped, as ``load``
|
|
229
|
+
always has."""
|
|
230
|
+
out = []
|
|
231
|
+
if not os.path.isfile(path):
|
|
232
|
+
return out
|
|
233
|
+
with open(path, encoding="utf-8") as fh:
|
|
234
|
+
if str(path).endswith(".json"):
|
|
235
|
+
try:
|
|
236
|
+
values = json.load(fh)
|
|
237
|
+
except (ValueError, UnicodeError) as e:
|
|
238
|
+
raise ValueError("Settings file is not valid JSON; it has not been changed.") from e
|
|
239
|
+
if not isinstance(values, dict) or any(
|
|
240
|
+
not isinstance(k, str) or not _ENVNAME_OK.fullmatch(k) or
|
|
241
|
+
not isinstance(v, str) or any(c in v for c in "\0\r\n")
|
|
242
|
+
for k, v in values.items()
|
|
243
|
+
):
|
|
244
|
+
raise ValueError("Settings must be an object of variable names and single-line strings.")
|
|
245
|
+
return [(f"{key}={value}", key, value) for key, value in values.items()]
|
|
246
|
+
for raw in fh:
|
|
247
|
+
line = raw.rstrip("\n")
|
|
248
|
+
s = line.strip()
|
|
249
|
+
if not s or s.startswith("#") or "=" not in s:
|
|
250
|
+
out.append((line, None, None))
|
|
251
|
+
continue
|
|
252
|
+
key, _, value = s.partition("=")
|
|
253
|
+
out.append((line, key.strip(), value.strip().strip('"').strip("'")))
|
|
254
|
+
return out
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def load(path: str) -> set:
|
|
258
|
+
"""Apply the file to the process environment: existing variables win.
|
|
259
|
+
Returns the names the file supplied."""
|
|
260
|
+
applied = set()
|
|
261
|
+
for _, key, value in parse(path):
|
|
262
|
+
if key and key not in os.environ:
|
|
263
|
+
os.environ[key] = value or ""
|
|
264
|
+
applied.add(key)
|
|
265
|
+
LOADED_KEYS.update(applied)
|
|
266
|
+
_FROM.update((k, os.path.abspath(path)) for k in applied)
|
|
267
|
+
return applied
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def refresh(path: str) -> Dict[str, List[str]]:
|
|
271
|
+
"""The file as it is now, under ``load``'s rule, for a file written or
|
|
272
|
+
edited while the application runs (``sync`` calls it when the file has
|
|
273
|
+
changed): a key the process environment lacks is applied; a key the file
|
|
274
|
+
supplied follows the file - a new value, or gone when its line is; a
|
|
275
|
+
variable the process was started with still wins. A key taken out of
|
|
276
|
+
the environment (a removal saved from the page, a test) is not put back.
|
|
277
|
+
Returns what changed, by name and never by value: the keys ``loaded``,
|
|
278
|
+
``updated`` and ``removed``, each sorted."""
|
|
279
|
+
here = os.path.abspath(path)
|
|
280
|
+
values = in_file(path)
|
|
281
|
+
done: Dict[str, List[str]] = {"loaded": [], "updated": [], "removed": []}
|
|
282
|
+
for key in [k for k in LOADED_KEYS if _FROM.get(k) == here and k not in values]:
|
|
283
|
+
if os.environ.pop(key, None) is not None:
|
|
284
|
+
done["removed"].append(key)
|
|
285
|
+
LOADED_KEYS.discard(key)
|
|
286
|
+
for key, value in values.items():
|
|
287
|
+
if key in LOADED_KEYS:
|
|
288
|
+
if key in os.environ and os.environ[key] != value:
|
|
289
|
+
os.environ[key] = value
|
|
290
|
+
done["updated"].append(key)
|
|
291
|
+
elif key not in os.environ:
|
|
292
|
+
os.environ[key] = value
|
|
293
|
+
LOADED_KEYS.add(key)
|
|
294
|
+
_FROM[key] = here
|
|
295
|
+
done["loaded"].append(key)
|
|
296
|
+
return {k: sorted(v) for k, v in done.items()}
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
_SEEN: Dict[str, Optional[tuple]] = {} # the file as last read by ``sync``: its stat, or None when it was absent
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def sync(path: str) -> Dict[str, List[str]]:
|
|
303
|
+
"""``refresh`` when the file is not as it was last read - another
|
|
304
|
+
size, time or inode, or come or gone - and nothing otherwise, so a
|
|
305
|
+
request costs one stat. Returns what ``refresh`` returns, every list
|
|
306
|
+
empty when the file was as read."""
|
|
307
|
+
here = os.path.abspath(path)
|
|
308
|
+
try:
|
|
309
|
+
st = os.stat(here)
|
|
310
|
+
now: Optional[tuple] = (st.st_mtime_ns, st.st_size, st.st_ino)
|
|
311
|
+
except OSError:
|
|
312
|
+
now = None
|
|
313
|
+
if here in _SEEN and _SEEN[here] == now:
|
|
314
|
+
return {"loaded": [], "updated": [], "removed": []}
|
|
315
|
+
done = refresh(path)
|
|
316
|
+
_SEEN[here] = now
|
|
317
|
+
return done
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def in_file(path: str) -> Dict[str, str]:
|
|
321
|
+
return {k: v or "" for _, k, v in parse(path) if k}
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def agent_key_names(env=None) -> List[str]:
|
|
325
|
+
"""The agent keys' names, as AGENT_KEYS lists them; a list that would
|
|
326
|
+
not validate names none."""
|
|
327
|
+
raw = (os.environ if env is None else env).get("AGENT_KEYS", "")
|
|
328
|
+
if _agent_key_names(raw):
|
|
329
|
+
return []
|
|
330
|
+
return [n.strip() for n in raw.split(",") if n.strip()]
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def agent_key_values(env=None) -> Dict[str, str]:
|
|
334
|
+
"""Each agent key that holds a value: what a Harbor run is sent."""
|
|
335
|
+
env = os.environ if env is None else env
|
|
336
|
+
return {n: env[n] for n in agent_key_names(env) if env.get(n)}
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def endpoint_ids(env=None) -> List[str]:
|
|
340
|
+
"""The ids of the named OpenAI-compatible endpoints: every id one of whose settings is set."""
|
|
341
|
+
env = os.environ if env is None else env
|
|
342
|
+
return sorted({m.group(1) for m in map(_ENDPOINT_VAR.match, env) if m and ENDPOINT_ID.match(m.group(1))})
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def endpoint_settings(i: str) -> List[str]:
|
|
346
|
+
"""The settings of the endpoint ``i``: its name, URL and key."""
|
|
347
|
+
return [f"MODEL_{i}_{part}" for part in ENDPOINT_PARTS]
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def endpoint_names(env=None) -> List[str]:
|
|
351
|
+
"""Every setting of a named endpoint that ``env`` holds: what a run never inherits."""
|
|
352
|
+
return sorted(n for n in (os.environ if env is None else env) if _ENDPOINT_VAR.match(n))
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def endpoint_variable(name: str) -> Optional[Variable]:
|
|
356
|
+
"""A named endpoint's setting as the page and ``save_many`` treat it, or None for another name."""
|
|
357
|
+
m = _ENDPOINT_VAR.match(name)
|
|
358
|
+
if not m or not ENDPOINT_ID.match(m.group(1)):
|
|
359
|
+
return None
|
|
360
|
+
what, kind = {"NAME": ("name", "text"), "API_URL": ("URL", "endpoint"), "API_KEY": ("key, optional", "localkey")}[m.group(2)]
|
|
361
|
+
return Variable(name, GROUPS[0], f"The {what} of the OpenAI-compatible endpoint {m.group(1)}.", kind)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def new_endpoint_id(name: str, taken) -> str:
|
|
365
|
+
"""An id for a new endpoint, from its name: capitals, digits and underscores, not one of ``taken``."""
|
|
366
|
+
base = re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")[:28].strip("_") or "ENDPOINT"
|
|
367
|
+
n, out = 1, base
|
|
368
|
+
while out in taken:
|
|
369
|
+
n += 1
|
|
370
|
+
out = f"{base}_{n}"
|
|
371
|
+
return out
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def agent_variable(name: str) -> Variable:
|
|
375
|
+
"""An agent key as the page and ``save_many`` treat it: a secret of its own."""
|
|
376
|
+
return Variable(name, GROUPS[2], f"An agent key, {name}: sent to Harbor runs, to the agent that reads it.", "secret")
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _write_lines(path: str, lines: List[str]) -> None:
|
|
380
|
+
"""Atomically replace one owner-only settings file; never print values."""
|
|
381
|
+
directory = os.path.dirname(os.path.abspath(path))
|
|
382
|
+
os.makedirs(directory, mode=0o700, exist_ok=True)
|
|
383
|
+
fd, tmp = tempfile.mkstemp(prefix=".settings-", dir=directory)
|
|
384
|
+
try:
|
|
385
|
+
os.fchmod(fd, 0o600)
|
|
386
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
387
|
+
if str(path).endswith(".json"):
|
|
388
|
+
values = {line.partition("=")[0]: line.partition("=")[2]
|
|
389
|
+
for line in lines if "=" in line and not line.lstrip().startswith("#")}
|
|
390
|
+
json.dump(values, fh, indent=2, ensure_ascii=False)
|
|
391
|
+
fh.write("\n")
|
|
392
|
+
else:
|
|
393
|
+
fh.write("\n".join(lines) + ("\n" if lines else ""))
|
|
394
|
+
os.replace(tmp, path)
|
|
395
|
+
except BaseException:
|
|
396
|
+
try:
|
|
397
|
+
os.unlink(tmp)
|
|
398
|
+
except OSError:
|
|
399
|
+
pass
|
|
400
|
+
raise
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def take(path: str, name: str) -> Optional[str]:
|
|
404
|
+
"""Move a line out of the file: ``name``'s value, its line removed and,
|
|
405
|
+
when the file put it there, its variable gone from the process
|
|
406
|
+
environment; None when the file has no such line. Only for a name no
|
|
407
|
+
setting reads (``unread``) - a benchmark's licence from before licences
|
|
408
|
+
were kept per benchmark (``nullius.secrets``), which its page moves
|
|
409
|
+
there on the user's word."""
|
|
410
|
+
if name in BY_NAME or name in STARTUP or name in set(agent_key_names()):
|
|
411
|
+
raise ValueError(f"{name} is a setting of its own; it is not moved")
|
|
412
|
+
value = in_file(path).get(name)
|
|
413
|
+
if value is None:
|
|
414
|
+
return None
|
|
415
|
+
lines = [line for line, key, _ in parse(path) if key != name]
|
|
416
|
+
while lines and not lines[-1].strip():
|
|
417
|
+
lines.pop()
|
|
418
|
+
_write_lines(path, lines)
|
|
419
|
+
if name in LOADED_KEYS:
|
|
420
|
+
os.environ.pop(name, None)
|
|
421
|
+
LOADED_KEYS.discard(name)
|
|
422
|
+
return value
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def unread(path: str) -> List[dict]:
|
|
426
|
+
"""Keys in the file that no setting or start-up option reads, each with
|
|
427
|
+
the page's own name when it is a known other spelling (``API_TOKEN``):
|
|
428
|
+
the file applies them to the environment, but nothing here reads them.
|
|
429
|
+
A run inherits those its launcher does not withhold
|
|
430
|
+
(``execution.withheld``: every other spelling of a key among them)."""
|
|
431
|
+
keys = set(agent_key_names()) | set(agent_key_names(in_file(path)))
|
|
432
|
+
return [{"name": k, "instead": INSTEAD.get(k, "")} for k in in_file(path)
|
|
433
|
+
if k not in BY_NAME and k not in STARTUP and k not in keys and not endpoint_variable(k)]
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# What a page shows of a secret that is set: the same eight stars whatever it is, so not even its length
|
|
437
|
+
# says anything about it.
|
|
438
|
+
SECRET_SHOWN = "********"
|
|
439
|
+
|
|
440
|
+
# Said beside an endpoint set without a key: allowed, the user's choice, but advised.
|
|
441
|
+
LOCAL_KEY_ADVICE = "No key: the run is sent the word " + repr(MODEL_KEY_PLACEHOLDER) + ". Fine for a server that checks none, but a key is advised."
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def state(var: Variable, file_values: Dict[str, str]) -> dict:
|
|
445
|
+
"""What the page shows: where the value comes from, never a secret."""
|
|
446
|
+
value = os.environ.get(var.name)
|
|
447
|
+
source = "unset" if value is None else ("file" if var.name in LOADED_KEYS else "environment")
|
|
448
|
+
shown = "" if value is None else (SECRET_SHOWN if var.secret else endpoint_url_shown(value) if var.kind == "endpoint" else value)
|
|
449
|
+
# an endpoint's URL saved or set before it was shown: said, and not saved again until fixed
|
|
450
|
+
problem = endpoint_url_problem(value) if var.kind == "endpoint" and value else None
|
|
451
|
+
return {"name": var.name, "group": var.group, "purpose": var.purpose, "kind": var.kind, "secret": var.secret,
|
|
452
|
+
"example": var.example, "source": source, "shown": shown,
|
|
453
|
+
"shadowed": source == "environment" and var.name in file_values and file_values[var.name] != value,
|
|
454
|
+
"advice": LOCAL_KEY_ADVICE if var.kind == "localkey" and value is None and os.environ.get(var.name[:-3] + "URL") else "",
|
|
455
|
+
"problem": f"This URL must be {problem}." if problem else ""}
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def states(path: str) -> List[dict]:
|
|
459
|
+
"""Every variable's state as the process environment holds it, the file
|
|
460
|
+
read only to say which values it shadows: a key written by hand after
|
|
461
|
+
start reads as before until ``refresh`` (``sync``, before a request) applies it."""
|
|
462
|
+
fv = in_file(path)
|
|
463
|
+
return ([state(v, fv) for v in VARIABLES] + [state(agent_variable(n), fv) for n in agent_key_names()]
|
|
464
|
+
+ [state(endpoint_variable(n), fv) for i in endpoint_ids() for n in endpoint_settings(i)])
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def save(path: str, name: str, value: Optional[str]) -> None:
|
|
468
|
+
"""One variable: see ``save_many``."""
|
|
469
|
+
save_many(path, {name: value})
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def save_many(path: str, changes: Dict[str, Optional[str]]) -> None:
|
|
473
|
+
"""Set (or, with None, remove) variables in the file, keeping every
|
|
474
|
+
other line as it was, and apply each change to the process environment
|
|
475
|
+
unless the environment already held that variable. Every value is
|
|
476
|
+
validated before any is written - one bad value and nothing changes;
|
|
477
|
+
the file is written whole to a temporary name beside the target, then
|
|
478
|
+
renamed."""
|
|
479
|
+
problems = []
|
|
480
|
+
listed = changes.get("AGENT_KEYS") if "AGENT_KEYS" in changes else os.environ.get("AGENT_KEYS", "")
|
|
481
|
+
agents = set() if listed is None or _agent_key_names(listed) else {n.strip() for n in listed.split(",") if n.strip()}
|
|
482
|
+
agents |= set(agent_key_names()) # a key being removed with its name is still one this file may hold
|
|
483
|
+
for name, value in changes.items():
|
|
484
|
+
var = BY_NAME.get(name) or endpoint_variable(name) or (agent_variable(name) if name in agents else None)
|
|
485
|
+
if var is None:
|
|
486
|
+
problems.append(f"{name} is not a setting this application reads.")
|
|
487
|
+
elif value is not None:
|
|
488
|
+
why = var.validate(value)
|
|
489
|
+
if why:
|
|
490
|
+
problems.append(f"{name} must be {why}.")
|
|
491
|
+
if problems:
|
|
492
|
+
raise ValueError(" ".join(problems))
|
|
493
|
+
lines, done = [], set()
|
|
494
|
+
for line, key, _ in parse(path):
|
|
495
|
+
if key in changes:
|
|
496
|
+
value = changes[key]
|
|
497
|
+
if value is not None and key not in done:
|
|
498
|
+
lines.append(f"{key}={value}")
|
|
499
|
+
done.add(key) # a duplicate line is dropped rather than kept
|
|
500
|
+
continue
|
|
501
|
+
lines.append(line)
|
|
502
|
+
for name, value in changes.items():
|
|
503
|
+
if value is not None and name not in done:
|
|
504
|
+
lines.append(f"{name}={value}")
|
|
505
|
+
while lines and not lines[-1].strip():
|
|
506
|
+
lines.pop()
|
|
507
|
+
_write_lines(path, lines)
|
|
508
|
+
for name, value in changes.items():
|
|
509
|
+
if name in os.environ and name not in LOADED_KEYS:
|
|
510
|
+
continue # the process environment holds it; the file is for the next start
|
|
511
|
+
if value is None:
|
|
512
|
+
os.environ.pop(name, None)
|
|
513
|
+
LOADED_KEYS.discard(name)
|
|
514
|
+
else:
|
|
515
|
+
os.environ[name] = value
|
|
516
|
+
LOADED_KEYS.add(name)
|
|
517
|
+
_FROM[name] = os.path.abspath(path)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def example() -> str:
|
|
521
|
+
"""``app/.env.example``: every variable, its purpose and an example,
|
|
522
|
+
none of them set."""
|
|
523
|
+
out = ["# Environment variable reference. Configure Nullius through Settings,",
|
|
524
|
+
"# which saves to your user settings.json, or export variables for automation.",
|
|
525
|
+
"# Existing app/.env files can be imported with nullius migrate --from CHECKOUT.",
|
|
526
|
+
"# Process environment variables win over saved settings. Never commit secrets.",
|
|
527
|
+
""]
|
|
528
|
+
out.append("# Start-up options, not on the Settings page:")
|
|
529
|
+
for name, what in STARTUP.items():
|
|
530
|
+
out.append(f"# {name}: {what}")
|
|
531
|
+
out.append("")
|
|
532
|
+
for g in GROUPS:
|
|
533
|
+
if not any(v.group == g for v in VARIABLES):
|
|
534
|
+
continue # a group of names read from elsewhere (the agent keys): none to list here
|
|
535
|
+
out.append(f"# --- {g} ---")
|
|
536
|
+
for v in VARIABLES:
|
|
537
|
+
if v.group != g:
|
|
538
|
+
continue
|
|
539
|
+
out.append(f"# {v.purpose}")
|
|
540
|
+
if v.example:
|
|
541
|
+
out.append(f"# e.g. {v.example}")
|
|
542
|
+
out.append(f"{v.name}=")
|
|
543
|
+
out.append("")
|
|
544
|
+
return "\n".join(out)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""The Nullius standard: benchmarks, agents and capabilities, matched by ports.
|
|
2
|
+
|
|
3
|
+
A benchmark *needs* ports, an agent or a capability *offers* them, and a pair fits
|
|
4
|
+
when every port the benchmark needs is offered and proven (``match``). A port
|
|
5
|
+
is a versioned contract (``ports``) with a conformance kit, a tiny benchmark
|
|
6
|
+
run against the agent or capability before its claim is written down
|
|
7
|
+
(``conform``). The definitions are files in the catalogue (``definitions``).
|
|
8
|
+
|
|
9
|
+
A run (``run``) sets each unit up with the benchmark's driver (``jsonl``),
|
|
10
|
+
drives the agent over ACP (``acp``) with the benchmark's environment as MCP
|
|
11
|
+
servers, sends every model call through a relay that holds the key and records
|
|
12
|
+
the call (``relay``), scores the unit with the driver, and writes a job in
|
|
13
|
+
Harbor's shape (``job``) that the rest of the application reads unchanged.
|
|
14
|
+
``docs/standard.md`` is the design.
|
|
15
|
+
"""
|