nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/progress.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"""What a run is doing, one line per stage: read by a person in a terminal,
|
|
2
|
+
and by the run's page.
|
|
3
|
+
|
|
4
|
+
A run spends minutes before its first unit - a host connected to, ``uv``
|
|
5
|
+
installed, this repository's ``app/`` sent, the docker base image built, an
|
|
6
|
+
agent installed, a dataset downloaded - and a log that only says "running"
|
|
7
|
+
looks stuck all that time. Each of those says where it is in one line::
|
|
8
|
+
|
|
9
|
+
[nullius] stage=image state=start detail="building nullius/base:229ac6f71272 (first time on this machine)" at=2026-10-12T14:03:41Z
|
|
10
|
+
[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...
|
|
11
|
+
[nullius] stage=unit i=3 of=10 unit=HumanEval/2 state=done outcome=passed reward=1.0 at=...
|
|
12
|
+
|
|
13
|
+
``key=value`` words after the prefix; a value with a space, a quote or an
|
|
14
|
+
``=`` is a JSON string. ``state`` is ``start``, ``done``, ``cached`` (there
|
|
15
|
+
already: nothing to do), ``failed`` or ``progress`` (a download's bytes so
|
|
16
|
+
far); ``at`` is when, in UTC, on the machine that said it. ``stage=plan``
|
|
17
|
+
names the stages to come (``stages=image,install,units``), so a page can
|
|
18
|
+
show them waiting before they start.
|
|
19
|
+
|
|
20
|
+
The standard's ``cli`` writes them on stderr - its stdout stays the job's
|
|
21
|
+
path and the units, as before - and into the job's ``progress.jsonl``, one
|
|
22
|
+
JSON object per line, so a mirrored job carries them. ``execution`` writes
|
|
23
|
+
its own for what it does on a host before the run starts (connect, uv, the
|
|
24
|
+
tasks, app/), into the run's log; and reads every one that comes through the
|
|
25
|
+
part's log, so the page follows a run on another machine as it goes
|
|
26
|
+
(``fold``).
|
|
27
|
+
|
|
28
|
+
Nothing here is a secret: a detail names an image, a package, a file.
|
|
29
|
+
"""
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import contextlib
|
|
33
|
+
import json
|
|
34
|
+
import os
|
|
35
|
+
import re
|
|
36
|
+
import threading
|
|
37
|
+
import time
|
|
38
|
+
from datetime import datetime, timezone
|
|
39
|
+
from typing import IO, Iterable, List, Optional
|
|
40
|
+
|
|
41
|
+
PREFIX = "[nullius]"
|
|
42
|
+
FILE = "progress.jsonl"
|
|
43
|
+
#: where every reporter also writes its lines, unless it was given a stream: the cli sets it to stderr
|
|
44
|
+
ECHO: Optional[IO] = None
|
|
45
|
+
#: seconds between two lines of one download's bytes
|
|
46
|
+
EVERY = 5.0
|
|
47
|
+
|
|
48
|
+
# What the page calls each stage. One not here is shown by its own name.
|
|
49
|
+
LABELS = {"connect": "Connect", "uv": "uv", "tasks": "Send the tasks", "app": "Send app/", "harbor": "Harbor",
|
|
50
|
+
"image": "Base image", "environment": "Environment", "upstream": "Benchmark source", "dataset": "Dataset",
|
|
51
|
+
"driver": "Driver environment", "install": "Install", "runtime": "Agent runtime", "units": "Units"}
|
|
52
|
+
|
|
53
|
+
_WORD = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=("(?:[^"\\]|\\.)*"|\S*)')
|
|
54
|
+
_BARE = re.compile(r'^[^\s"=]+$')
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _value(v) -> str:
|
|
58
|
+
if isinstance(v, bool):
|
|
59
|
+
return "true" if v else "false"
|
|
60
|
+
if isinstance(v, float):
|
|
61
|
+
v = round(v, 3)
|
|
62
|
+
return str(int(v)) if v == int(v) and abs(v) >= 10 else str(v)
|
|
63
|
+
s = str(v)
|
|
64
|
+
return s if _BARE.match(s) else json.dumps(s, ensure_ascii=False)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def stamp(t: Optional[float] = None) -> str:
|
|
68
|
+
return datetime.fromtimestamp(time.time() if t is None else t, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def line(stage: str, state: Optional[str] = None, t: Optional[float] = None, **fields) -> str:
|
|
72
|
+
"""One stage's line: the prefix, ``stage``, the fields in the order given
|
|
73
|
+
(``state`` second), ``at`` last. A field that is None is left out."""
|
|
74
|
+
words = [f"stage={_value(stage)}"] + ([f"state={state}"] if state else [])
|
|
75
|
+
words += [f"{k}={_value(v)}" for k, v in fields.items() if v is not None]
|
|
76
|
+
return " ".join([PREFIX] + words + [f"at={stamp(t)}"])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _plain(s: str):
|
|
80
|
+
if re.fullmatch(r"-?\d+", s):
|
|
81
|
+
return int(s)
|
|
82
|
+
if re.fullmatch(r"-?\d+\.\d*(?:[eE][-+]?\d+)?", s):
|
|
83
|
+
return float(s)
|
|
84
|
+
return {"true": True, "false": False}.get(s, s)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def parse(text: str) -> Optional[dict]:
|
|
88
|
+
"""The event a line says, or None for any other line. ``t`` is ``at`` in
|
|
89
|
+
seconds since the epoch, when it is one."""
|
|
90
|
+
s = text.strip()
|
|
91
|
+
if not s.startswith(PREFIX + " "):
|
|
92
|
+
return None
|
|
93
|
+
ev = {}
|
|
94
|
+
for k, v in _WORD.findall(s[len(PREFIX):]):
|
|
95
|
+
if v.startswith('"'):
|
|
96
|
+
try:
|
|
97
|
+
ev[k] = json.loads(v)
|
|
98
|
+
except ValueError:
|
|
99
|
+
ev[k] = v.strip('"')
|
|
100
|
+
else:
|
|
101
|
+
ev[k] = _plain(v)
|
|
102
|
+
if not isinstance(ev.get("stage"), str) or not ev["stage"]:
|
|
103
|
+
return None
|
|
104
|
+
try:
|
|
105
|
+
ev["t"] = datetime.strptime(str(ev.get("at", "")), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp()
|
|
106
|
+
except ValueError:
|
|
107
|
+
pass
|
|
108
|
+
return ev
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def read(job_dir: str) -> List[dict]:
|
|
112
|
+
"""A job's ``progress.jsonl``: every event it holds, a line that is not
|
|
113
|
+
one left out; [] when there is none."""
|
|
114
|
+
out = []
|
|
115
|
+
try:
|
|
116
|
+
with open(os.path.join(job_dir, FILE), encoding="utf-8", errors="replace") as fh:
|
|
117
|
+
for raw in fh:
|
|
118
|
+
try:
|
|
119
|
+
ev = json.loads(raw)
|
|
120
|
+
except ValueError:
|
|
121
|
+
continue
|
|
122
|
+
if isinstance(ev, dict) and ev.get("stage"):
|
|
123
|
+
out.append(ev)
|
|
124
|
+
except OSError:
|
|
125
|
+
pass
|
|
126
|
+
return out
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def refusal(events: Iterable[dict]) -> Optional[str]:
|
|
130
|
+
"""Why the run was refused before its first unit, when it was: the detail
|
|
131
|
+
of its ``refused`` event; None otherwise."""
|
|
132
|
+
for ev in events:
|
|
133
|
+
if ev.get("stage") == "refused":
|
|
134
|
+
return str(ev.get("detail") or "refused")
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Reporter:
|
|
139
|
+
"""Says each stage: a line on ``echo`` (``ECHO`` when none is given;
|
|
140
|
+
nowhere when that is None too) and an event in ``<job_dir>/progress.jsonl``
|
|
141
|
+
(none without a job). Safe from several threads: units run at once."""
|
|
142
|
+
|
|
143
|
+
def __init__(self, job_dir: Optional[str] = None, echo: Optional[IO] = None):
|
|
144
|
+
self.job_dir = job_dir
|
|
145
|
+
self.echo = echo
|
|
146
|
+
self._lock = threading.Lock()
|
|
147
|
+
|
|
148
|
+
def emit(self, stage: str, state: Optional[str] = None, **fields) -> dict:
|
|
149
|
+
t = time.time()
|
|
150
|
+
text = line(stage, state, t, **fields)
|
|
151
|
+
ev = dict({"stage": stage}, **({"state": state} if state else {}),
|
|
152
|
+
**{k: v for k, v in fields.items() if v is not None}, at=stamp(t), t=round(t, 3))
|
|
153
|
+
with self._lock:
|
|
154
|
+
out = self.echo or ECHO
|
|
155
|
+
if out is not None:
|
|
156
|
+
try:
|
|
157
|
+
out.write(text + "\n")
|
|
158
|
+
out.flush()
|
|
159
|
+
except (OSError, ValueError):
|
|
160
|
+
pass
|
|
161
|
+
if self.job_dir:
|
|
162
|
+
try:
|
|
163
|
+
os.makedirs(self.job_dir, exist_ok=True)
|
|
164
|
+
with open(os.path.join(self.job_dir, FILE), "a", encoding="utf-8") as fh:
|
|
165
|
+
fh.write(json.dumps(ev) + "\n")
|
|
166
|
+
except OSError:
|
|
167
|
+
pass
|
|
168
|
+
return ev
|
|
169
|
+
|
|
170
|
+
@contextlib.contextmanager
|
|
171
|
+
def stage(self, stage: str, *, cached: bool = False, start: Optional[str] = None, **fields):
|
|
172
|
+
"""``with reporter.stage("install", agent="codex@2.2.2"):`` says it
|
|
173
|
+
started, then that it is done with its seconds - or failed, with what
|
|
174
|
+
was raised, which goes on up. ``cached``: there already, said once as
|
|
175
|
+
such. ``start``: the start line's detail (what is being done)."""
|
|
176
|
+
if cached:
|
|
177
|
+
self.emit(stage, "cached", **fields)
|
|
178
|
+
yield
|
|
179
|
+
return
|
|
180
|
+
t0 = time.monotonic()
|
|
181
|
+
self.emit(stage, "start", **dict(fields, **({"detail": start} if start else {})))
|
|
182
|
+
try:
|
|
183
|
+
yield
|
|
184
|
+
except BaseException as e:
|
|
185
|
+
self.emit(stage, "failed", seconds=round(time.monotonic() - t0, 1), **dict(fields, detail=str(e)[:300] or type(e).__name__))
|
|
186
|
+
raise
|
|
187
|
+
self.emit(stage, "done", seconds=round(time.monotonic() - t0, 1), **fields)
|
|
188
|
+
|
|
189
|
+
def download(self, stage: str, **fields):
|
|
190
|
+
"""A callback for a download's ``(done, total)`` bytes, said at most
|
|
191
|
+
every ``EVERY`` seconds - and once more at the end, whatever the time."""
|
|
192
|
+
last = [0.0]
|
|
193
|
+
|
|
194
|
+
def say(done: int, total: Optional[int], final: bool = False):
|
|
195
|
+
now = time.monotonic()
|
|
196
|
+
if final or now - last[0] >= EVERY:
|
|
197
|
+
last[0] = now
|
|
198
|
+
self.emit(stage, "progress", bytes=int(done), total=int(total) if total else None, **fields)
|
|
199
|
+
return say
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _key(ev: dict) -> str:
|
|
203
|
+
"""One stage of the list: its name, and what it is of when there are
|
|
204
|
+
several (an environment's name, the agent installed)."""
|
|
205
|
+
of = ev.get("name") or ev.get("agent") or ""
|
|
206
|
+
return f"{ev['stage']}:{of}" if of else str(ev["stage"])
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def label(ev: dict) -> str:
|
|
210
|
+
base = LABELS.get(str(ev["stage"]), str(ev["stage"]))
|
|
211
|
+
of = ev.get("name") or ev.get("agent")
|
|
212
|
+
return f"{base} {of}" if of else base
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def fold(events: Iterable[dict]) -> dict:
|
|
216
|
+
"""What the page draws, from the events in the order they came:
|
|
217
|
+
|
|
218
|
+
- ``stages``: each stage once, in the order planned (``stage=plan``) or
|
|
219
|
+
first said - ``waiting`` until it starts, then ``running`` (with ``t``,
|
|
220
|
+
when it started), ``done`` or ``cached`` with its ``seconds``, or
|
|
221
|
+
``failed`` with its detail. A planned stage still waiting when a later
|
|
222
|
+
one has started was not needed (a benchmark with nothing to download),
|
|
223
|
+
and is left out.
|
|
224
|
+
- ``units``: ``n`` to run, the ``noun``, how many ended and how: passed,
|
|
225
|
+
failed, scored, errors; ``running`` the ones started and not ended. A
|
|
226
|
+
run of several parts counts every part's: ``n`` grows as each begins.
|
|
227
|
+
- ``current``: the stage running now, else None; ``ended``: ``done``,
|
|
228
|
+
``refused`` (with ``detail``) or None while the run goes on.
|
|
229
|
+
"""
|
|
230
|
+
stages: dict = {}
|
|
231
|
+
units = None
|
|
232
|
+
ended, detail = None, None
|
|
233
|
+
started_units = 0
|
|
234
|
+
for ev in events:
|
|
235
|
+
st = ev.get("stage")
|
|
236
|
+
state = ev.get("state")
|
|
237
|
+
if st == "plan":
|
|
238
|
+
# a run of several parts (each agent, each dataset) says each part's plan: the one after an end goes on
|
|
239
|
+
ended, detail = None, None
|
|
240
|
+
for name in str(ev.get("stages") or "").split(","):
|
|
241
|
+
name = name.strip()
|
|
242
|
+
if name and name not in stages:
|
|
243
|
+
stages[name] = {"key": name, "stage": name, "label": LABELS.get(name, name), "state": "waiting"}
|
|
244
|
+
continue
|
|
245
|
+
if st == "unit":
|
|
246
|
+
units = units or {"n": None, "noun": "units", "done": 0, "passed": 0, "failed": 0, "scored": 0, "errors": 0, "running": 0}
|
|
247
|
+
if state == "start":
|
|
248
|
+
started_units += 1
|
|
249
|
+
elif state == "done":
|
|
250
|
+
units["done"] += 1
|
|
251
|
+
outcome = ev.get("outcome")
|
|
252
|
+
bucket = {"passed": "passed", "failed": "failed", "scored": "scored"}.get(outcome, "errors")
|
|
253
|
+
units[bucket] += 1
|
|
254
|
+
units["running"] = max(0, started_units - units["done"])
|
|
255
|
+
continue
|
|
256
|
+
if st in ("done", "refused"):
|
|
257
|
+
ended, detail = st, ev.get("detail")
|
|
258
|
+
continue
|
|
259
|
+
key = _key(ev)
|
|
260
|
+
if key not in stages:
|
|
261
|
+
# a planned stage, now said with what it is of: the plan's entry becomes it
|
|
262
|
+
if key != st and st in stages and stages[st]["state"] == "waiting":
|
|
263
|
+
stages = {(key if k == st else k): v for k, v in stages.items()}
|
|
264
|
+
stages[key]["key"] = key
|
|
265
|
+
stages.setdefault(key, {"key": key, "stage": st, "label": label(ev), "state": "waiting"})
|
|
266
|
+
row = stages[key]
|
|
267
|
+
row["label"] = label(ev)
|
|
268
|
+
if ev.get("detail") is not None:
|
|
269
|
+
row["detail"] = ev["detail"]
|
|
270
|
+
if state == "start":
|
|
271
|
+
row.update(state="running", t=ev.get("t"))
|
|
272
|
+
elif state in ("done", "cached", "failed"):
|
|
273
|
+
row["state"] = state
|
|
274
|
+
if ev.get("seconds") is not None:
|
|
275
|
+
row["seconds"] = ev["seconds"]
|
|
276
|
+
elif state == "progress":
|
|
277
|
+
if row["state"] == "waiting":
|
|
278
|
+
row.update(state="running", t=ev.get("t"))
|
|
279
|
+
row["bytes"], row["total"] = ev.get("bytes"), ev.get("total")
|
|
280
|
+
if st == "units":
|
|
281
|
+
units = units or {"n": None, "noun": "units", "done": 0, "passed": 0, "failed": 0, "scored": 0, "errors": 0, "running": 0}
|
|
282
|
+
if ev.get("n") is not None and state == "start":
|
|
283
|
+
units["n"] = (units["n"] or 0) + ev["n"] # each part's units: the run's are all of them
|
|
284
|
+
if ev.get("noun"):
|
|
285
|
+
units["noun"] = ev["noun"]
|
|
286
|
+
rows = list(stages.values())
|
|
287
|
+
seen = [i for i, r in enumerate(rows) if r["state"] != "waiting"]
|
|
288
|
+
last = seen[-1] if seen else -1
|
|
289
|
+
rows = [r for i, r in enumerate(rows) if r["state"] != "waiting" or i > last]
|
|
290
|
+
if ended: # a run that ended has nothing running, nor anything still to come
|
|
291
|
+
rows = [dict(r, state="failed") if r["state"] == "running" and ended == "refused" else r
|
|
292
|
+
for r in rows if r["state"] != "waiting"]
|
|
293
|
+
current = next((r for r in rows if r["state"] == "running"), None)
|
|
294
|
+
return {"stages": rows, "units": units, "current": current["key"] if current else None, "ended": ended, "detail": detail}
|
nullius/providers.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
"""Where a run's model calls go, and the builder's steps.
|
|
2
|
+
|
|
3
|
+
The builder's three providers (command-builder-core.js, ``PROVIDERS``), named
|
|
4
|
+
the same here: what each is called, where it is served, the setting its key is
|
|
5
|
+
read from, and what makes it one a run can start on here. The third,
|
|
6
|
+
``custom``, is any number of OpenAI-compatible endpoints: the one at
|
|
7
|
+
MODEL_API_URL, and one per id in Settings (``settings.endpoint_ids``). A
|
|
8
|
+
recipe names one by ``provider: custom`` and its ``endpoint`` id; the builder
|
|
9
|
+
and the judge pickers by a choice, ``custom:<id>`` (``choice``, ``split``). Also the builder's
|
|
10
|
+
step numbers, so an error from a compiled recipe names the step that edits it.
|
|
11
|
+
Settings' Test (``test``) asks a provider with free GETs only, and keeps the last
|
|
12
|
+
result per choice (``tests``, ``remember``) for its card and Review.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import http.client
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import secrets
|
|
20
|
+
import socket
|
|
21
|
+
import ssl
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.parse
|
|
26
|
+
import urllib.request
|
|
27
|
+
from typing import List
|
|
28
|
+
|
|
29
|
+
from nullius import paths, settings
|
|
30
|
+
|
|
31
|
+
# The builder's steps (command-builder-core.js, STEP): an error names the step that edits it.
|
|
32
|
+
STEP = {"benchmark": 0, "runner": 1, "dataset": 2, "setup": 3, "agent": 4, "model": 5, "limits": 6, "where": 7, "review": 8}
|
|
33
|
+
|
|
34
|
+
# Harbor's card on the builder's Runner step: compiled by the builder's own core
|
|
35
|
+
# (static/builder/command-builder-core.js), not by the application.
|
|
36
|
+
HARBOR = {"key": "harbor", "label": "Harbor", "builtin": True,
|
|
37
|
+
"summary": "Task directories in containers, with the oracle and nop bracket."}
|
|
38
|
+
|
|
39
|
+
# A recipe whose harness is neither Harbor nor the standard: a runner of a benchmark's own scripts,
|
|
40
|
+
# no longer offered - those scripts run as the benchmark's Original setup on the Nullius runner.
|
|
41
|
+
GONE = ("The Native runner is no longer offered: the benchmark's own code runs as its Original setup "
|
|
42
|
+
"on the Nullius runner.")
|
|
43
|
+
|
|
44
|
+
PROVIDERS = ("openrouter", "openai", "custom")
|
|
45
|
+
PROVIDER_LABEL = {"openrouter": "OpenRouter", "openai": "OpenAI", "custom": settings.DEFAULT_ENDPOINT_NAME}
|
|
46
|
+
# Where a provider is served: OpenRouter and OpenAI at their own endpoint,
|
|
47
|
+
# fixed; an OpenAI-compatible endpoint has no default and must be named
|
|
48
|
+
# (Settings: MODEL_API_URL, or an endpoint's own).
|
|
49
|
+
PROVIDER_BASE = {"openrouter": "https://openrouter.ai/api/v1", "openai": "https://api.openai.com/v1", "custom": ""}
|
|
50
|
+
# The setting a run's key is read from, per provider (web/execute.py,
|
|
51
|
+
# credentials_for): a key of its own for OpenRouter and for OpenAI, the
|
|
52
|
+
# endpoint's key for an OpenAI-compatible one.
|
|
53
|
+
PROVIDER_KEY = {"openrouter": "OPENROUTER_API_KEY", "openai": "OPENAI_API_KEY", "custom": "MODEL_API_KEY"}
|
|
54
|
+
# What makes a provider one a run can start on here: its key, for OpenRouter
|
|
55
|
+
# and OpenAI; its endpoint, for an OpenAI-compatible one, whose key is optional.
|
|
56
|
+
PROVIDER_NEEDS = dict(PROVIDER_KEY, custom="MODEL_API_URL")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def endpoint_vars(endpoint: str = "") -> tuple:
|
|
60
|
+
"""(URL setting, key setting) of an OpenAI-compatible endpoint: MODEL_API_URL
|
|
61
|
+
and MODEL_API_KEY for none named, else that endpoint's own. An id that is not
|
|
62
|
+
one names settings that are never set, so a run on it never falls back to another."""
|
|
63
|
+
return ("MODEL_API_URL", "MODEL_API_KEY") if not endpoint else (f"MODEL_{endpoint}_API_URL", f"MODEL_{endpoint}_API_KEY")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def choice(provider: str, endpoint: str = "") -> str:
|
|
67
|
+
"""A provider as the builder and the judge pickers offer it: ``custom:<id>`` for a named endpoint."""
|
|
68
|
+
return f"custom:{endpoint}" if provider == "custom" and endpoint else provider
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def split(value: str) -> tuple:
|
|
72
|
+
"""(provider, endpoint id) of a choice; a ``custom:`` with no valid id stays whole, so it is no provider."""
|
|
73
|
+
p, _, e = str(value or "").partition(":")
|
|
74
|
+
return (p, e) if p == "custom" and settings.ENDPOINT_ID.match(e) else (str(value or ""), "")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _with_url(env) -> List[tuple]:
|
|
78
|
+
"""(id, URL) of each OpenAI-compatible endpoint whose URL is set; "" is MODEL_API_URL's."""
|
|
79
|
+
return [(e, env[endpoint_vars(e)[0]]) for e in ("", *settings.endpoint_ids(env)) if env.get(endpoint_vars(e)[0])]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def endpoints(env=os.environ) -> List[dict]:
|
|
83
|
+
"""The OpenAI-compatible endpoints a run can be sent to, each whose URL is set and
|
|
84
|
+
holds no credential (``broken``): the one at MODEL_API_URL first, then the named
|
|
85
|
+
ones by name. Names only, never a value."""
|
|
86
|
+
out = [{"id": e, "name": (env.get(f"MODEL_{e}_NAME") if e else "") or e or settings.DEFAULT_ENDPOINT_NAME}
|
|
87
|
+
for e, url in _with_url(env) if not settings.endpoint_url_problem(url)]
|
|
88
|
+
return sorted(out, key=lambda x: (x["id"] != "", x["name"].casefold()))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def broken(env=os.environ) -> List[str]:
|
|
92
|
+
"""The endpoint choices whose URL is set but is refused (a credential in it, or no
|
|
93
|
+
http(s) URL): never offered, and a recipe on one is told to fix it in Settings."""
|
|
94
|
+
return [choice("custom", e) for e, url in _with_url(env) if settings.endpoint_url_problem(url)]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def label(value: str, env=os.environ) -> str:
|
|
98
|
+
"""What a choice is called: the provider's name, or the endpoint's."""
|
|
99
|
+
p, e = split(value)
|
|
100
|
+
if not e:
|
|
101
|
+
return PROVIDER_LABEL.get(p, p)
|
|
102
|
+
return next((x["name"] for x in endpoints(env) if x["id"] == e), env.get(f"MODEL_{e}_NAME") or e)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def needs(value: str) -> str:
|
|
106
|
+
"""The setting that makes a choice one a run can start on."""
|
|
107
|
+
p, e = split(value)
|
|
108
|
+
return endpoint_vars(e)[0] if e else PROVIDER_NEEDS.get(p, PROVIDER_NEEDS["custom"])
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# OpenRouter names a model with its maker in front (openai/gpt-4o); a
|
|
112
|
+
# benchmark's official judge or simulator is written as its maker spells it.
|
|
113
|
+
_MAKERS = (("openai/", ("gpt-", "o1", "o3", "o4", "chatgpt-", "text-embedding-")),
|
|
114
|
+
("anthropic/", ("claude-",)), ("google/", ("gemini-",)))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def served_as(model: str, provider: str) -> str:
|
|
118
|
+
"""An official model's name as ``provider`` serves it: on OpenRouter a
|
|
119
|
+
bare name gets its maker's prefix (gpt-4o-2024-08-06 is
|
|
120
|
+
openai/gpt-4o-2024-08-06 there); a name already prefixed, an unknown
|
|
121
|
+
maker or another provider leave it as written."""
|
|
122
|
+
if provider != "openrouter" or "/" in model:
|
|
123
|
+
return model
|
|
124
|
+
for prefix, starts in _MAKERS:
|
|
125
|
+
if model.startswith(starts):
|
|
126
|
+
return prefix + model
|
|
127
|
+
return model
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def available(env=os.environ) -> List[str]:
|
|
131
|
+
"""The choices whose setting is set (``needs``), in the order the builder
|
|
132
|
+
prefers them: OpenRouter, OpenAI, then each OpenAI-compatible endpoint."""
|
|
133
|
+
return [p for p in PROVIDERS[:2] if env.get(PROVIDER_NEEDS[p])] + [choice("custom", e["id"]) for e in endpoints(env)]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def embedding_problem(model: str, provider: str) -> str:
|
|
137
|
+
"""Bare OpenAI embedding names work on OpenAI or OpenRouter; qualified names require OpenRouter."""
|
|
138
|
+
allowed = ("openrouter",) if "/" in model else ("openai", "openrouter")
|
|
139
|
+
if provider not in allowed:
|
|
140
|
+
names = " or ".join(PROVIDER_LABEL[p] for p in allowed)
|
|
141
|
+
return f"{model} needs the {names} provider for this search."
|
|
142
|
+
return ""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _where(provider: str, endpoint: str, env) -> tuple:
|
|
146
|
+
"""(base URL, key) a choice is asked at; ValueError, before any request, when it cannot be asked."""
|
|
147
|
+
url_var, key_var = endpoint_vars(endpoint)
|
|
148
|
+
base = PROVIDER_BASE[provider] if provider != "custom" else env.get(url_var, "")
|
|
149
|
+
key = env.get(key_var if provider == "custom" else PROVIDER_KEY[provider], "")
|
|
150
|
+
if not base:
|
|
151
|
+
raise ValueError("Set the provider endpoint in Settings, or enter a model ID.")
|
|
152
|
+
if provider == "openai" and not key:
|
|
153
|
+
raise ValueError("Add the provider key in Settings, or enter a model ID.")
|
|
154
|
+
parsed = urllib.parse.urlsplit(base)
|
|
155
|
+
if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
156
|
+
raise ValueError("The provider endpoint must be an HTTP or HTTPS base URL.")
|
|
157
|
+
return base.rstrip("/"), key
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
161
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
_MOST = 8 * 1024 * 1024
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _get(url: str, headers: dict, opener=None) -> bytes:
|
|
169
|
+
"""One GET, never another method: no redirect followed, so a key stays at its endpoint; at most 8 MiB read."""
|
|
170
|
+
req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": "nullius", **headers})
|
|
171
|
+
with (opener or urllib.request.build_opener(_NoRedirect()).open)(req, timeout=12) as response:
|
|
172
|
+
content = response.read(_MOST + 1)
|
|
173
|
+
if len(content) > _MOST:
|
|
174
|
+
raise ValueError("response too large")
|
|
175
|
+
return content
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _models(content: bytes) -> list[dict]:
|
|
179
|
+
"""The model list of a ``{"data": [{"id": ...}]}`` answer, by name; ValueError when there is none."""
|
|
180
|
+
data = json.loads(content)
|
|
181
|
+
rows = data.get("data") if isinstance(data, dict) else None
|
|
182
|
+
if not isinstance(rows, list):
|
|
183
|
+
raise ValueError("missing model list")
|
|
184
|
+
models = {}
|
|
185
|
+
for row in rows:
|
|
186
|
+
if not isinstance(row, dict):
|
|
187
|
+
continue
|
|
188
|
+
model = row.get("id")
|
|
189
|
+
if not isinstance(model, str) or not model or len(model) > 256 or any(c.isspace() or ord(c) < 32 for c in model):
|
|
190
|
+
continue
|
|
191
|
+
label = row.get("name") or row.get("display_name") or model
|
|
192
|
+
models[model] = {"id": model, "name": label[:256] if isinstance(label, str) else model}
|
|
193
|
+
return sorted(models.values(), key=lambda m: (m["name"].casefold(), m["id"]))
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _bearer(key: str) -> dict:
|
|
197
|
+
return {"Authorization": "Bearer " + key} if key else {}
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def list_models(provider: str, env=None, *, opener=None) -> list[dict]:
|
|
201
|
+
"""Read model IDs from the selected provider, or ``custom:<id>``'s endpoint; credentials never leave its endpoint."""
|
|
202
|
+
env = os.environ if env is None else env
|
|
203
|
+
provider, endpoint = split(provider)
|
|
204
|
+
if provider not in PROVIDERS:
|
|
205
|
+
raise ValueError("Choose a model provider.")
|
|
206
|
+
base, key = _where(provider, endpoint, env)
|
|
207
|
+
try:
|
|
208
|
+
return _models(_get(base + "/models", _bearer(key), opener))
|
|
209
|
+
except (OSError, ValueError, TypeError, http.client.HTTPException):
|
|
210
|
+
# Provider errors can contain keys or private endpoint details; do not echo them.
|
|
211
|
+
raise ValueError("Model list unavailable. Enter a model ID manually.") from None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# Settings' Test, per provider: only requests that cost nothing. A model list everywhere; OpenRouter's is
|
|
215
|
+
# public, so its key is asked at its own key-information endpoint. TypeSafe has no free request: no Test.
|
|
216
|
+
TESTED = ("openrouter", "openai", "anthropic", "custom")
|
|
217
|
+
ANTHROPIC = ("https://api.anthropic.com/v1", "ANTHROPIC_API_KEY")
|
|
218
|
+
_TESTS_LOCK = threading.Lock()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _unreached(reason) -> str:
|
|
222
|
+
"""Why a request reached nothing, in a few fixed words: never the error's own text, which may hold the URL."""
|
|
223
|
+
for kind, said in ((socket.gaierror, "name not resolved"), (ConnectionRefusedError, "connection refused"),
|
|
224
|
+
(TimeoutError, "timed out"), (ssl.SSLError, "TLS failed")):
|
|
225
|
+
if isinstance(reason, kind):
|
|
226
|
+
return said
|
|
227
|
+
return "not reached"
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def test(choice: str, env=None, *, opener=None, clock=time.monotonic) -> dict:
|
|
231
|
+
"""A provider asked with GETs that cost nothing, never a completion: its model list (and, for OpenRouter,
|
|
232
|
+
its key's information). An OpenAI-compatible endpoint that answers with a key is asked once more with a
|
|
233
|
+
wrong one: an answer then means it does not check keys. ``{at, ok, reached, key, models, ms, error}``,
|
|
234
|
+
``key`` one of accepted, refused, not checked (the server checks none), not tested; never a key.
|
|
235
|
+
ValueError, before any request, when the provider has no test or its settings are incomplete."""
|
|
236
|
+
env = os.environ if env is None else env
|
|
237
|
+
provider, endpoint = split(choice)
|
|
238
|
+
if provider not in TESTED:
|
|
239
|
+
raise ValueError("This provider has no test.")
|
|
240
|
+
if provider == "anthropic":
|
|
241
|
+
from nullius.standard.relay import ANTHROPIC_VERSION
|
|
242
|
+
base, key = ANTHROPIC[0], env.get(ANTHROPIC[1], "")
|
|
243
|
+
auth, models = {"x-api-key": key, "anthropic-version": ANTHROPIC_VERSION}, base + "/models?limit=1000"
|
|
244
|
+
else:
|
|
245
|
+
base, key = _where(provider, endpoint, env)
|
|
246
|
+
auth, models = _bearer(key), base + "/models"
|
|
247
|
+
if not key and provider != "custom":
|
|
248
|
+
raise ValueError("Add the provider key in Settings.")
|
|
249
|
+
out = {"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "reached": False, "key": "not tested",
|
|
250
|
+
"models": None, "ms": 0, "error": ""}
|
|
251
|
+
start = clock()
|
|
252
|
+
try:
|
|
253
|
+
# OpenRouter's model list is public: its key goes to its key endpoint only
|
|
254
|
+
out["models"] = len(_models(_get(models, {} if provider == "openrouter" else auth, opener)))
|
|
255
|
+
out["reached"] = True
|
|
256
|
+
if provider == "openrouter":
|
|
257
|
+
_get(base + "/key", auth, opener)
|
|
258
|
+
out["key"] = "accepted"
|
|
259
|
+
elif provider != "custom":
|
|
260
|
+
out["key"] = "accepted"
|
|
261
|
+
elif not key:
|
|
262
|
+
out["key"] = "not checked"
|
|
263
|
+
else:
|
|
264
|
+
try:
|
|
265
|
+
_get(models, _bearer("nullius-wrong-" + secrets.token_hex(8)), opener)
|
|
266
|
+
out["key"] = "not checked"
|
|
267
|
+
except urllib.error.HTTPError as e:
|
|
268
|
+
e.close()
|
|
269
|
+
out["key"] = "accepted" if e.code in (401, 403) else "not tested"
|
|
270
|
+
except (OSError, http.client.HTTPException):
|
|
271
|
+
pass
|
|
272
|
+
except urllib.error.HTTPError as e:
|
|
273
|
+
e.close()
|
|
274
|
+
out["reached"] = True
|
|
275
|
+
if e.code in (401, 403):
|
|
276
|
+
out["key"] = "refused"
|
|
277
|
+
else:
|
|
278
|
+
out["error"] = f"HTTP {e.code}"
|
|
279
|
+
except urllib.error.URLError as e:
|
|
280
|
+
out["error"] = _unreached(e.reason)
|
|
281
|
+
except (OSError, http.client.HTTPException) as e:
|
|
282
|
+
out["error"] = _unreached(e)
|
|
283
|
+
except ValueError:
|
|
284
|
+
out.update(reached=True, error="no model list")
|
|
285
|
+
out["ms"] = round((clock() - start) * 1000)
|
|
286
|
+
out["ok"] = out["reached"] and not out["error"] and out["key"] != "refused"
|
|
287
|
+
return out
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _tests_file(env=None) -> str:
|
|
291
|
+
return os.path.join(paths.data_dir(env), "provider-tests.json")
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def tests(env=None) -> dict:
|
|
295
|
+
"""Each provider's last test by choice; none when the file is missing or unreadable."""
|
|
296
|
+
try:
|
|
297
|
+
with open(_tests_file(env), encoding="utf-8") as f:
|
|
298
|
+
got = json.load(f)
|
|
299
|
+
except (OSError, ValueError):
|
|
300
|
+
return {}
|
|
301
|
+
return got if isinstance(got, dict) else {}
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def remember(results: dict, env=None) -> None:
|
|
305
|
+
"""Set (a result) or forget (None) the last test of each choice, the file written whole and renamed."""
|
|
306
|
+
with _TESTS_LOCK:
|
|
307
|
+
kept = {k: v for k, v in {**tests(env), **results}.items() if v is not None}
|
|
308
|
+
out = _tests_file(env)
|
|
309
|
+
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
310
|
+
part = f"{out}.{os.getpid()}.part"
|
|
311
|
+
with open(part, "w", encoding="utf-8") as f:
|
|
312
|
+
json.dump(kept, f, indent=1, sort_keys=True)
|
|
313
|
+
os.replace(part, out)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def mark(result: dict) -> str:
|
|
317
|
+
"""ok, warn (it answers but its key is not known good) or bad: the colour of a test's line."""
|
|
318
|
+
return "bad" if not result.get("ok") else "ok" if result.get("key") == "accepted" else "warn"
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
_KEY_SAID = {"refused": "key refused", "not checked": "does not check keys", "not tested": "key not tested"}
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def said(result: dict) -> str:
|
|
325
|
+
"""A test in one line: what failed, or the key, the model count and the time it took; then when."""
|
|
326
|
+
when = str(result.get("at", "")).replace("T", " ")[:16] + " UTC"
|
|
327
|
+
if result.get("error"):
|
|
328
|
+
head = ("" if result.get("reached") else "Not reached: ") + result["error"]
|
|
329
|
+
elif result.get("key") == "refused":
|
|
330
|
+
head = "Key refused"
|
|
331
|
+
else:
|
|
332
|
+
n = result.get("models")
|
|
333
|
+
head = " · ".join([("Key accepted" if result.get("key") == "accepted" else "Reached · " + _KEY_SAID.get(result.get("key"), "")),
|
|
334
|
+
f"{n} model{'s' * (n != 1)}", f"{result.get('ms', 0)} ms"])
|
|
335
|
+
return f"{head} · {when}"
|
|
File without changes
|