nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,785 @@
|
|
|
1
|
+
"""The model endpoint every agent, driver and capability of a run is given.
|
|
2
|
+
|
|
3
|
+
It listens on 127.0.0.1, holds the real key and adds it on the way out, so
|
|
4
|
+
what it relays for never holds one: their requests carry a placeholder. Each
|
|
5
|
+
unit has a route of its own, ``http://127.0.0.1:<port>/<tag>/v1``, so a call is
|
|
6
|
+
known by the unit that made it without reading its content.
|
|
7
|
+
|
|
8
|
+
A reply is passed through as it arrives - a streamed one (server-sent events)
|
|
9
|
+
byte for byte - and recorded once it is complete, one line per call in
|
|
10
|
+
``<record_dir>/<tag>.jsonl``: when it started and ended, the model, the status,
|
|
11
|
+
the tokens, and an abridged request and reply. Three API shapes are read,
|
|
12
|
+
Chat Completions, Responses and Anthropic's Messages, streamed or not. The
|
|
13
|
+
cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an
|
|
14
|
+
estimate from ``prices`` when one is given for exactly that model (the
|
|
15
|
+
user's price list, ``nullius.prices``), marked as an estimate.
|
|
16
|
+
|
|
17
|
+
What the relay changes on the way is kept small, so a call through it costs
|
|
18
|
+
what the same call made directly would: the connection to the provider is
|
|
19
|
+
kept open between calls (a new TLS handshake per call was most of what the
|
|
20
|
+
relay added); the caller's headers go on as they came - an agent's session
|
|
21
|
+
and capability headers are the provider's to read - except the credentials,
|
|
22
|
+
which the relay replaces with its own key, and the headers of the hop itself;
|
|
23
|
+
the provider's headers come back the same way (its request id, its rate
|
|
24
|
+
limits). It never answers for the provider: a 429 reaches the caller and the
|
|
25
|
+
record as it was sent, which is what makes rate limits measurable.
|
|
26
|
+
|
|
27
|
+
One answer is the relay's own: a run with a spending cap (``ledger``) books
|
|
28
|
+
each call before it goes out, and a call that does not fit is not sent. Its
|
|
29
|
+
caller gets a 402 that names the relay and the cap, and the record says
|
|
30
|
+
``refused``, so it never reads as the provider's error.
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import hashlib
|
|
35
|
+
import http.client
|
|
36
|
+
import json
|
|
37
|
+
import os
|
|
38
|
+
import re
|
|
39
|
+
import select
|
|
40
|
+
import socket
|
|
41
|
+
import threading
|
|
42
|
+
import time
|
|
43
|
+
import urllib.parse
|
|
44
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
45
|
+
from typing import Dict, List, Optional
|
|
46
|
+
|
|
47
|
+
from nullius.redact import redact
|
|
48
|
+
from nullius.standard import ledger as ledgers
|
|
49
|
+
|
|
50
|
+
PLACEHOLDER_KEY = "nullius-relay"
|
|
51
|
+
_TAG = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
|
|
52
|
+
TIMEOUT = 900 # one call, streamed or not: a long reasoning turn is still one call
|
|
53
|
+
# The headers of one hop, which a proxy never passes on (RFC 9110, 7.6.1).
|
|
54
|
+
_HOP = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "proxy-connection", "te", "trailer",
|
|
55
|
+
"trailers", "transfer-encoding", "upgrade"}
|
|
56
|
+
# Not passed on from the caller: the hop's, the ones the relay sets itself
|
|
57
|
+
# (the body it may have rewritten has a length of its own; it asks for an
|
|
58
|
+
# uncompressed reply so it can read the reply), and cookies.
|
|
59
|
+
_NOT_SENT = _HOP | {"host", "content-length", "accept-encoding", "cookie", "cookie2"}
|
|
60
|
+
# A credential, whatever the vendor calls it: the relay's key replaces it.
|
|
61
|
+
# The account headers go with them - the relay's key says whose account it is.
|
|
62
|
+
_CREDENTIAL = re.compile(r"authorization|api[-_]?key|auth|token|secret|password|credential|"
|
|
63
|
+
r"^openai-organization$|^openai-project$", re.I)
|
|
64
|
+
# Not passed back: the hop's, cookies, and what this server writes itself.
|
|
65
|
+
_NOT_RETURNED = _HOP | {"set-cookie", "set-cookie2", "content-length", "date", "server"}
|
|
66
|
+
# A kept connection the provider closed while it sat idle fails like this on
|
|
67
|
+
# its next use, before any of the reply is read: worth one new connection.
|
|
68
|
+
_STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError)
|
|
69
|
+
ANTHROPIC_VERSION = "2023-06-01"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _cut(text, n=4000):
|
|
73
|
+
if not isinstance(text, str) or len(text) <= n:
|
|
74
|
+
return text
|
|
75
|
+
return text[: n // 4] + f"\n[... {len(text) - n} characters left out ...]\n" + text[-(3 * n // 4):]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _said(status: int, body: bytes) -> str:
|
|
79
|
+
try:
|
|
80
|
+
e = json.loads(body).get("error") or {}
|
|
81
|
+
m = e.get("message") if isinstance(e, dict) else str(e)
|
|
82
|
+
except Exception:
|
|
83
|
+
m = body[:200].decode("utf-8", "replace")
|
|
84
|
+
return f"HTTP {status}: {m or ''}".strip()[:300]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _usage(u: Optional[dict]) -> Optional[dict]:
|
|
88
|
+
"""Chat Completions', Responses' and Messages' usage, in one shape:
|
|
89
|
+
``input`` counts every prompt token, ``cached`` those of them a cache
|
|
90
|
+
served. ``cache_write`` counts writes separately; ``cache_write_1h`` is
|
|
91
|
+
their one-hour subset when reported. Anthropic's ``input_tokens`` excludes
|
|
92
|
+
cache reads and writes, so both are added to input, once. Chat Completions
|
|
93
|
+
already includes both in ``prompt_tokens``."""
|
|
94
|
+
if not isinstance(u, dict):
|
|
95
|
+
return None
|
|
96
|
+
inp = u.get("prompt_tokens", u.get("input_tokens")) or 0
|
|
97
|
+
out = u.get("completion_tokens", u.get("output_tokens")) or 0
|
|
98
|
+
det = u.get("prompt_tokens_details") or u.get("input_tokens_details") or {}
|
|
99
|
+
rdet = u.get("completion_tokens_details") or u.get("output_tokens_details") or {}
|
|
100
|
+
cached = (det or {}).get("cached_tokens") or 0
|
|
101
|
+
written = u.get("cache_creation_input_tokens") or 0
|
|
102
|
+
if "cache_read_input_tokens" in u or "cache_creation_input_tokens" in u:
|
|
103
|
+
cached = u.get("cache_read_input_tokens") or 0
|
|
104
|
+
inp += cached + written
|
|
105
|
+
else:
|
|
106
|
+
written = (det or {}).get("cache_write_tokens") or 0
|
|
107
|
+
got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0}
|
|
108
|
+
if written:
|
|
109
|
+
got["cache_write"] = written
|
|
110
|
+
hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0
|
|
111
|
+
if hour:
|
|
112
|
+
got["cache_write_1h"] = min(hour, written)
|
|
113
|
+
if isinstance(u.get("cost"), (int, float)):
|
|
114
|
+
got["cost"] = u["cost"]
|
|
115
|
+
return got
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _text_of_responses(resp: dict) -> (str, List[dict]):
|
|
119
|
+
text, calls = [], []
|
|
120
|
+
for item in resp.get("output") or []:
|
|
121
|
+
if item.get("type") == "message":
|
|
122
|
+
for c in item.get("content") or []:
|
|
123
|
+
if c.get("type") in ("output_text", "text"):
|
|
124
|
+
text.append(c.get("text") or "")
|
|
125
|
+
elif item.get("type") in ("function_call", "custom_tool_call", "local_shell_call"):
|
|
126
|
+
calls.append({"name": item.get("name") or item.get("type"),
|
|
127
|
+
"arguments": _cut(item.get("arguments") or item.get("input") or json.dumps(item.get("action")), 2000)})
|
|
128
|
+
return "".join(text), calls
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _text_of_messages(msg: dict) -> (str, List[dict]):
|
|
132
|
+
text, calls = [], []
|
|
133
|
+
for c in msg.get("content") or []:
|
|
134
|
+
if c.get("type") == "text":
|
|
135
|
+
text.append(c.get("text") or "")
|
|
136
|
+
elif c.get("type") in ("tool_use", "server_tool_use"):
|
|
137
|
+
calls.append({"name": c.get("name"), "arguments": _cut(json.dumps(c.get("input")), 2000)})
|
|
138
|
+
return "".join(text), calls
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def read_reply(path: str, body: bytes, streamed: bool) -> dict:
|
|
142
|
+
"""What a reply said: its text, its tool calls, its usage."""
|
|
143
|
+
if not streamed:
|
|
144
|
+
try:
|
|
145
|
+
got = json.loads(body)
|
|
146
|
+
except ValueError:
|
|
147
|
+
return {}
|
|
148
|
+
if "choices" in got:
|
|
149
|
+
m = ((got.get("choices") or [{}])[0].get("message")) or {}
|
|
150
|
+
calls = [{"name": (c.get("function") or {}).get("name"), "arguments": _cut((c.get("function") or {}).get("arguments"), 2000)}
|
|
151
|
+
for c in m.get("tool_calls") or []]
|
|
152
|
+
return {"text": m.get("content"), "tool_calls": calls, "usage": _usage(got.get("usage"))}
|
|
153
|
+
if got.get("type") == "message" and isinstance(got.get("content"), list):
|
|
154
|
+
text, calls = _text_of_messages(got)
|
|
155
|
+
return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}
|
|
156
|
+
if got.get("object") == "response" or "output" in got:
|
|
157
|
+
text, calls = _text_of_responses(got)
|
|
158
|
+
return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}
|
|
159
|
+
if got.get("object") == "list" and path.split("?")[0].endswith("/embeddings"): # how many inputs, and the usage: never the vectors
|
|
160
|
+
return {"embeddings": len(got.get("data") or []), "usage": _usage(got.get("usage"))}
|
|
161
|
+
return {}
|
|
162
|
+
text, calls, usage, args = [], {}, None, {}
|
|
163
|
+
told = None # Messages: the usage of message_start, then message_delta's on it
|
|
164
|
+
for line in body.decode("utf-8", "replace").splitlines():
|
|
165
|
+
if not line.startswith("data:"):
|
|
166
|
+
continue
|
|
167
|
+
data = line[5:].strip()
|
|
168
|
+
if not data or data == "[DONE]":
|
|
169
|
+
continue
|
|
170
|
+
try:
|
|
171
|
+
ev = json.loads(data)
|
|
172
|
+
except ValueError:
|
|
173
|
+
continue
|
|
174
|
+
kind = ev.get("type")
|
|
175
|
+
if kind == "message_start":
|
|
176
|
+
told = dict(((ev.get("message") or {}).get("usage")) or {})
|
|
177
|
+
continue
|
|
178
|
+
if kind == "message_delta":
|
|
179
|
+
told = {**(told or {}), **{k: v for k, v in (ev.get("usage") or {}).items() if v is not None}}
|
|
180
|
+
continue
|
|
181
|
+
if kind == "content_block_start":
|
|
182
|
+
b, i = ev.get("content_block") or {}, ev.get("index", 0)
|
|
183
|
+
if b.get("type") in ("tool_use", "server_tool_use"):
|
|
184
|
+
calls[i], args[i] = b.get("name"), ""
|
|
185
|
+
elif b.get("text"):
|
|
186
|
+
text.append(b["text"])
|
|
187
|
+
continue
|
|
188
|
+
if kind == "content_block_delta":
|
|
189
|
+
d, i = ev.get("delta") or {}, ev.get("index", 0)
|
|
190
|
+
if d.get("type") == "text_delta":
|
|
191
|
+
text.append(d.get("text") or "")
|
|
192
|
+
elif d.get("type") == "input_json_delta":
|
|
193
|
+
args[i] = args.get(i, "") + (d.get("partial_json") or "")
|
|
194
|
+
continue
|
|
195
|
+
if kind in ("response.completed", "response.done", "response.incomplete"):
|
|
196
|
+
t, c = _text_of_responses(ev.get("response") or {})
|
|
197
|
+
return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))}
|
|
198
|
+
for ch in ev.get("choices") or []:
|
|
199
|
+
d = ch.get("delta") or {}
|
|
200
|
+
if d.get("content"):
|
|
201
|
+
text.append(d["content"])
|
|
202
|
+
for tc in d.get("tool_calls") or []:
|
|
203
|
+
i = tc.get("index", 0)
|
|
204
|
+
f = tc.get("function") or {}
|
|
205
|
+
if f.get("name"):
|
|
206
|
+
calls[i] = f["name"]
|
|
207
|
+
args[i] = args.get(i, "") + (f.get("arguments") or "")
|
|
208
|
+
if ev.get("usage"):
|
|
209
|
+
usage = _usage(ev["usage"])
|
|
210
|
+
return {"text": "".join(text), "tool_calls": [{"name": calls[i], "arguments": _cut(args.get(i, ""), 2000)} for i in sorted(calls)],
|
|
211
|
+
"usage": _usage(told) if told is not None else usage}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _request_tail(req: dict) -> dict:
|
|
215
|
+
"""The part of a request a reader needs: the model, how many messages,
|
|
216
|
+
the last two, abridged, and the reasoning effort it asked for (chat's
|
|
217
|
+
``reasoning_effort``, the Responses API's ``reasoning.effort``; None when
|
|
218
|
+
it sent none), so a run shows the effort its agent really sent. A coding
|
|
219
|
+
agent resends its whole context on every call; the trajectory comes from
|
|
220
|
+
the session, not from here."""
|
|
221
|
+
items = req.get("messages") if isinstance(req.get("messages"), list) else req.get("input")
|
|
222
|
+
if isinstance(items, str):
|
|
223
|
+
items = [{"role": "user", "content": items}]
|
|
224
|
+
items = items if isinstance(items, list) else []
|
|
225
|
+
tail = []
|
|
226
|
+
for m in items[-2:]:
|
|
227
|
+
if isinstance(m, dict):
|
|
228
|
+
c = m.get("content")
|
|
229
|
+
if isinstance(c, list):
|
|
230
|
+
c = " ".join(str(x.get("text") or "") for x in c if isinstance(x, dict))
|
|
231
|
+
tail.append({"role": m.get("role") or m.get("type"), "content": _cut(c if isinstance(c, str) else json.dumps(c) if c else "", 2000)})
|
|
232
|
+
effort = req.get("reasoning_effort")
|
|
233
|
+
if effort is None and isinstance(req.get("reasoning"), dict):
|
|
234
|
+
effort = req["reasoning"].get("effort")
|
|
235
|
+
return {"n_messages": len(items), "last": tail, "stream": bool(req.get("stream")),
|
|
236
|
+
"reasoning_effort": effort if isinstance(effort, str) else None}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _digest(obj) -> str:
|
|
240
|
+
text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True)
|
|
241
|
+
return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16]
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def request_fields(req: dict) -> dict:
|
|
245
|
+
"""Every top-level field of a request, for ``fidelity`` to compare two
|
|
246
|
+
ways of driving one agent - and no prompt: ``instructions``, each tool
|
|
247
|
+
and each ``input``/``messages`` item are kept as a digest and a length (an
|
|
248
|
+
item with its type and role, digested without its own ``id``), the rest as
|
|
249
|
+
sent, a large value digested too."""
|
|
250
|
+
out = {}
|
|
251
|
+
for k, v in req.items():
|
|
252
|
+
if k in ("input", "messages"):
|
|
253
|
+
items = [{"role": "user", "content": v}] if isinstance(v, str) else v if isinstance(v, list) else []
|
|
254
|
+
items = [{n: x for n, x in i.items() if n != "id"} if isinstance(i, dict) else i for i in items] # an item's own id
|
|
255
|
+
out[k] = [{"type": i.get("type"), "role": i.get("role"), "sha256": _digest(i), "chars": len(json.dumps(i))}
|
|
256
|
+
if isinstance(i, dict) else {"sha256": _digest(i)} for i in items]
|
|
257
|
+
elif k == "tools" and isinstance(v, list):
|
|
258
|
+
out[k] = [{"name": t.get("name") or (t.get("function") or {}).get("name") or t.get("type"), "sha256": _digest(t)}
|
|
259
|
+
if isinstance(t, dict) else {"sha256": _digest(t)} for t in v]
|
|
260
|
+
elif k == "instructions" or len(json.dumps(v)) > 600:
|
|
261
|
+
out[k] = {"sha256": _digest(v), "chars": len(v) if isinstance(v, str) else len(json.dumps(v))}
|
|
262
|
+
else:
|
|
263
|
+
out[k] = v
|
|
264
|
+
return out
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def forwarded(headers) -> Dict[str, str]:
|
|
268
|
+
"""The caller's request headers the relay passes on: all of them but the
|
|
269
|
+
hop's, the ones it sets itself, cookies and every credential."""
|
|
270
|
+
named = {n.strip().lower() for n in (headers.get("Connection") or "").split(",")} # hop headers by name
|
|
271
|
+
out = {}
|
|
272
|
+
for name, value in headers.items():
|
|
273
|
+
low = name.lower()
|
|
274
|
+
if low in _NOT_SENT or low in named or low.startswith("proxy-") or _CREDENTIAL.search(low):
|
|
275
|
+
continue
|
|
276
|
+
out[name] = value
|
|
277
|
+
return out
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def returned(headers) -> List[tuple]:
|
|
281
|
+
"""The provider's reply headers passed back to the caller."""
|
|
282
|
+
named = {n.strip().lower() for n in (headers.get("Connection") or "").split(",")}
|
|
283
|
+
return [(n, v) for n, v in headers.items() if n.lower() not in _NOT_RETURNED and n.lower() not in named]
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class _Pool:
|
|
287
|
+
"""Kept connections to the provider. A connection is taken for one call
|
|
288
|
+
and given back once its reply was read to the end, so no two calls ever
|
|
289
|
+
share one at a time; calls made one after another, from whichever
|
|
290
|
+
thread, reuse it. (The relay's server starts a thread for each of its
|
|
291
|
+
callers' connections, so a pool per thread would rarely be reused.)"""
|
|
292
|
+
|
|
293
|
+
KEEP = 16 # idle connections kept per provider; more are closed
|
|
294
|
+
|
|
295
|
+
def __init__(self, base: str):
|
|
296
|
+
u = urllib.parse.urlsplit(base)
|
|
297
|
+
self.https = u.scheme == "https"
|
|
298
|
+
self.host, self.port = u.hostname, u.port or (443 if self.https else 80)
|
|
299
|
+
self.prefix = u.path.rstrip("/")
|
|
300
|
+
self.idle: List[http.client.HTTPConnection] = []
|
|
301
|
+
self.lock = threading.Lock()
|
|
302
|
+
self.opened = 0 # connections made, for a test to count
|
|
303
|
+
|
|
304
|
+
def take(self, fresh: bool = False):
|
|
305
|
+
if not fresh:
|
|
306
|
+
with self.lock:
|
|
307
|
+
if self.idle:
|
|
308
|
+
return self.idle.pop(), True
|
|
309
|
+
kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection
|
|
310
|
+
with self.lock:
|
|
311
|
+
self.opened += 1
|
|
312
|
+
return kind(self.host, self.port, timeout=TIMEOUT), False
|
|
313
|
+
|
|
314
|
+
def give(self, conn, r):
|
|
315
|
+
"""Back to the pool when the reply was read whole and the provider
|
|
316
|
+
keeps the connection open; else closed."""
|
|
317
|
+
if r.isclosed() and not r.will_close:
|
|
318
|
+
with self.lock:
|
|
319
|
+
if len(self.idle) < self.KEEP:
|
|
320
|
+
self.idle.append(conn)
|
|
321
|
+
return
|
|
322
|
+
conn.close()
|
|
323
|
+
|
|
324
|
+
def send(self, method: str, path: str, body: Optional[bytes], headers: Dict[str, str]):
|
|
325
|
+
"""The connection and the provider's response, its headers read. A
|
|
326
|
+
kept connection found closed is replaced once - only before any of
|
|
327
|
+
the reply was read, so a call is never sent twice for an answer."""
|
|
328
|
+
conn, reused = self.take()
|
|
329
|
+
while True:
|
|
330
|
+
try:
|
|
331
|
+
if conn.sock is None:
|
|
332
|
+
try:
|
|
333
|
+
conn.connect() # the address, the connection, TLS: a failure here is before anything was sent
|
|
334
|
+
except Exception as e:
|
|
335
|
+
e.unsent = True
|
|
336
|
+
raise
|
|
337
|
+
conn.request(method, self.prefix + path, body=body, headers=headers)
|
|
338
|
+
return conn, conn.getresponse()
|
|
339
|
+
except _STALE:
|
|
340
|
+
conn.close()
|
|
341
|
+
if not reused:
|
|
342
|
+
raise
|
|
343
|
+
conn, reused = self.take(fresh=True)
|
|
344
|
+
except BaseException:
|
|
345
|
+
conn.close()
|
|
346
|
+
raise
|
|
347
|
+
|
|
348
|
+
def close(self):
|
|
349
|
+
with self.lock:
|
|
350
|
+
idle, self.idle = self.idle, []
|
|
351
|
+
for c in idle:
|
|
352
|
+
c.close()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class Relay:
|
|
356
|
+
def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None,
|
|
357
|
+
model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False,
|
|
358
|
+
ledger: Optional[str] = None, unit_caps: bool = True):
|
|
359
|
+
"""``base``: the provider's endpoint (``https://api.openai.com/v1``).
|
|
360
|
+
``model``: when given, every call is sent as this model, whatever the
|
|
361
|
+
caller named - the run's model is the recipe's, not the agent's. A
|
|
362
|
+
route of its own may name another (``url(tag, model=...)``): a
|
|
363
|
+
benchmark's judge is asked as the judge, not as the run's model.
|
|
364
|
+
``side_models``: the models an agent's own side jobs ask for (a
|
|
365
|
+
review, a summary) that it may call besides the run's model - its
|
|
366
|
+
definition's ``model.side_models``, or a setup's: the upstream's
|
|
367
|
+
code asking a model of its own besides the answerer. Each is a name,
|
|
368
|
+
sent as asked, or ``asked=sent``, sent as the provider names it; a
|
|
369
|
+
side call is recorded ``side``. Any other model is replaced by the
|
|
370
|
+
run's, recorded with ``model_asked``. ``key_header``: how the provider takes its key - a
|
|
371
|
+
bearer ``authorization``, or Anthropic's ``x-api-key``.
|
|
372
|
+
``fields``: each record also holds the request's fields as sent
|
|
373
|
+
(:func:`request_fields`) - ``fidelity``'s, never a normal run's.
|
|
374
|
+
``ledger``: the file of the run's spending cap (``ledger``), shared
|
|
375
|
+
by every relay of the job: a call is booked there before it is sent,
|
|
376
|
+
and not sent when it does not fit."""
|
|
377
|
+
self.fields = fields
|
|
378
|
+
self.ledger = ledgers.Ledger(ledger) if ledger else None
|
|
379
|
+
# False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them
|
|
380
|
+
self.unit_caps = unit_caps
|
|
381
|
+
self.stopping = threading.Event() # said by stop(): a call waiting for room gives up
|
|
382
|
+
self.base, self.key, self.dir = base.rstrip("/"), key, record_dir
|
|
383
|
+
self.prices, self.model = prices or {}, model
|
|
384
|
+
self.side_models = {a: (b or a) for a, _, b in (str(m).partition("=") for m in side_models or ())}
|
|
385
|
+
self.key_header = key_header.lower()
|
|
386
|
+
self.models: Dict[str, str] = {}
|
|
387
|
+
self.embedding_models: Dict[str, str] = {}
|
|
388
|
+
self.lock = threading.Lock()
|
|
389
|
+
self.pool = _Pool(self.base)
|
|
390
|
+
self.server: Optional[ThreadingHTTPServer] = None
|
|
391
|
+
os.makedirs(record_dir, exist_ok=True)
|
|
392
|
+
|
|
393
|
+
def url(self, tag: str, model: Optional[str] = None, *, embeddings: Optional[str] = None) -> str:
|
|
394
|
+
"""A route's model pin, with an optional override only for /embeddings requests on this tag.
|
|
395
|
+
Reusing the URL without new pins preserves them; other tags keep their own routing."""
|
|
396
|
+
if not _TAG.match(tag):
|
|
397
|
+
raise ValueError(f"tag {tag!r}")
|
|
398
|
+
if model:
|
|
399
|
+
self.models[tag] = model
|
|
400
|
+
if embeddings:
|
|
401
|
+
self.embedding_models[tag] = embeddings
|
|
402
|
+
return f"{self.public or self._own()}/{tag}/v1"
|
|
403
|
+
|
|
404
|
+
def _own(self) -> str:
|
|
405
|
+
host, port = self.server.server_address[:2]
|
|
406
|
+
return f"http://{'127.0.0.1' if host in ('0.0.0.0', '') else host}:{port}"
|
|
407
|
+
|
|
408
|
+
def calls(self, tag: str) -> List[dict]:
|
|
409
|
+
try:
|
|
410
|
+
with open(os.path.join(self.dir, tag + ".jsonl")) as f:
|
|
411
|
+
return [json.loads(l) for l in f if l.strip()]
|
|
412
|
+
except OSError:
|
|
413
|
+
return []
|
|
414
|
+
|
|
415
|
+
def _cost(self, model: str, usage: Optional[dict]):
|
|
416
|
+
if not usage:
|
|
417
|
+
return None, None
|
|
418
|
+
if "cost" in usage:
|
|
419
|
+
return usage["cost"], "provider"
|
|
420
|
+
p = self.prices.get(model) # exactly this model: one whose name only starts the same is another
|
|
421
|
+
if not p:
|
|
422
|
+
return None, None
|
|
423
|
+
written = min(usage.get("cache_write") or 0, max(0, usage["input"] - usage["cached"]))
|
|
424
|
+
hour = min(usage.get("cache_write_1h") or 0, written)
|
|
425
|
+
fresh = max(0, usage["input"] - usage["cached"] - written)
|
|
426
|
+
write_rate = p.get("cache_write", p["input"])
|
|
427
|
+
return (fresh * p["input"] + usage["cached"] * p.get("cached", p["input"])
|
|
428
|
+
+ (written - hour) * write_rate + hour * p.get("cache_write_1h", write_rate)
|
|
429
|
+
+ usage["output"] * p["output"]) / 1e6, "estimate"
|
|
430
|
+
|
|
431
|
+
def _said(self, e: Exception) -> str:
|
|
432
|
+
"""An exception as the agent and the record may see it: http.client
|
|
433
|
+
quotes a header value it refuses (a key with a stray CR, a BOM), so
|
|
434
|
+
the key itself is cut out of the text whatever else it says."""
|
|
435
|
+
text = f"{type(e).__name__}: {e}"
|
|
436
|
+
for k in {self.key, self.key.strip()} - {""}:
|
|
437
|
+
text = text.replace(k, "[REDACTED]")
|
|
438
|
+
return redact(text)[:200]
|
|
439
|
+
|
|
440
|
+
def _headers(self, caller) -> Dict[str, str]:
|
|
441
|
+
out = forwarded(caller)
|
|
442
|
+
has = {k.lower() for k in out} # a name in any case is the same header: never send it twice
|
|
443
|
+
for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \
|
|
444
|
+
(("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"):
|
|
445
|
+
if name.lower() not in has:
|
|
446
|
+
out[name] = value
|
|
447
|
+
if self.key_header == "x-api-key":
|
|
448
|
+
out["x-api-key"] = self.key
|
|
449
|
+
else:
|
|
450
|
+
out["Authorization"] = "Bearer " + self.key
|
|
451
|
+
return out
|
|
452
|
+
|
|
453
|
+
def _forward(self, h: BaseHTTPRequestHandler):
|
|
454
|
+
"""One call of a caller, ``h``: sent on, passed back, recorded."""
|
|
455
|
+
bits = h.path.split("/", 2)
|
|
456
|
+
if len(bits) < 3 or not _TAG.match(bits[1]):
|
|
457
|
+
h.send_error(404, "no route: /<tag>/v1/...")
|
|
458
|
+
return
|
|
459
|
+
tag, rest = bits[1], "/" + bits[2]
|
|
460
|
+
rest = rest[3:] if rest.startswith("/v1/") else rest
|
|
461
|
+
raw = h.rfile.read(int(h.headers.get("Content-Length") or 0))
|
|
462
|
+
try:
|
|
463
|
+
req = json.loads(raw) if raw else {}
|
|
464
|
+
except ValueError:
|
|
465
|
+
req = {}
|
|
466
|
+
asked = req.get("model") if isinstance(req, dict) else None
|
|
467
|
+
side = False
|
|
468
|
+
if isinstance(req, dict) and req:
|
|
469
|
+
if rest.split("?")[0] == "/embeddings" and self.embedding_models.get(tag):
|
|
470
|
+
req["model"] = self.embedding_models[tag]
|
|
471
|
+
elif self.models.get(tag):
|
|
472
|
+
req["model"] = self.models[tag]
|
|
473
|
+
elif self.model and asked != self.model and asked in self.side_models:
|
|
474
|
+
side = True
|
|
475
|
+
req["model"] = self.side_models[asked]
|
|
476
|
+
elif self.model:
|
|
477
|
+
req["model"] = self.model
|
|
478
|
+
if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"):
|
|
479
|
+
req.setdefault("stream_options", {})["include_usage"] = True
|
|
480
|
+
raw = json.dumps(req).encode()
|
|
481
|
+
booking = booked = None
|
|
482
|
+
# a call that names its model is booked, whatever its route, but a token count, which costs nothing and
|
|
483
|
+
# reports no usage; one that names none (a cancellation, a body that is no request) has no price to book
|
|
484
|
+
if self.ledger is not None and h.command == "POST":
|
|
485
|
+
if isinstance(req, dict) and req.get("model") and not _free(rest):
|
|
486
|
+
booking, booked, why, by = self._book(req, raw, rest, tag, h)
|
|
487
|
+
else: # nothing to price: sent as it is, but not once the run's cap is reached
|
|
488
|
+
try:
|
|
489
|
+
why, by = (("the run's spending cap was reached: no further call of this run is sent", "cap")
|
|
490
|
+
if self.ledger.state()["closed"] else (None, None))
|
|
491
|
+
except ledgers.Unavailable as e: # as for a call it would book: not sent, and no cap reached
|
|
492
|
+
why, by = str(e), "ledger"
|
|
493
|
+
if by == "gone": # its caller hung up while it waited: nothing was booked, nothing is sent
|
|
494
|
+
return
|
|
495
|
+
if why:
|
|
496
|
+
self._refuse(h, tag, rest, req, asked, why, by)
|
|
497
|
+
return
|
|
498
|
+
t0 = time.time()
|
|
499
|
+
conn, r, back, body = None, None, [], b""
|
|
500
|
+
cut = False # the reply was cut off: the call may have been billed
|
|
501
|
+
try:
|
|
502
|
+
conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers))
|
|
503
|
+
status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers)
|
|
504
|
+
except Exception as e:
|
|
505
|
+
status, kind = 502, "application/json"
|
|
506
|
+
# the connection was never made (refused, unknown or unreachable address, a timeout or TLS failing as
|
|
507
|
+
# it opened): never sent. Anything else came after the request may have gone out: it may have been
|
|
508
|
+
# billed, so its booking is kept
|
|
509
|
+
cut = not getattr(e, "unsent", False)
|
|
510
|
+
body = json.dumps({"error": {"message": "the relay could not reach the endpoint: " + self._said(e)}}).encode()
|
|
511
|
+
streamed = r is not None and "text/event-stream" in kind
|
|
512
|
+
whole = False # the reply read to its end: its connection can be kept
|
|
513
|
+
if r is not None and not streamed:
|
|
514
|
+
try:
|
|
515
|
+
body, whole = r.read(), True
|
|
516
|
+
except Exception as e: # cut off mid-reply
|
|
517
|
+
status, kind, back, cut = 502, "application/json", [], True
|
|
518
|
+
body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode()
|
|
519
|
+
|
|
520
|
+
def record(body):
|
|
521
|
+
rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None,
|
|
522
|
+
"model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {},
|
|
523
|
+
"headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value
|
|
524
|
+
if side:
|
|
525
|
+
rec["side"] = True
|
|
526
|
+
if self.fields and isinstance(req, dict) and req:
|
|
527
|
+
rec["fields"] = request_fields(req)
|
|
528
|
+
cost = None
|
|
529
|
+
if status == 200:
|
|
530
|
+
rec["reply"] = read_reply(rest, body, streamed)
|
|
531
|
+
cost, how = self._cost(rec["model"], rec["reply"].get("usage"))
|
|
532
|
+
if cost is not None:
|
|
533
|
+
rec["cost_usd"], rec["cost_from"] = cost, how
|
|
534
|
+
else:
|
|
535
|
+
rec["error"] = _said(status, body)
|
|
536
|
+
if booking is not None:
|
|
537
|
+
rec["booked_usd"] = booked
|
|
538
|
+
try:
|
|
539
|
+
if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole
|
|
540
|
+
self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output"))
|
|
541
|
+
else: # an error of the provider's, or never reached: not billed
|
|
542
|
+
self.ledger.release(booking)
|
|
543
|
+
except ledgers.Unavailable:
|
|
544
|
+
pass # the booking stays held, against the cap
|
|
545
|
+
with self.lock:
|
|
546
|
+
with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f:
|
|
547
|
+
f.write(json.dumps(rec) + "\n")
|
|
548
|
+
|
|
549
|
+
h.send_response(status)
|
|
550
|
+
if not any(n.lower() == "content-type" for n, _ in back):
|
|
551
|
+
back = back + [("Content-Type", kind)]
|
|
552
|
+
for n, v in back:
|
|
553
|
+
h.send_header(n, v)
|
|
554
|
+
if streamed:
|
|
555
|
+
if not r.getheader("Cache-Control"):
|
|
556
|
+
h.send_header("Cache-Control", "no-cache")
|
|
557
|
+
h.end_headers()
|
|
558
|
+
parts = []
|
|
559
|
+
try:
|
|
560
|
+
while True:
|
|
561
|
+
chunk = r.read1(65536)
|
|
562
|
+
if not chunk:
|
|
563
|
+
whole = True
|
|
564
|
+
break
|
|
565
|
+
parts.append(chunk)
|
|
566
|
+
try:
|
|
567
|
+
h.wfile.write(chunk)
|
|
568
|
+
h.wfile.flush()
|
|
569
|
+
except OSError:
|
|
570
|
+
pass # the caller went away; the record still completes
|
|
571
|
+
except Exception:
|
|
572
|
+
pass
|
|
573
|
+
record(b"".join(parts)) # before the connection closes: the caller's end of stream
|
|
574
|
+
else:
|
|
575
|
+
record(body) # before the caller has the reply, so a reader after it finds the call
|
|
576
|
+
if conn is not None: # read whole: back to the pool before the caller can make its next call
|
|
577
|
+
if whole:
|
|
578
|
+
self.pool.give(conn, r)
|
|
579
|
+
else:
|
|
580
|
+
conn.close()
|
|
581
|
+
conn = None
|
|
582
|
+
h.send_header("Content-Length", str(len(body)))
|
|
583
|
+
h.end_headers()
|
|
584
|
+
try:
|
|
585
|
+
h.wfile.write(body)
|
|
586
|
+
except OSError:
|
|
587
|
+
pass
|
|
588
|
+
if conn is not None:
|
|
589
|
+
if whole:
|
|
590
|
+
self.pool.give(conn, r)
|
|
591
|
+
else:
|
|
592
|
+
conn.close()
|
|
593
|
+
|
|
594
|
+
def _book(self, req: dict, raw: bytes, rest: str, tag: str = "", h: Optional[BaseHTTPRequestHandler] = None) -> tuple:
|
|
595
|
+
"""(booking, its amount, None, None) for a call the cap lets out;
|
|
596
|
+
(None, its amount, why, by) for one it does not - ``by`` the run's
|
|
597
|
+
cap (``cap``), the cap of the unit ``tag`` routes for (``unit``),
|
|
598
|
+
``price`` (a model with no price is not sent either, a capped run
|
|
599
|
+
sending no call it cannot price) or ``ledger`` (it could not be
|
|
600
|
+
read or written) - and those two are no cap reached. ``gone``: its
|
|
601
|
+
caller ``h`` hung up, or this relay stopped, while it waited."""
|
|
602
|
+
row = self.prices.get(req.get("model"))
|
|
603
|
+
if not row:
|
|
604
|
+
return None, None, f"no price for the model {req.get('model')!r}: a run with a spending cap sends no call it cannot price", "price"
|
|
605
|
+
try:
|
|
606
|
+
# a call only the calls in flight keep out waits for them: their bookings are estimates, and settle
|
|
607
|
+
deadline = time.time() + ledgers.WAIT
|
|
608
|
+
least = ledgers.bound(req, raw, row, rest, output=False)
|
|
609
|
+
while True:
|
|
610
|
+
booking, seen = self.ledger.reserve(lambda usual: ledgers.bound(req, raw, row, rest, usual),
|
|
611
|
+
ledgers.unit_of(tag) if self.unit_caps and tag else None, req.get("model"),
|
|
612
|
+
wait=time.time() < deadline, least=least)
|
|
613
|
+
if booking or seen["by"] != "wait":
|
|
614
|
+
break
|
|
615
|
+
if self.stopping.is_set() or _gone(h):
|
|
616
|
+
return None, None, None, "gone" # no one waits for it: not booked later, and the cap not closed in its name
|
|
617
|
+
time.sleep(0.2)
|
|
618
|
+
except ledgers.Unavailable as e: # not a cap reached: the unit is not the cap's, and the run goes on if the ledger reads again
|
|
619
|
+
return None, None, str(e), "ledger"
|
|
620
|
+
amount = seen["amount"]
|
|
621
|
+
return (booking, amount, None, None) if booking else (None, amount, ledgers.refusal(seen, amount), seen["by"])
|
|
622
|
+
|
|
623
|
+
def _refuse(self, h: BaseHTTPRequestHandler, tag: str, rest: str, req, asked, why: str, by: str = "cap") -> None:
|
|
624
|
+
"""A call a capped run did not let out - at the run's cap (``by`` is
|
|
625
|
+
``cap``), at its unit's own (``unit``), or for want of a price
|
|
626
|
+
(``price``): answered here and recorded as the relay's own
|
|
627
|
+
(``refused``), nothing sent."""
|
|
628
|
+
t = time.time()
|
|
629
|
+
rec = {"t0": t, "t1": t, "path": rest, "model": req.get("model") if isinstance(req, dict) else None, "model_asked": asked,
|
|
630
|
+
"status": 402, "refused": by, "error": why, "request": _request_tail(req) if isinstance(req, dict) else {},
|
|
631
|
+
"headers": sorted(k.lower() for k in forwarded(h.headers))}
|
|
632
|
+
with self.lock:
|
|
633
|
+
with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f:
|
|
634
|
+
f.write(json.dumps(rec) + "\n")
|
|
635
|
+
body = json.dumps({"type": "error", "error": {"type": "spending_cap", "code": {"unit": "unit_spending_cap", "price": "unpriced_model", "ledger": "ledger_unavailable"}.get(by, "spending_cap"),
|
|
636
|
+
"message": "nullius relay: " + why}}).encode()
|
|
637
|
+
h.send_response(402)
|
|
638
|
+
for n, v in (("Content-Type", "application/json"), ("X-Nullius-Refused", by), ("Content-Length", str(len(body)))):
|
|
639
|
+
h.send_header(n, v)
|
|
640
|
+
h.end_headers()
|
|
641
|
+
try:
|
|
642
|
+
h.wfile.write(body)
|
|
643
|
+
except OSError:
|
|
644
|
+
pass
|
|
645
|
+
|
|
646
|
+
# The address ``url`` hands out: the listening one unless the caller sets
|
|
647
|
+
# another - a sidecar listens on 0.0.0.0 and is reached as ``relay``.
|
|
648
|
+
public: Optional[str] = None
|
|
649
|
+
|
|
650
|
+
def start(self, host: str = "127.0.0.1", port: int = 0) -> "Relay":
|
|
651
|
+
relay = self
|
|
652
|
+
|
|
653
|
+
class Handler(BaseHTTPRequestHandler):
|
|
654
|
+
def log_message(self, *a):
|
|
655
|
+
pass
|
|
656
|
+
|
|
657
|
+
def do_GET(self):
|
|
658
|
+
self.forward()
|
|
659
|
+
|
|
660
|
+
def do_POST(self):
|
|
661
|
+
self.forward()
|
|
662
|
+
|
|
663
|
+
def forward(self):
|
|
664
|
+
relay._forward(self)
|
|
665
|
+
|
|
666
|
+
self.server = ThreadingHTTPServer((host, port), Handler)
|
|
667
|
+
self.server.daemon_threads = True
|
|
668
|
+
# stop() waits for the serving loop's next look: at the default half-second that idled a worker
|
|
669
|
+
# slot up to 0.5 s at the end of every unit (minutes over a 1319-unit GSM8K run)
|
|
670
|
+
threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True).start()
|
|
671
|
+
return self
|
|
672
|
+
|
|
673
|
+
def stop(self):
|
|
674
|
+
self.stopping.set()
|
|
675
|
+
if self.server:
|
|
676
|
+
self.server.shutdown()
|
|
677
|
+
self.server.server_close()
|
|
678
|
+
self.pool.close()
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _free(rest: str) -> bool:
|
|
682
|
+
"""A route that names a model and is not billed: a token count."""
|
|
683
|
+
return rest.split("?")[0].rstrip("/").endswith("/count_tokens")
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool:
|
|
687
|
+
"""Whether a caller hung up while its call waited: its end of the connection reads closed."""
|
|
688
|
+
if h is None:
|
|
689
|
+
return False
|
|
690
|
+
try:
|
|
691
|
+
return bool(select.select([h.connection], [], [], 0)[0]) and h.connection.recv(1, socket.MSG_PEEK) == b""
|
|
692
|
+
except (OSError, ValueError):
|
|
693
|
+
return True
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def failure(calls: List[dict]) -> Optional[tuple]:
|
|
697
|
+
"""When a unit's model calls ended failing, the exception a trial carries
|
|
698
|
+
for it - named so ``nullius.runs.classify`` sorts it as the
|
|
699
|
+
infrastructure it was - and the endpoint's words. None when the last call
|
|
700
|
+
succeeded (a failure the agent recovered from is not the unit's). A call
|
|
701
|
+
the run's spending cap refused is the unit's wherever it falls: once one
|
|
702
|
+
is refused, every later one is."""
|
|
703
|
+
cut = next((c for c in calls if c.get("refused") == "cap"), None)
|
|
704
|
+
if cut:
|
|
705
|
+
return ledgers.ERROR, cut.get("error") or ""
|
|
706
|
+
over = next((c for c in calls if c.get("refused") == "unit"), None)
|
|
707
|
+
if over: # its own cap: the unit's doing, and the run goes on
|
|
708
|
+
return ledgers.UNIT_ERROR, over.get("error") or ""
|
|
709
|
+
if not calls or calls[-1].get("status") == 200:
|
|
710
|
+
return None
|
|
711
|
+
last = calls[-1]
|
|
712
|
+
s, said = last.get("status"), last.get("error") or ""
|
|
713
|
+
if s == 429:
|
|
714
|
+
return "RateLimitError", said
|
|
715
|
+
if s in (401, 403):
|
|
716
|
+
return "AgentAuthenticationError", said
|
|
717
|
+
if s == 404 and "model" in said.lower():
|
|
718
|
+
return "ModelNotFoundError", said
|
|
719
|
+
if s == 502:
|
|
720
|
+
return "NetworkConnectionError", said
|
|
721
|
+
return "ApiError", said
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
# -- the relay as a process of its own: a container backend's sidecar ----------
|
|
725
|
+
def serve(argv=None) -> int:
|
|
726
|
+
"""``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL
|
|
727
|
+
--record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME]
|
|
728
|
+
[--public URL] [--prices JSON] [--ledger FILE]``.
|
|
729
|
+
|
|
730
|
+
``--base`` is the provider's endpoint, ``--route`` a route sent as another
|
|
731
|
+
model (a benchmark's judge), ``--public`` the address ``url`` hands out.
|
|
732
|
+
The real key is read from ``--key-file`` and nowhere else: an argument or
|
|
733
|
+
an environment variable is visible to anyone who lists processes or
|
|
734
|
+
inspects a container. One JSON line, ``{"listening": "http://host:port"}``, says it
|
|
735
|
+
is ready; SIGTERM (or SIGINT) stops it."""
|
|
736
|
+
import argparse
|
|
737
|
+
import signal
|
|
738
|
+
ap = argparse.ArgumentParser(prog="nullius.standard.relay serve")
|
|
739
|
+
ap.add_argument("--listen", default="127.0.0.1:0", help="HOST:PORT")
|
|
740
|
+
ap.add_argument("--base", required=True, help="the provider's endpoint")
|
|
741
|
+
ap.add_argument("--record", required=True, help="the folder each route's calls are recorded in")
|
|
742
|
+
ap.add_argument("--key-file", required=True, help="the file that holds the provider's key")
|
|
743
|
+
ap.add_argument("--model", help="every call sent as this model")
|
|
744
|
+
ap.add_argument("--route", action="append", default=[], help="TAG=MODEL")
|
|
745
|
+
ap.add_argument("--side-model", action="append", default=[], help="a model the agent may call as asked")
|
|
746
|
+
ap.add_argument("--key-header", default="authorization", choices=("authorization", "x-api-key"))
|
|
747
|
+
ap.add_argument("--public", help="the address url() hands out, when not the listening one")
|
|
748
|
+
ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate")
|
|
749
|
+
ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent")
|
|
750
|
+
a = ap.parse_args(argv)
|
|
751
|
+
try:
|
|
752
|
+
with open(a.key_file) as f:
|
|
753
|
+
key = f.read().strip()
|
|
754
|
+
except OSError as e:
|
|
755
|
+
print(json.dumps({"error": f"cannot read --key-file: {e}"}), flush=True)
|
|
756
|
+
return 2
|
|
757
|
+
if not key:
|
|
758
|
+
print(json.dumps({"error": f"--key-file {a.key_file} contains no key"}), flush=True)
|
|
759
|
+
return 2
|
|
760
|
+
host, _, port = a.listen.rpartition(":")
|
|
761
|
+
r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model,
|
|
762
|
+
side_models=a.side_model, key_header=a.key_header, ledger=a.ledger)
|
|
763
|
+
for route in a.route:
|
|
764
|
+
tag, _, model = route.partition("=")
|
|
765
|
+
if not _TAG.match(tag) or not model:
|
|
766
|
+
print(json.dumps({"error": f"--route {route!r}: TAG=MODEL"}), flush=True)
|
|
767
|
+
return 2
|
|
768
|
+
r.models[tag] = model
|
|
769
|
+
r.public = a.public
|
|
770
|
+
stop = threading.Event()
|
|
771
|
+
signal.signal(signal.SIGTERM, lambda *_: stop.set())
|
|
772
|
+
signal.signal(signal.SIGINT, lambda *_: stop.set())
|
|
773
|
+
r.start(host or "127.0.0.1", int(port or 0))
|
|
774
|
+
print(json.dumps({"listening": r._own()}), flush=True)
|
|
775
|
+
while not stop.wait(1):
|
|
776
|
+
pass
|
|
777
|
+
r.stop()
|
|
778
|
+
return 0
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
if __name__ == "__main__":
|
|
782
|
+
import sys
|
|
783
|
+
if sys.argv[1:2] != ["serve"]:
|
|
784
|
+
sys.exit("python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M]")
|
|
785
|
+
sys.exit(serve(sys.argv[2:]))
|