wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/gateway/app.py
ADDED
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
"""The gateway's HTTP app: provider proxies, session state and an admin API.
|
|
2
|
+
|
|
3
|
+
Runner SDKs point their ``base_url`` here and present a session token instead
|
|
4
|
+
of a key:
|
|
5
|
+
|
|
6
|
+
============= ============================================================
|
|
7
|
+
anthropic ``{gateway}/v1/anthropic`` → api.anthropic.com
|
|
8
|
+
openai ``{gateway}/v1/openai/v1`` → api.openai.com/v1
|
|
9
|
+
google ``{gateway}/v1/google`` → generativelanguage.googleapis.com
|
|
10
|
+
hf-router ``{gateway}/v1/hf-router/v1`` → router.huggingface.co/v1
|
|
11
|
+
compat ``{gateway}/v1/compat/<name>`` → a configured endpoint
|
|
12
|
+
============= ============================================================
|
|
13
|
+
|
|
14
|
+
The admin API (bearer admin token, held by the control plane and the user's
|
|
15
|
+
CLI, never by a runner or a lab) manages limits, prices, the halt flag and
|
|
16
|
+
pauses. No agent tool can reach it.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
from collections.abc import AsyncIterator, Callable
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
import httpx
|
|
30
|
+
from starlette.applications import Starlette
|
|
31
|
+
from starlette.requests import Request
|
|
32
|
+
from starlette.responses import JSONResponse, Response, StreamingResponse
|
|
33
|
+
from starlette.routing import Route
|
|
34
|
+
|
|
35
|
+
from wspx.auth import TokenError, check_bearer, verify
|
|
36
|
+
from wspx.gateway.ledger import Ledger, Limit
|
|
37
|
+
from wspx.gateway.prices import Price
|
|
38
|
+
from wspx.gateway.usage import (
|
|
39
|
+
AnthropicStream,
|
|
40
|
+
GeminiStream,
|
|
41
|
+
Observed,
|
|
42
|
+
OpenAIStream,
|
|
43
|
+
SSEParser,
|
|
44
|
+
Usage,
|
|
45
|
+
anthropic_json,
|
|
46
|
+
gemini_json,
|
|
47
|
+
openai_json,
|
|
48
|
+
)
|
|
49
|
+
from wspx.loops import LoopDetector
|
|
50
|
+
|
|
51
|
+
UPSTREAMS = {
|
|
52
|
+
"anthropic": "https://api.anthropic.com",
|
|
53
|
+
"openai": "https://api.openai.com",
|
|
54
|
+
"google": "https://generativelanguage.googleapis.com",
|
|
55
|
+
"hf-router": "https://router.huggingface.co",
|
|
56
|
+
}
|
|
57
|
+
KEY_ENV = {
|
|
58
|
+
"anthropic": ("ANTHROPIC_API_KEY",),
|
|
59
|
+
"openai": ("OPENAI_API_KEY",),
|
|
60
|
+
"google": ("GEMINI_API_KEY", "GOOGLE_API_KEY"),
|
|
61
|
+
"hf-router": ("HF_TOKEN",),
|
|
62
|
+
}
|
|
63
|
+
DEFAULT_MAX_OUTPUT = 128_000
|
|
64
|
+
KEY_CHECKS = {
|
|
65
|
+
"anthropic": ("/v1/models", {"anthropic-version": "2023-06-01"}),
|
|
66
|
+
"openai": ("/v1/models", {}),
|
|
67
|
+
"google": ("/v1beta/models", {}),
|
|
68
|
+
"hf-router": ("https://huggingface.co/api/whoami-v2", {}),
|
|
69
|
+
}
|
|
70
|
+
"""A free, read-only request per provider that succeeds only with a valid key."""
|
|
71
|
+
"""Worst case used when a request does not state its output cap."""
|
|
72
|
+
HOP_HEADERS = frozenset(
|
|
73
|
+
{
|
|
74
|
+
"host",
|
|
75
|
+
"content-length",
|
|
76
|
+
"connection",
|
|
77
|
+
"transfer-encoding",
|
|
78
|
+
"accept-encoding",
|
|
79
|
+
"authorization",
|
|
80
|
+
"x-api-key",
|
|
81
|
+
"x-goog-api-key",
|
|
82
|
+
"keep-alive",
|
|
83
|
+
}
|
|
84
|
+
)
|
|
85
|
+
SPEND_LIMIT_CODES = frozenset(
|
|
86
|
+
{
|
|
87
|
+
"insufficient_quota",
|
|
88
|
+
"project_spend_limit_exceeded",
|
|
89
|
+
"organization_spend_limit_exceeded",
|
|
90
|
+
"billing_error",
|
|
91
|
+
"billing_hard_limit_reached",
|
|
92
|
+
}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class GatewayConfig:
|
|
98
|
+
secret: str
|
|
99
|
+
admin_token: str
|
|
100
|
+
ledger_path: str
|
|
101
|
+
keys: dict[str, str] = field(default_factory=dict)
|
|
102
|
+
upstreams: dict[str, str] = field(default_factory=lambda: dict(UPSTREAMS))
|
|
103
|
+
compat: dict[str, tuple[str, str | None]] = field(default_factory=dict)
|
|
104
|
+
"""name → (base URL, optional key) for ``openai-compatible`` endpoints."""
|
|
105
|
+
loop_threshold: int = 3
|
|
106
|
+
loop_window: int = 10
|
|
107
|
+
loops_enabled: bool = True
|
|
108
|
+
transport: httpx.AsyncBaseTransport | None = None
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
def from_env(cls) -> GatewayConfig:
|
|
112
|
+
keys = {}
|
|
113
|
+
for provider, names in KEY_ENV.items():
|
|
114
|
+
for name in names:
|
|
115
|
+
if os.environ.get(name):
|
|
116
|
+
keys[provider] = os.environ[name]
|
|
117
|
+
break
|
|
118
|
+
compat: dict[str, tuple[str, str | None]] = {}
|
|
119
|
+
for name, value in os.environ.items():
|
|
120
|
+
if name.startswith("WSPX_COMPAT_") and name.endswith("_URL"):
|
|
121
|
+
label = name[len("WSPX_COMPAT_") : -len("_URL")].lower()
|
|
122
|
+
compat[label] = (value, os.environ.get(f"WSPX_COMPAT_{label.upper()}_KEY"))
|
|
123
|
+
return cls(
|
|
124
|
+
secret=os.environ["WSPX_GATEWAY_SECRET"],
|
|
125
|
+
admin_token=os.environ["WSPX_ADMIN_TOKEN"],
|
|
126
|
+
ledger_path=os.environ.get("WSPX_LEDGER", "/ledger/ledger.db"),
|
|
127
|
+
keys=keys,
|
|
128
|
+
compat=compat,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass
|
|
133
|
+
class Call:
|
|
134
|
+
provider: str
|
|
135
|
+
upstream: str
|
|
136
|
+
model: str
|
|
137
|
+
stream: bool
|
|
138
|
+
max_output: int
|
|
139
|
+
metered: bool
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def create_app(config: GatewayConfig) -> Starlette:
|
|
143
|
+
ledger = Ledger(config.ledger_path)
|
|
144
|
+
loops = LoopDetector(
|
|
145
|
+
threshold=config.loop_threshold, window=config.loop_window, enabled=config.loops_enabled
|
|
146
|
+
)
|
|
147
|
+
client = httpx.AsyncClient(
|
|
148
|
+
timeout=httpx.Timeout(900.0, connect=15.0), transport=config.transport
|
|
149
|
+
)
|
|
150
|
+
last_body: dict[str, tuple[bytes, int]] = {}
|
|
151
|
+
|
|
152
|
+
def session_of(request: Request) -> str:
|
|
153
|
+
token = (
|
|
154
|
+
request.headers.get("x-api-key")
|
|
155
|
+
or request.headers.get("x-goog-api-key")
|
|
156
|
+
or request.query_params.get("key")
|
|
157
|
+
or ""
|
|
158
|
+
)
|
|
159
|
+
auth = request.headers.get("authorization", "")
|
|
160
|
+
if not token and auth.lower().startswith("bearer "):
|
|
161
|
+
token = auth[7:]
|
|
162
|
+
return verify(config.secret, token).session
|
|
163
|
+
|
|
164
|
+
def error(status: int, kind: str, message: str) -> JSONResponse:
|
|
165
|
+
return JSONResponse(
|
|
166
|
+
{"type": "error", "error": {"type": kind, "message": message}},
|
|
167
|
+
status_code=status,
|
|
168
|
+
headers={"x-should-retry": "false"},
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
async def proxy(request: Request) -> Response:
|
|
172
|
+
provider = request.path_params["provider"]
|
|
173
|
+
path = request.path_params["path"]
|
|
174
|
+
try:
|
|
175
|
+
session = session_of(request)
|
|
176
|
+
except TokenError as exc:
|
|
177
|
+
return error(401, "wspx_auth", str(exc))
|
|
178
|
+
if (reason := ledger.halted()) is not None:
|
|
179
|
+
return error(
|
|
180
|
+
503, "wspx_halted", f"wspx is halted ({reason}); `wspx resume` to continue"
|
|
181
|
+
)
|
|
182
|
+
if (reason := ledger.paused(session)) is not None:
|
|
183
|
+
return error(403, "wspx_paused", f"session {session} is paused: {reason}")
|
|
184
|
+
body = await request.body()
|
|
185
|
+
try:
|
|
186
|
+
call = describe(config, provider, path, body)
|
|
187
|
+
except ValueError as exc:
|
|
188
|
+
return error(404, "wspx_route", str(exc))
|
|
189
|
+
key = config.keys.get(provider) if provider != "compat" else None
|
|
190
|
+
if provider == "compat":
|
|
191
|
+
key = config.compat[path.split("/", 1)[0]][1]
|
|
192
|
+
elif key is None:
|
|
193
|
+
return error(
|
|
194
|
+
500, "wspx_no_key", f"no {provider} key is configured; `wspx keys set {provider}`"
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
reservation = None
|
|
198
|
+
price = ledger.price_table().get(call.provider, call.model) if call.metered else None
|
|
199
|
+
if call.metered:
|
|
200
|
+
bound = input_bound(last_body.get(session), body)
|
|
201
|
+
reservation, refusal = ledger.reserve(
|
|
202
|
+
session,
|
|
203
|
+
call.provider,
|
|
204
|
+
call.model,
|
|
205
|
+
input_bound=bound,
|
|
206
|
+
max_output=call.max_output,
|
|
207
|
+
price=price,
|
|
208
|
+
)
|
|
209
|
+
if refusal is not None:
|
|
210
|
+
ledger.pause(session, f"limit reached: {refusal.message}")
|
|
211
|
+
return error(429, "wspx_limit", refusal.message)
|
|
212
|
+
|
|
213
|
+
headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_HEADERS}
|
|
214
|
+
headers.update(auth_headers(provider, key))
|
|
215
|
+
url = call.upstream + ("?" + request.url.query if request.url.query else "")
|
|
216
|
+
upstream = client.build_request(request.method, url, headers=headers, content=body)
|
|
217
|
+
try:
|
|
218
|
+
response = await client.send(upstream, stream=True)
|
|
219
|
+
except httpx.HTTPError as exc:
|
|
220
|
+
ledger.settle(
|
|
221
|
+
reservation,
|
|
222
|
+
session,
|
|
223
|
+
call.provider,
|
|
224
|
+
call.model,
|
|
225
|
+
cost=0.0,
|
|
226
|
+
status=502,
|
|
227
|
+
error=str(exc),
|
|
228
|
+
)
|
|
229
|
+
return error(502, "wspx_upstream", f"{provider} unreachable: {exc}")
|
|
230
|
+
|
|
231
|
+
def finish(usage: Usage, obs: Observed, status: int, err: str | None) -> list[str]:
|
|
232
|
+
cost = usage.cost(price) if usage.complete else None
|
|
233
|
+
if not call.metered:
|
|
234
|
+
return []
|
|
235
|
+
ledger.settle(
|
|
236
|
+
reservation,
|
|
237
|
+
session,
|
|
238
|
+
call.provider,
|
|
239
|
+
call.model,
|
|
240
|
+
input=usage.input,
|
|
241
|
+
output=usage.output,
|
|
242
|
+
cache_read=usage.cache_read,
|
|
243
|
+
cache_write=usage.cache_write,
|
|
244
|
+
cost=cost if usage.complete else None,
|
|
245
|
+
status=status,
|
|
246
|
+
error=err,
|
|
247
|
+
)
|
|
248
|
+
if usage.complete:
|
|
249
|
+
last_body[session] = (body, usage.input + usage.cache_read + usage.cache_write)
|
|
250
|
+
return watch(session, obs, status, err)
|
|
251
|
+
|
|
252
|
+
if response.status_code >= 400:
|
|
253
|
+
raw = await response.aread()
|
|
254
|
+
await response.aclose()
|
|
255
|
+
err, spend_limit = classify_error(raw)
|
|
256
|
+
spend_limit = (
|
|
257
|
+
spend_limit or response.status_code == 402
|
|
258
|
+
) # Gemini: prepaid credit exhausted
|
|
259
|
+
out_headers = passthrough(response.headers)
|
|
260
|
+
if spend_limit:
|
|
261
|
+
ledger.pause(session, f"provider spend limit: {err}")
|
|
262
|
+
out_headers["x-should-retry"] = "false"
|
|
263
|
+
finish(Usage(complete=True), Observed(), response.status_code, err)
|
|
264
|
+
return Response(
|
|
265
|
+
raw,
|
|
266
|
+
status_code=response.status_code,
|
|
267
|
+
headers=out_headers,
|
|
268
|
+
media_type=response.headers.get("content-type"),
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
if not call.stream:
|
|
272
|
+
raw = await response.aread()
|
|
273
|
+
await response.aclose()
|
|
274
|
+
try:
|
|
275
|
+
data = json.loads(raw)
|
|
276
|
+
except json.JSONDecodeError:
|
|
277
|
+
data = {}
|
|
278
|
+
usage, obs = parse_json(provider, path, data)
|
|
279
|
+
finish(usage, obs, response.status_code, None)
|
|
280
|
+
return Response(
|
|
281
|
+
raw,
|
|
282
|
+
status_code=response.status_code,
|
|
283
|
+
headers=passthrough(response.headers),
|
|
284
|
+
media_type=response.headers.get("content-type"),
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
tracker = stream_tracker(provider, path)
|
|
288
|
+
parser = SSEParser()
|
|
289
|
+
|
|
290
|
+
async def relay() -> AsyncIterator[bytes]:
|
|
291
|
+
done = False
|
|
292
|
+
try:
|
|
293
|
+
async for chunk in response.aiter_bytes(): # decoded: content-encoding is stripped
|
|
294
|
+
for name, data in parser.feed(chunk):
|
|
295
|
+
tracker.event(name, data)
|
|
296
|
+
yield chunk
|
|
297
|
+
done = True
|
|
298
|
+
finally:
|
|
299
|
+
await response.aclose()
|
|
300
|
+
usage, obs = tracker.finish()
|
|
301
|
+
if not done:
|
|
302
|
+
usage.complete = False # broken stream: the hold stays charged
|
|
303
|
+
finish(usage, obs, response.status_code, None if done else "stream interrupted")
|
|
304
|
+
|
|
305
|
+
return StreamingResponse(
|
|
306
|
+
relay(),
|
|
307
|
+
status_code=response.status_code,
|
|
308
|
+
headers=passthrough(response.headers),
|
|
309
|
+
media_type=response.headers.get("content-type", "text/event-stream"),
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
def watch(session: str, obs: Observed, status: int, err: str | None) -> list[str]:
|
|
313
|
+
"""Feed one response to the loop detector; record nudges and stops."""
|
|
314
|
+
verdicts = []
|
|
315
|
+
loops.turn(session)
|
|
316
|
+
if err:
|
|
317
|
+
verdicts.append(
|
|
318
|
+
loops.observe(session, "error", f"{status}:{err}", label=f"HTTP {status}")
|
|
319
|
+
)
|
|
320
|
+
else:
|
|
321
|
+
for name, args in obs.tool_calls:
|
|
322
|
+
verdicts.append(
|
|
323
|
+
loops.observe(session, "call", {"tool": name, "args": args}, label=name)
|
|
324
|
+
)
|
|
325
|
+
if obs.text.strip():
|
|
326
|
+
verdicts.append(loops.observe(session, "output", obs.text))
|
|
327
|
+
if not obs.tool_calls and not obs.text.strip():
|
|
328
|
+
verdicts.append(loops.observe(session, "empty", "empty", label="empty turn"))
|
|
329
|
+
messages = []
|
|
330
|
+
for v in verdicts:
|
|
331
|
+
if v is None:
|
|
332
|
+
continue
|
|
333
|
+
ledger.loop_event(session, v.to_dict())
|
|
334
|
+
if v.action == "stop":
|
|
335
|
+
ledger.pause(session, f"loop detected: {v.message}")
|
|
336
|
+
else:
|
|
337
|
+
ledger.add_nudge(session, v.message)
|
|
338
|
+
messages.append(v.message)
|
|
339
|
+
return messages
|
|
340
|
+
|
|
341
|
+
async def session_state(request: Request) -> Response:
|
|
342
|
+
try:
|
|
343
|
+
session = session_of(request)
|
|
344
|
+
except TokenError as exc:
|
|
345
|
+
return error(401, "wspx_auth", str(exc))
|
|
346
|
+
return JSONResponse(
|
|
347
|
+
{
|
|
348
|
+
"session": session,
|
|
349
|
+
"halted": ledger.halted(),
|
|
350
|
+
"paused": ledger.paused(session),
|
|
351
|
+
"nudges": ledger.take_nudges(session),
|
|
352
|
+
}
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
# ------------------------------------------------------------------ admin
|
|
356
|
+
|
|
357
|
+
def admin(handler: Callable[[Request], Any]) -> Callable[[Request], Any]:
|
|
358
|
+
async def wrapped(request: Request) -> Response:
|
|
359
|
+
auth = request.headers.get("authorization", "")
|
|
360
|
+
if not check_bearer(
|
|
361
|
+
config.admin_token, auth[7:] if auth.lower().startswith("bearer ") else None
|
|
362
|
+
):
|
|
363
|
+
return error(401, "wspx_auth", "admin token required")
|
|
364
|
+
try:
|
|
365
|
+
result = await handler(request)
|
|
366
|
+
except (ValueError, KeyError) as exc:
|
|
367
|
+
return error(400, "wspx_bad_request", str(exc))
|
|
368
|
+
return result if isinstance(result, Response) else JSONResponse(result)
|
|
369
|
+
|
|
370
|
+
return wrapped
|
|
371
|
+
|
|
372
|
+
async def limits(request: Request) -> Any:
|
|
373
|
+
if request.method == "POST":
|
|
374
|
+
limit = Limit.model_validate(await request.json())
|
|
375
|
+
return ledger.set_limit(limit).model_dump()
|
|
376
|
+
if request.method == "DELETE":
|
|
377
|
+
return {"removed": ledger.remove_limit(request.query_params["id"])}
|
|
378
|
+
return {
|
|
379
|
+
"limits": [lim.model_dump() | {"describe": lim.describe()} for lim in ledger.limits()]
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
async def usage(request: Request) -> Any:
|
|
383
|
+
q = request.query_params
|
|
384
|
+
return ledger.usage_summary(session=q.get("session"), since=float(q.get("since", 0))) | {
|
|
385
|
+
"notices": ledger.notices(float(q.get("since", 0)))
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
async def halt(request: Request) -> Any:
|
|
389
|
+
data = await request.json() if await request.body() else {}
|
|
390
|
+
ledger.set_halt(True, data.get("reason", "kill --all"))
|
|
391
|
+
return {"halted": True}
|
|
392
|
+
|
|
393
|
+
async def resume(request: Request) -> Any:
|
|
394
|
+
data = await request.json() if await request.body() else {}
|
|
395
|
+
session = data.get("session")
|
|
396
|
+
ledger.resume(session)
|
|
397
|
+
if session:
|
|
398
|
+
loops.reset(session)
|
|
399
|
+
return {"resumed": session or "all"}
|
|
400
|
+
|
|
401
|
+
async def pause(request: Request) -> Any:
|
|
402
|
+
data = await request.json()
|
|
403
|
+
ledger.pause(request.path_params["session"], data.get("reason", "paused by user"))
|
|
404
|
+
return {"paused": request.path_params["session"]}
|
|
405
|
+
|
|
406
|
+
async def session_info(request: Request) -> Any:
|
|
407
|
+
sid = request.path_params["session"]
|
|
408
|
+
return {
|
|
409
|
+
"session": sid,
|
|
410
|
+
"paused": ledger.paused(sid),
|
|
411
|
+
"usage": ledger.usage_summary(session=sid),
|
|
412
|
+
"loop_events": ledger.loop_events(sid),
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
async def prices(request: Request) -> Any:
|
|
416
|
+
data = await request.json()
|
|
417
|
+
provider, _, model = data["model"].partition(":")
|
|
418
|
+
ledger.set_price(
|
|
419
|
+
provider,
|
|
420
|
+
model,
|
|
421
|
+
Price(
|
|
422
|
+
float(data["input"]),
|
|
423
|
+
float(data["output"]),
|
|
424
|
+
data.get("cache_read"),
|
|
425
|
+
data.get("cache_write"),
|
|
426
|
+
),
|
|
427
|
+
)
|
|
428
|
+
return {"ok": True}
|
|
429
|
+
|
|
430
|
+
async def observe(request: Request) -> Any:
|
|
431
|
+
"""The control plane reports tool calls and outcomes; repetition there also counts."""
|
|
432
|
+
data = await request.json()
|
|
433
|
+
session = data["session"]
|
|
434
|
+
verdict = loops.observe(session, data["kind"], data["key"], label=data.get("label", ""))
|
|
435
|
+
if verdict is None:
|
|
436
|
+
return {"verdict": None}
|
|
437
|
+
ledger.loop_event(session, verdict.to_dict())
|
|
438
|
+
if verdict.action == "stop":
|
|
439
|
+
ledger.pause(session, f"loop detected: {verdict.message}")
|
|
440
|
+
return {"verdict": verdict.to_dict()}
|
|
441
|
+
|
|
442
|
+
async def check_key(provider: str) -> dict[str, Any]:
|
|
443
|
+
key = config.keys.get(provider)
|
|
444
|
+
if not key:
|
|
445
|
+
return {"set": False}
|
|
446
|
+
path, extra = KEY_CHECKS[provider]
|
|
447
|
+
url = path if path.startswith("https://") else config.upstreams[provider] + path
|
|
448
|
+
try:
|
|
449
|
+
r = await client.get(url, headers=auth_headers(provider, key) | extra, timeout=20.0)
|
|
450
|
+
except httpx.HTTPError as exc:
|
|
451
|
+
return {"set": True, "ok": None, "detail": f"unreachable: {exc}"}
|
|
452
|
+
if r.status_code < 400:
|
|
453
|
+
return {"set": True, "ok": True}
|
|
454
|
+
return {"set": True, "ok": False, "detail": f"{r.status_code}: {r.text[:200]}"}
|
|
455
|
+
|
|
456
|
+
async def keys(_request: Request) -> Any:
|
|
457
|
+
"""Which provider keys this gateway holds and whether each works (never the values)."""
|
|
458
|
+
results = await asyncio.gather(*(check_key(p) for p in KEY_CHECKS))
|
|
459
|
+
return {"keys": dict(zip(KEY_CHECKS, results, strict=True))}
|
|
460
|
+
|
|
461
|
+
async def recycle(_request: Request) -> Any:
|
|
462
|
+
"""Load changed keys: on Modal, retire this container so the next call starts fresh."""
|
|
463
|
+
import modal
|
|
464
|
+
|
|
465
|
+
if modal.is_local():
|
|
466
|
+
return {"recycled": False, "detail": "restart the gateway to load changed keys"}
|
|
467
|
+
import modal.experimental
|
|
468
|
+
|
|
469
|
+
modal.experimental.stop_fetching_inputs()
|
|
470
|
+
return {"recycled": True}
|
|
471
|
+
|
|
472
|
+
async def health(_request: Request) -> Response:
|
|
473
|
+
return JSONResponse({"ok": True, "halted": ledger.halted() is not None})
|
|
474
|
+
|
|
475
|
+
routes = [
|
|
476
|
+
Route("/healthz", health),
|
|
477
|
+
Route("/v1/session", session_state),
|
|
478
|
+
Route("/v1/{provider:str}/{path:path}", proxy, methods=["POST", "GET"]),
|
|
479
|
+
Route("/admin/limits", admin(limits), methods=["GET", "POST", "DELETE"]),
|
|
480
|
+
Route("/admin/usage", admin(usage)),
|
|
481
|
+
Route("/admin/halt", admin(halt), methods=["POST"]),
|
|
482
|
+
Route("/admin/resume", admin(resume), methods=["POST"]),
|
|
483
|
+
Route("/admin/sessions/{session}/pause", admin(pause), methods=["POST"]),
|
|
484
|
+
Route("/admin/sessions/{session}", admin(session_info)),
|
|
485
|
+
Route("/admin/prices", admin(prices), methods=["POST"]),
|
|
486
|
+
Route("/admin/observe", admin(observe), methods=["POST"]),
|
|
487
|
+
Route("/admin/keys", admin(keys)),
|
|
488
|
+
Route("/admin/recycle", admin(recycle), methods=["POST"]),
|
|
489
|
+
]
|
|
490
|
+
app = Starlette(routes=routes)
|
|
491
|
+
app.state.ledger = ledger
|
|
492
|
+
app.state.loops = loops
|
|
493
|
+
return app
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
# --------------------------------------------------------------------------- request shapes
|
|
497
|
+
|
|
498
|
+
_GEMINI_RE = re.compile(r"models/(?P<model>[^/:]+):(?P<method>\w+)")
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def describe(config: GatewayConfig, provider: str, path: str, body: bytes) -> Call:
|
|
502
|
+
try:
|
|
503
|
+
data = json.loads(body) if body else {}
|
|
504
|
+
except json.JSONDecodeError:
|
|
505
|
+
data = {}
|
|
506
|
+
if provider == "compat":
|
|
507
|
+
name, _, rest = path.partition("/")
|
|
508
|
+
if name not in config.compat:
|
|
509
|
+
raise ValueError(f"no openai-compatible endpoint named {name!r}")
|
|
510
|
+
base = config.compat[name][0].rstrip("/")
|
|
511
|
+
return Call(
|
|
512
|
+
"compat:" + name,
|
|
513
|
+
f"{base}/{rest}",
|
|
514
|
+
str(data.get("model", "")),
|
|
515
|
+
bool(data.get("stream")),
|
|
516
|
+
int(data.get("max_completion_tokens") or data.get("max_tokens") or DEFAULT_MAX_OUTPUT),
|
|
517
|
+
rest.endswith(("chat/completions", "responses")),
|
|
518
|
+
)
|
|
519
|
+
if provider not in config.upstreams:
|
|
520
|
+
raise ValueError(f"unknown provider {provider!r}")
|
|
521
|
+
upstream = f"{config.upstreams[provider].rstrip('/')}/{path}"
|
|
522
|
+
if provider == "anthropic":
|
|
523
|
+
metered = path.endswith("messages")
|
|
524
|
+
return Call(
|
|
525
|
+
provider,
|
|
526
|
+
upstream,
|
|
527
|
+
str(data.get("model", "")),
|
|
528
|
+
bool(data.get("stream")),
|
|
529
|
+
int(data.get("max_tokens") or DEFAULT_MAX_OUTPUT),
|
|
530
|
+
metered,
|
|
531
|
+
)
|
|
532
|
+
if provider == "google":
|
|
533
|
+
m = _GEMINI_RE.search(path)
|
|
534
|
+
model = m.group("model") if m else ""
|
|
535
|
+
method = m.group("method") if m else ""
|
|
536
|
+
cap = (data.get("generationConfig") or {}).get("maxOutputTokens") or DEFAULT_MAX_OUTPUT
|
|
537
|
+
return Call(
|
|
538
|
+
provider,
|
|
539
|
+
upstream,
|
|
540
|
+
model,
|
|
541
|
+
method == "streamGenerateContent",
|
|
542
|
+
int(cap),
|
|
543
|
+
method in ("generateContent", "streamGenerateContent"),
|
|
544
|
+
)
|
|
545
|
+
cap = (
|
|
546
|
+
data.get("max_output_tokens") or data.get("max_completion_tokens") or data.get("max_tokens")
|
|
547
|
+
)
|
|
548
|
+
return Call(
|
|
549
|
+
provider,
|
|
550
|
+
upstream,
|
|
551
|
+
str(data.get("model", "")),
|
|
552
|
+
bool(data.get("stream")),
|
|
553
|
+
int(cap or DEFAULT_MAX_OUTPUT),
|
|
554
|
+
path.endswith(("responses", "chat/completions")),
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def input_bound(previous: tuple[bytes, int] | None, body: bytes, slack: int = 256) -> int:
|
|
559
|
+
"""An upper bound on input tokens: every token covers at least one byte.
|
|
560
|
+
|
|
561
|
+
When this body extends the previous one (a growing conversation), the bound is
|
|
562
|
+
the previous exact count plus the new bytes; otherwise it is the body length.
|
|
563
|
+
"""
|
|
564
|
+
if previous is not None:
|
|
565
|
+
prev, tokens = previous
|
|
566
|
+
cut = max(0, len(prev) - slack)
|
|
567
|
+
if len(body) >= cut and body[:cut] == prev[:cut]:
|
|
568
|
+
return tokens + (len(body) - cut)
|
|
569
|
+
return len(body)
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def auth_headers(provider: str, key: str | None) -> dict[str, str]:
|
|
573
|
+
if key is None:
|
|
574
|
+
return {}
|
|
575
|
+
if provider == "anthropic":
|
|
576
|
+
return {"x-api-key": key}
|
|
577
|
+
if provider == "google":
|
|
578
|
+
return {"x-goog-api-key": key}
|
|
579
|
+
return {"authorization": f"Bearer {key}"}
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def passthrough(headers: httpx.Headers) -> dict[str, str]:
|
|
583
|
+
drop = {"content-length", "content-encoding", "transfer-encoding", "connection"}
|
|
584
|
+
return {k: v for k, v in headers.items() if k.lower() not in drop}
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def classify_error(raw: bytes) -> tuple[str, bool]:
|
|
588
|
+
"""``(code or message, is a spend limit)``. Spend limits are never retried."""
|
|
589
|
+
try:
|
|
590
|
+
data = json.loads(raw)
|
|
591
|
+
except json.JSONDecodeError:
|
|
592
|
+
return raw[:200].decode("utf-8", "replace"), False
|
|
593
|
+
err = data.get("error") if isinstance(data, dict) else None
|
|
594
|
+
if not isinstance(err, dict):
|
|
595
|
+
return str(data)[:200], False
|
|
596
|
+
code = str(err.get("code") or err.get("type") or err.get("status") or "")
|
|
597
|
+
message = str(err.get("message") or code)
|
|
598
|
+
spend = (
|
|
599
|
+
code in SPEND_LIMIT_CODES
|
|
600
|
+
or "spend limit" in message.lower()
|
|
601
|
+
or (err.get("status") == "RESOURCE_EXHAUSTED" and "billing" in message.lower())
|
|
602
|
+
)
|
|
603
|
+
return code or message[:120], spend
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def parse_json(provider: str, path: str, data: dict[str, Any]) -> tuple[Usage, Observed]:
|
|
607
|
+
if provider == "anthropic":
|
|
608
|
+
return anthropic_json(data)
|
|
609
|
+
if provider == "google":
|
|
610
|
+
return gemini_json(data)
|
|
611
|
+
del path
|
|
612
|
+
return openai_json(data)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def stream_tracker(provider: str, path: str) -> Any:
|
|
616
|
+
del path
|
|
617
|
+
if provider == "anthropic":
|
|
618
|
+
return AnthropicStream()
|
|
619
|
+
if provider == "google":
|
|
620
|
+
return GeminiStream()
|
|
621
|
+
return OpenAIStream()
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def create_app_from_env() -> Starlette:
|
|
625
|
+
"""Factory for ``uvicorn --factory`` (self-hosting)."""
|
|
626
|
+
return create_app(GatewayConfig.from_env())
|