llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,521 @@
|
|
|
1
|
+
"""The registry document (`LLMQ_REGISTRY`) as frozen data, built with every problem collected.
|
|
2
|
+
|
|
3
|
+
`build()` turns the JSON document into a `Registry` and lists all problems at once, each with
|
|
4
|
+
its path, so one fix-and-rerun is enough. Cross-field checks that need resolved values live in
|
|
5
|
+
`registry.validate`. Unknown names are errors for the process's own config (`strict`) and
|
|
6
|
+
warnings for a stored document (read leniently, so an older pod still starts).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import difflib
|
|
12
|
+
import ipaddress
|
|
13
|
+
import re
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from types import MappingProxyType
|
|
17
|
+
from typing import Any
|
|
18
|
+
from urllib.parse import urlparse
|
|
19
|
+
|
|
20
|
+
from llm_queue.errors import ConfigError, UnknownModel
|
|
21
|
+
from llm_queue.registry.settings import SETTINGS, A, D, M, R, settings_at
|
|
22
|
+
|
|
23
|
+
NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]*$")
|
|
24
|
+
# Queue (stream) names: short and plain, so key names and metric labels stay readable.
|
|
25
|
+
QUEUE_NAME_RE = re.compile(r"^[A-Za-z0-9_-]{1,71}$") # leaves room for "-priority"
|
|
26
|
+
# Account kind -> the provider it serves. `register_account_kind` adds more.
|
|
27
|
+
ACCOUNT_KINDS: dict[str, str] = {"aws": "bedrock", "openai": "openai"}
|
|
28
|
+
MODEL_TYPES = ("chat", "embedding")
|
|
29
|
+
_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "host.docker.internal"})
|
|
30
|
+
|
|
31
|
+
_TOP = ("registry_version", "min_lib_version", "defaults", "routing", "residency")
|
|
32
|
+
_TOP_KEYS = frozenset((*_TOP, "accounts", "models"))
|
|
33
|
+
_ACCOUNT_FIELDS = frozenset(
|
|
34
|
+
{"kind", "enabled", "role_arn", "external_id_ref", "base_url", "api_key_ref", "settings"}
|
|
35
|
+
)
|
|
36
|
+
_MODEL_FIELDS = frozenset({"type", "provider", "model_id", "deployments", "residency"})
|
|
37
|
+
_DEPLOYMENT_FIELDS = frozenset({"confirmed", "fallback", "overrides"})
|
|
38
|
+
_ROUTING_FIELDS = frozenset({"kill_switch", "allow_users", "blocked_regions", "blocked_accounts"})
|
|
39
|
+
_PROFILE_FIELDS = frozenset({"name", "region", "profile_id", "routes_to"})
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def register_account_kind(kind: str, provider: str) -> None:
|
|
43
|
+
"""Allow accounts of `kind` to serve models of `provider` (for a custom provider)."""
|
|
44
|
+
if not NAME_RE.match(kind) or not NAME_RE.match(provider):
|
|
45
|
+
raise ConfigError("kind and provider must be lowercase names")
|
|
46
|
+
ACCOUNT_KINDS[kind] = provider
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _frozen(value: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
50
|
+
return MappingProxyType(dict(value))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True, slots=True)
|
|
54
|
+
class Account:
|
|
55
|
+
"""Where calls are made from: an AWS account (pod identity or assumed role) or an endpoint."""
|
|
56
|
+
|
|
57
|
+
name: str
|
|
58
|
+
kind: str
|
|
59
|
+
enabled: bool = True
|
|
60
|
+
role_arn: str | None = None
|
|
61
|
+
external_id_ref: str | None = None
|
|
62
|
+
base_url: str | None = None
|
|
63
|
+
api_key_ref: str | None = None
|
|
64
|
+
settings: Mapping[str, Any] = field(default_factory=dict)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def provider(self) -> str:
|
|
68
|
+
"""The provider this account serves."""
|
|
69
|
+
return ACCOUNT_KINDS[self.kind]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True, slots=True)
|
|
73
|
+
class RegionEntry:
|
|
74
|
+
"""One region (or inference profile) of a model in one account: a deployment before resolve.
|
|
75
|
+
|
|
76
|
+
`name` is the entry's name (the region, or the profile's own name); `region` is where the
|
|
77
|
+
client calls; `settings` are its deployment-level values (profile fields and `overrides`).
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
name: str
|
|
81
|
+
region: str
|
|
82
|
+
group: str
|
|
83
|
+
profile_id: str | None = None
|
|
84
|
+
routes_to: tuple[str, ...] = ()
|
|
85
|
+
settings: Mapping[str, Any] = field(default_factory=dict)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True, slots=True)
|
|
89
|
+
class ModelEntry:
|
|
90
|
+
"""One model (alias): its provider model id, model-level settings and region entries."""
|
|
91
|
+
|
|
92
|
+
alias: str
|
|
93
|
+
type: str
|
|
94
|
+
provider: str
|
|
95
|
+
model_id: str
|
|
96
|
+
settings: Mapping[str, Any]
|
|
97
|
+
deployments: Mapping[str, tuple[RegionEntry, ...]]
|
|
98
|
+
residency: tuple[str, ...] | None = None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(frozen=True, slots=True)
|
|
102
|
+
class Routing:
|
|
103
|
+
"""Top-level routing switches: kill switch, users always queued, live blocks."""
|
|
104
|
+
|
|
105
|
+
kill_switch: bool = False
|
|
106
|
+
allow_users: tuple[str, ...] = ()
|
|
107
|
+
blocked_regions: tuple[str, ...] = ()
|
|
108
|
+
blocked_accounts: tuple[str, ...] = ()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass(frozen=True, slots=True)
|
|
112
|
+
class Registry:
|
|
113
|
+
"""The whole registry: version, defaults, routing, residency, accounts and models.
|
|
114
|
+
|
|
115
|
+
`warnings` lists non-fatal findings (unknown names in a stored document, unused fallback
|
|
116
|
+
regions, deployments outside the residency list).
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
version: int
|
|
120
|
+
defaults: Mapping[str, Any]
|
|
121
|
+
routing: Routing
|
|
122
|
+
accounts: Mapping[str, Account]
|
|
123
|
+
models: Mapping[str, ModelEntry]
|
|
124
|
+
residency: tuple[str, ...] | None = None
|
|
125
|
+
min_lib_version: str | None = None
|
|
126
|
+
warnings: tuple[str, ...] = ()
|
|
127
|
+
raw: Mapping[str, Any] = field(default_factory=dict, repr=False)
|
|
128
|
+
|
|
129
|
+
def alias_of(self, model: str) -> str:
|
|
130
|
+
"""The alias for `model`, given as an alias or as a provider model id (`UnknownModel`)."""
|
|
131
|
+
if model in self.models:
|
|
132
|
+
return model
|
|
133
|
+
for entry in self.models.values():
|
|
134
|
+
if entry.model_id == model:
|
|
135
|
+
return entry.alias
|
|
136
|
+
raise UnknownModel(model)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class _Collector:
|
|
140
|
+
"""Gathers errors and warnings with their paths while the document is walked."""
|
|
141
|
+
|
|
142
|
+
def __init__(self, strict: bool) -> None:
|
|
143
|
+
self.strict = strict
|
|
144
|
+
self.errors: list[str] = []
|
|
145
|
+
self.warnings: list[str] = []
|
|
146
|
+
|
|
147
|
+
def error(self, path: str, message: str) -> None:
|
|
148
|
+
self.errors.append(f"{path}: {message}")
|
|
149
|
+
|
|
150
|
+
def warn(self, path: str, message: str) -> None:
|
|
151
|
+
self.warnings.append(f"{path}: {message}")
|
|
152
|
+
|
|
153
|
+
def unknown(self, path: str, name: str, known: frozenset[str], where: str) -> None:
|
|
154
|
+
"""An unknown name: error in strict mode, warning (value ignored) otherwise."""
|
|
155
|
+
close = difflib.get_close_matches(name, sorted(known), n=1)
|
|
156
|
+
hint = f"; did you mean `{close[0]}`?" if close else ""
|
|
157
|
+
if name in SETTINGS:
|
|
158
|
+
message = f"`{name}` cannot be set {where}"
|
|
159
|
+
self.error(path, message)
|
|
160
|
+
return
|
|
161
|
+
message = f"unknown name `{name}`{hint}"
|
|
162
|
+
if self.strict:
|
|
163
|
+
self.error(path, message)
|
|
164
|
+
else:
|
|
165
|
+
self.warn(path, message + " (ignored)")
|
|
166
|
+
|
|
167
|
+
def mapping(self, path: str, value: Any) -> dict[str, Any] | None:
|
|
168
|
+
if isinstance(value, dict):
|
|
169
|
+
return value
|
|
170
|
+
self.error(path, "must be an object")
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
def names(self, path: str, value: Any) -> tuple[str, ...]:
|
|
174
|
+
if isinstance(value, list) and all(isinstance(v, str) for v in value):
|
|
175
|
+
return tuple(value)
|
|
176
|
+
self.error(path, "must be a list of strings")
|
|
177
|
+
return ()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _settings(
|
|
181
|
+
out: _Collector,
|
|
182
|
+
path: str,
|
|
183
|
+
raw: Mapping[str, Any],
|
|
184
|
+
level: str,
|
|
185
|
+
skip: frozenset[str] = frozenset(),
|
|
186
|
+
) -> dict[str, Any]:
|
|
187
|
+
"""Check the settings of one level; return the valid ones."""
|
|
188
|
+
allowed = settings_at(level)
|
|
189
|
+
where = {D: "in defaults", A: "in account settings", M: "on a model", R: "on a deployment"}
|
|
190
|
+
kept: dict[str, Any] = {}
|
|
191
|
+
for name, value in raw.items():
|
|
192
|
+
if name in skip:
|
|
193
|
+
continue
|
|
194
|
+
if name not in allowed:
|
|
195
|
+
out.unknown(f"{path}.{name}", name, allowed | skip, where[level])
|
|
196
|
+
continue
|
|
197
|
+
problem = SETTINGS[name].check(value)
|
|
198
|
+
if problem:
|
|
199
|
+
out.error(f"{path}.{name}", problem)
|
|
200
|
+
continue
|
|
201
|
+
if name == "queue" and level != M and "name" in value:
|
|
202
|
+
out.error(f"{path}.queue.name", "a queue name can only be set on a model")
|
|
203
|
+
continue
|
|
204
|
+
kept[name] = value
|
|
205
|
+
return kept
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# Secrets of the process itself: never an account's key (a registry edit must not exfiltrate).
|
|
209
|
+
_PROCESS_SECRETS = frozenset(
|
|
210
|
+
{
|
|
211
|
+
"LLMQ_STORE_PASSWORD",
|
|
212
|
+
"LLMQ_QUEUE_STORE_PASSWORD",
|
|
213
|
+
"LLMQ_ID_HASH_SALT",
|
|
214
|
+
"LLMQ_STORE_URL",
|
|
215
|
+
"LLMQ_QUEUE_STORE_URL",
|
|
216
|
+
"AWS_ACCESS_KEY_ID",
|
|
217
|
+
"AWS_SECRET_ACCESS_KEY",
|
|
218
|
+
"AWS_SESSION_TOKEN",
|
|
219
|
+
}
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _is_link_local(host: str) -> bool:
|
|
224
|
+
"""169.254.0.0/16 or fe80::/10: instance metadata endpoints live there."""
|
|
225
|
+
try:
|
|
226
|
+
return ipaddress.ip_address(host.strip("[]")).is_link_local
|
|
227
|
+
except ValueError:
|
|
228
|
+
return False
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _is_local(host: str) -> bool:
|
|
232
|
+
return host in _LOCAL_HOSTS or host.endswith(".localhost") or "." not in host
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _account(
|
|
236
|
+
out: _Collector, name: str, raw: Any, allowed_hosts: tuple[str, ...]
|
|
237
|
+
) -> Account | None:
|
|
238
|
+
path = f"accounts.{name}"
|
|
239
|
+
if not NAME_RE.match(name):
|
|
240
|
+
out.error(path, "account names are lowercase letters, digits, `_`, `.`, `-`")
|
|
241
|
+
body = out.mapping(path, raw)
|
|
242
|
+
if body is None:
|
|
243
|
+
return None
|
|
244
|
+
for key in body:
|
|
245
|
+
if key not in _ACCOUNT_FIELDS:
|
|
246
|
+
out.unknown(f"{path}.{key}", key, _ACCOUNT_FIELDS, "on an account")
|
|
247
|
+
kind = body.get("kind")
|
|
248
|
+
if kind not in ACCOUNT_KINDS:
|
|
249
|
+
out.error(f"{path}.kind", f"must be one of {', '.join(sorted(ACCOUNT_KINDS))}")
|
|
250
|
+
return None
|
|
251
|
+
texts: dict[str, str | None] = {}
|
|
252
|
+
for key in ("role_arn", "external_id_ref", "base_url", "api_key_ref"):
|
|
253
|
+
value = body.get(key)
|
|
254
|
+
if value is not None and not (isinstance(value, str) and value):
|
|
255
|
+
out.error(f"{path}.{key}", "must be a non-empty string")
|
|
256
|
+
value = None
|
|
257
|
+
texts[key] = value
|
|
258
|
+
if kind != "aws":
|
|
259
|
+
for key in ("role_arn", "external_id_ref"):
|
|
260
|
+
if texts[key]:
|
|
261
|
+
out.error(f"{path}.{key}", f"only an `aws` account takes `{key}`")
|
|
262
|
+
if texts["external_id_ref"] and not texts["role_arn"]:
|
|
263
|
+
out.error(f"{path}.external_id_ref", "needs `role_arn`")
|
|
264
|
+
for key in ("external_id_ref", "api_key_ref"):
|
|
265
|
+
ref = texts[key]
|
|
266
|
+
if ref and ref in _PROCESS_SECRETS:
|
|
267
|
+
# A registry author could otherwise send the store password or the pod's AWS
|
|
268
|
+
# keys as a Bearer token to a base_url they choose.
|
|
269
|
+
out.error(f"{path}.{key}", f"`{ref}` is a process secret, not an account's")
|
|
270
|
+
if kind == "aws":
|
|
271
|
+
for key in ("base_url", "api_key_ref"):
|
|
272
|
+
if texts[key]:
|
|
273
|
+
out.error(f"{path}.{key}", f"an `aws` account does not take `{key}`")
|
|
274
|
+
if kind == "openai":
|
|
275
|
+
url = texts["base_url"]
|
|
276
|
+
if not url:
|
|
277
|
+
out.error(f"{path}.base_url", "an `openai` account needs `base_url`")
|
|
278
|
+
else:
|
|
279
|
+
parsed = urlparse(url)
|
|
280
|
+
host = parsed.hostname or ""
|
|
281
|
+
if parsed.scheme not in ("https", "http") or not host:
|
|
282
|
+
out.error(f"{path}.base_url", "must be an http(s) URL")
|
|
283
|
+
elif _is_link_local(host):
|
|
284
|
+
out.error(f"{path}.base_url", "a link-local host (instance metadata) is refused")
|
|
285
|
+
elif parsed.scheme == "http" and not _is_local(host):
|
|
286
|
+
out.error(f"{path}.base_url", "must use https unless the host is local")
|
|
287
|
+
elif allowed_hosts and host not in allowed_hosts:
|
|
288
|
+
out.error(f"{path}.base_url", f"host `{host}` is not in the allowed hosts")
|
|
289
|
+
enabled = body.get("enabled", True)
|
|
290
|
+
if not isinstance(enabled, bool):
|
|
291
|
+
out.error(f"{path}.enabled", "must be true or false")
|
|
292
|
+
enabled = True
|
|
293
|
+
settings_raw = out.mapping(f"{path}.settings", body.get("settings", {})) or {}
|
|
294
|
+
settings = _settings(out, f"{path}.settings", settings_raw, A)
|
|
295
|
+
return Account(
|
|
296
|
+
name=name,
|
|
297
|
+
kind=kind,
|
|
298
|
+
enabled=enabled,
|
|
299
|
+
role_arn=texts["role_arn"],
|
|
300
|
+
external_id_ref=texts["external_id_ref"],
|
|
301
|
+
base_url=texts["base_url"],
|
|
302
|
+
api_key_ref=texts["api_key_ref"],
|
|
303
|
+
settings=_frozen(settings),
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _region_entry(out: _Collector, path: str, raw: Any, group: str) -> RegionEntry | None:
|
|
308
|
+
if isinstance(raw, str):
|
|
309
|
+
if not NAME_RE.match(raw):
|
|
310
|
+
out.error(path, f"`{raw}` is not a valid region name")
|
|
311
|
+
return None
|
|
312
|
+
return RegionEntry(name=raw, region=raw, group=group)
|
|
313
|
+
body = out.mapping(path, raw)
|
|
314
|
+
if body is None:
|
|
315
|
+
return None
|
|
316
|
+
name, region = body.get("name"), body.get("region")
|
|
317
|
+
for key, value in (("name", name), ("region", region)):
|
|
318
|
+
if not (isinstance(value, str) and NAME_RE.match(value)):
|
|
319
|
+
out.error(f"{path}.{key}", "needs a valid name")
|
|
320
|
+
profile = body.get("profile_id")
|
|
321
|
+
if not (isinstance(profile, str) and profile):
|
|
322
|
+
out.error(f"{path}.profile_id", "an inference profile needs `profile_id`")
|
|
323
|
+
routes = out.names(f"{path}.routes_to", body.get("routes_to", []))
|
|
324
|
+
extra = {k: v for k, v in body.items() if k not in _PROFILE_FIELDS}
|
|
325
|
+
settings = _settings(out, path, extra, R)
|
|
326
|
+
if not (isinstance(name, str) and isinstance(region, str) and isinstance(profile, str)):
|
|
327
|
+
return None
|
|
328
|
+
return RegionEntry(
|
|
329
|
+
name=name,
|
|
330
|
+
region=region,
|
|
331
|
+
group=group,
|
|
332
|
+
profile_id=profile,
|
|
333
|
+
routes_to=routes,
|
|
334
|
+
settings=_frozen(settings),
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _deployments(
|
|
339
|
+
out: _Collector, path: str, raw: Any, accounts: Mapping[str, Account]
|
|
340
|
+
) -> dict[str, tuple[RegionEntry, ...]]:
|
|
341
|
+
result: dict[str, tuple[RegionEntry, ...]] = {}
|
|
342
|
+
body = out.mapping(path, raw)
|
|
343
|
+
if not body:
|
|
344
|
+
if body is not None:
|
|
345
|
+
out.error(path, "needs at least one account")
|
|
346
|
+
return result
|
|
347
|
+
for account_name, spec_raw in body.items():
|
|
348
|
+
apath = f"{path}.{account_name}"
|
|
349
|
+
if account_name not in accounts:
|
|
350
|
+
close = difflib.get_close_matches(account_name, sorted(accounts), n=1)
|
|
351
|
+
hint = f"; did you mean `{close[0]}`?" if close else ""
|
|
352
|
+
out.error(apath, f"unknown account `{account_name}`{hint}")
|
|
353
|
+
spec = out.mapping(apath, spec_raw)
|
|
354
|
+
if spec is None:
|
|
355
|
+
continue
|
|
356
|
+
for key in spec:
|
|
357
|
+
if key not in _DEPLOYMENT_FIELDS:
|
|
358
|
+
out.unknown(f"{apath}.{key}", key, _DEPLOYMENT_FIELDS, "in deployments")
|
|
359
|
+
entries: list[RegionEntry] = []
|
|
360
|
+
for group in ("confirmed", "fallback"):
|
|
361
|
+
listed = spec.get(group, [])
|
|
362
|
+
if not isinstance(listed, list):
|
|
363
|
+
out.error(f"{apath}.{group}", "must be a list")
|
|
364
|
+
continue
|
|
365
|
+
for i, item in enumerate(listed):
|
|
366
|
+
entry = _region_entry(out, f"{apath}.{group}[{i}]", item, group)
|
|
367
|
+
if entry:
|
|
368
|
+
entries.append(entry)
|
|
369
|
+
seen: dict[str, str] = {}
|
|
370
|
+
regions = {e.region for e in entries if e.profile_id is None}
|
|
371
|
+
for entry in entries:
|
|
372
|
+
if entry.name in seen:
|
|
373
|
+
where = (
|
|
374
|
+
"both confirmed and fallback" if seen[entry.name] != entry.group else "twice"
|
|
375
|
+
)
|
|
376
|
+
out.error(f"{apath}", f"`{entry.name}` is listed {where}")
|
|
377
|
+
seen[entry.name] = entry.group
|
|
378
|
+
if entry.profile_id is not None and entry.name in regions:
|
|
379
|
+
out.error(f"{apath}", f"profile name `{entry.name}` equals a region name")
|
|
380
|
+
overrides = out.mapping(f"{apath}.overrides", spec.get("overrides", {})) or {}
|
|
381
|
+
by_name = {e.name: e for e in entries}
|
|
382
|
+
for target, values_raw in overrides.items():
|
|
383
|
+
opath = f"{apath}.overrides.{target}"
|
|
384
|
+
if target not in by_name:
|
|
385
|
+
out.error(opath, f"`{target}` is not a region or profile of this account")
|
|
386
|
+
continue
|
|
387
|
+
values = out.mapping(opath, values_raw)
|
|
388
|
+
if values is None:
|
|
389
|
+
continue
|
|
390
|
+
entry = by_name[target]
|
|
391
|
+
merged = {**entry.settings, **_settings(out, opath, values, R)}
|
|
392
|
+
by_name[target] = RegionEntry(
|
|
393
|
+
name=entry.name,
|
|
394
|
+
region=entry.region,
|
|
395
|
+
group=entry.group,
|
|
396
|
+
profile_id=entry.profile_id,
|
|
397
|
+
routes_to=entry.routes_to,
|
|
398
|
+
settings=_frozen(merged),
|
|
399
|
+
)
|
|
400
|
+
result[account_name] = tuple(by_name[e.name] for e in entries if e.name in by_name)
|
|
401
|
+
return result
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _residency(out: _Collector, path: str, raw: Any) -> tuple[str, ...] | None:
|
|
405
|
+
if raw is None:
|
|
406
|
+
return None
|
|
407
|
+
body = out.mapping(path, raw)
|
|
408
|
+
if body is None:
|
|
409
|
+
return None
|
|
410
|
+
for key in body:
|
|
411
|
+
if key != "allowed_regions":
|
|
412
|
+
out.unknown(f"{path}.{key}", key, frozenset({"allowed_regions"}), "in residency")
|
|
413
|
+
return out.names(f"{path}.allowed_regions", body.get("allowed_regions", []))
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _model(
|
|
417
|
+
out: _Collector, alias: str, raw: Any, accounts: Mapping[str, Account]
|
|
418
|
+
) -> ModelEntry | None:
|
|
419
|
+
path = f"models.{alias}"
|
|
420
|
+
if not NAME_RE.match(alias):
|
|
421
|
+
out.error(path, "an alias is lowercase letters, digits, `_`, `.`, `-`")
|
|
422
|
+
body = out.mapping(path, raw)
|
|
423
|
+
if body is None:
|
|
424
|
+
return None
|
|
425
|
+
kind = body.get("type")
|
|
426
|
+
if kind not in MODEL_TYPES:
|
|
427
|
+
out.error(f"{path}.type", "must be chat or embedding")
|
|
428
|
+
provider = body.get("provider")
|
|
429
|
+
if provider not in ACCOUNT_KINDS.values():
|
|
430
|
+
known = ", ".join(sorted(set(ACCOUNT_KINDS.values())))
|
|
431
|
+
out.error(f"{path}.provider", f"must be one of {known}")
|
|
432
|
+
model_id = body.get("model_id")
|
|
433
|
+
if not (isinstance(model_id, str) and model_id):
|
|
434
|
+
out.error(f"{path}.model_id", "is required")
|
|
435
|
+
settings = _settings(out, path, body, M, skip=_MODEL_FIELDS)
|
|
436
|
+
deployments = _deployments(out, f"{path}.deployments", body.get("deployments"), accounts)
|
|
437
|
+
if not any(e.group == "confirmed" for entries in deployments.values() for e in entries):
|
|
438
|
+
out.error(f"{path}.deployments", "needs at least one confirmed region")
|
|
439
|
+
residency = _residency(out, f"{path}.residency", body.get("residency"))
|
|
440
|
+
if kind not in MODEL_TYPES or not isinstance(provider, str) or not isinstance(model_id, str):
|
|
441
|
+
return None
|
|
442
|
+
return ModelEntry(
|
|
443
|
+
alias=alias,
|
|
444
|
+
type=kind,
|
|
445
|
+
provider=provider,
|
|
446
|
+
model_id=model_id,
|
|
447
|
+
settings=_frozen(settings),
|
|
448
|
+
deployments=MappingProxyType(deployments),
|
|
449
|
+
residency=residency,
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _routing(out: _Collector, raw: Any) -> Routing:
|
|
454
|
+
body = out.mapping("routing", raw if raw is not None else {})
|
|
455
|
+
if body is None:
|
|
456
|
+
return Routing()
|
|
457
|
+
for key in body:
|
|
458
|
+
if key not in _ROUTING_FIELDS:
|
|
459
|
+
out.unknown(f"routing.{key}", key, _ROUTING_FIELDS, "in routing")
|
|
460
|
+
kill = body.get("kill_switch", False)
|
|
461
|
+
if not isinstance(kill, bool):
|
|
462
|
+
out.error("routing.kill_switch", "must be true or false")
|
|
463
|
+
kill = False
|
|
464
|
+
return Routing(
|
|
465
|
+
kill_switch=kill,
|
|
466
|
+
allow_users=out.names("routing.allow_users", body.get("allow_users", [])),
|
|
467
|
+
blocked_regions=out.names("routing.blocked_regions", body.get("blocked_regions", [])),
|
|
468
|
+
blocked_accounts=out.names("routing.blocked_accounts", body.get("blocked_accounts", [])),
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def build(
|
|
473
|
+
raw: Mapping[str, Any], *, strict: bool = True, allowed_hosts: tuple[str, ...] = ()
|
|
474
|
+
) -> tuple[Registry | None, list[str], list[str]]:
|
|
475
|
+
"""Build a `Registry` from the document; return it (None on errors), errors and warnings.
|
|
476
|
+
|
|
477
|
+
Only structural checks run here; `registry.validate.check` adds the cross-field ones.
|
|
478
|
+
"""
|
|
479
|
+
out = _Collector(strict)
|
|
480
|
+
for key in raw:
|
|
481
|
+
if key not in _TOP_KEYS:
|
|
482
|
+
out.unknown(key, key, _TOP_KEYS, "at the top level")
|
|
483
|
+
version = raw.get("registry_version")
|
|
484
|
+
if not (isinstance(version, int) and not isinstance(version, bool) and version >= 1):
|
|
485
|
+
out.error("registry_version", "is required: an integer of at least 1")
|
|
486
|
+
version = 0
|
|
487
|
+
min_lib = raw.get("min_lib_version")
|
|
488
|
+
if min_lib is not None and not (isinstance(min_lib, str) and min_lib):
|
|
489
|
+
out.error("min_lib_version", "must be a version string")
|
|
490
|
+
min_lib = None
|
|
491
|
+
defaults_raw = out.mapping("defaults", raw.get("defaults", {})) or {}
|
|
492
|
+
defaults = _settings(out, "defaults", defaults_raw, D)
|
|
493
|
+
accounts_raw = out.mapping("accounts", raw.get("accounts")) or {}
|
|
494
|
+
if not accounts_raw:
|
|
495
|
+
out.error("accounts", "needs at least one account")
|
|
496
|
+
accounts = {
|
|
497
|
+
name: account
|
|
498
|
+
for name, body in accounts_raw.items()
|
|
499
|
+
if (account := _account(out, name, body, allowed_hosts)) is not None
|
|
500
|
+
}
|
|
501
|
+
models_raw = out.mapping("models", raw.get("models")) or {}
|
|
502
|
+
if not models_raw:
|
|
503
|
+
out.error("models", "needs at least one model")
|
|
504
|
+
models = {
|
|
505
|
+
alias: model
|
|
506
|
+
for alias, body in models_raw.items()
|
|
507
|
+
if (model := _model(out, alias, body, accounts)) is not None
|
|
508
|
+
}
|
|
509
|
+
registry = Registry(
|
|
510
|
+
version=version,
|
|
511
|
+
defaults=_frozen(defaults),
|
|
512
|
+
routing=_routing(out, raw.get("routing")),
|
|
513
|
+
accounts=MappingProxyType(accounts),
|
|
514
|
+
models=MappingProxyType(models),
|
|
515
|
+
residency=_residency(out, "residency", raw.get("residency")),
|
|
516
|
+
min_lib_version=min_lib,
|
|
517
|
+
raw=_frozen(raw),
|
|
518
|
+
)
|
|
519
|
+
if out.errors:
|
|
520
|
+
return None, out.errors, out.warnings
|
|
521
|
+
return registry, out.errors, out.warnings
|