llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Map Bedrock and OpenAI SDK exceptions to `ErrorInfo` (throttle / retryable / fatal).
|
|
2
|
+
|
|
3
|
+
Decisions use error *families*, not exact class names or codes, because real services and
|
|
4
|
+
fakes spell the same failure differently (`ThrottlingException`, `Throttling`, `throttled`).
|
|
5
|
+
Used by `BedrockProvider.classify` and `OpenAICompatProvider.classify`; never imports an SDK.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from email.utils import parsedate_to_datetime
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from llm_queue.errors import ErrorInfo, ErrorKind
|
|
18
|
+
|
|
19
|
+
# Prefixes of normalised codes (lower-case letters only, "exception"/"error" suffix dropped).
|
|
20
|
+
_THROTTLE_PREFIXES = (
|
|
21
|
+
"throttl",
|
|
22
|
+
"toomanyrequests",
|
|
23
|
+
"modelnotready",
|
|
24
|
+
"servicequotaexceeded",
|
|
25
|
+
"requestlimitexceeded",
|
|
26
|
+
"provisionedthroughputexceeded",
|
|
27
|
+
"ratelimit",
|
|
28
|
+
"slowdown",
|
|
29
|
+
)
|
|
30
|
+
_RETRYABLE_PREFIXES = (
|
|
31
|
+
"serviceunavailable",
|
|
32
|
+
"internal",
|
|
33
|
+
"modeltimeout",
|
|
34
|
+
"modelerror",
|
|
35
|
+
"modelstreamerror",
|
|
36
|
+
"requesttimeout",
|
|
37
|
+
"timeout",
|
|
38
|
+
"serviceerror",
|
|
39
|
+
"unavailable",
|
|
40
|
+
)
|
|
41
|
+
_FATAL_PREFIXES = (
|
|
42
|
+
"validation",
|
|
43
|
+
"accessdenied",
|
|
44
|
+
"resourcenotfound",
|
|
45
|
+
"unrecognizedclient",
|
|
46
|
+
"notauthorized",
|
|
47
|
+
"unauthorized",
|
|
48
|
+
"expiredtoken",
|
|
49
|
+
"invalid",
|
|
50
|
+
"badrequest",
|
|
51
|
+
)
|
|
52
|
+
# Fatal families that name one target (model, profile, region, endpoint or the key for that
|
|
53
|
+
# URL), not the request: the next target may serve it. Credentials errors are not here.
|
|
54
|
+
_TARGET_FAMILIES = (
|
|
55
|
+
"resourcenotfound",
|
|
56
|
+
"accessdenied",
|
|
57
|
+
"permissiondenied",
|
|
58
|
+
"modelnotfound",
|
|
59
|
+
"notfound",
|
|
60
|
+
"endpointnotfound",
|
|
61
|
+
"unknownoperation",
|
|
62
|
+
)
|
|
63
|
+
# A validation / bad-request family counts as target-specific only when its message says so.
|
|
64
|
+
_TARGET_MESSAGE = re.compile(
|
|
65
|
+
r"model identifier is invalid|invalid model identifier|on-demand throughput"
|
|
66
|
+
r"|inference profile|model_not_found|unknown model|no such model|end of (?:its )?life"
|
|
67
|
+
r"|not (?:available|supported|enabled|accessible) (?:in|for) (?:this|the|your) region"
|
|
68
|
+
r"|\b(?:model|endpoint)\b[^.]{0,80}?\b(?:not found|does not exist|doesn't exist"
|
|
69
|
+
r"|not available|unavailable|is not supported|isn't supported)",
|
|
70
|
+
re.IGNORECASE,
|
|
71
|
+
)
|
|
72
|
+
# Request-fatal however the code is spelled: a different target would refuse it the same way.
|
|
73
|
+
_REQUEST_MESSAGE = re.compile(
|
|
74
|
+
r"too long|too large|too many (?:input )?tokens|context (?:length|window)"
|
|
75
|
+
r"|maximum (?:input|context|token)|malformed",
|
|
76
|
+
re.IGNORECASE,
|
|
77
|
+
)
|
|
78
|
+
# Built lazily from whichever modules are already imported; an SDK that was never imported
|
|
79
|
+
# cannot have raised the exception being classified.
|
|
80
|
+
_TRANSPORT_TYPES: dict[str, tuple[str, ...]] = {
|
|
81
|
+
"botocore.exceptions": (
|
|
82
|
+
"ReadTimeoutError",
|
|
83
|
+
"ConnectTimeoutError",
|
|
84
|
+
"EndpointConnectionError",
|
|
85
|
+
"ConnectionClosedError",
|
|
86
|
+
"HTTPClientError",
|
|
87
|
+
),
|
|
88
|
+
"aiohttp": ("ClientError",),
|
|
89
|
+
"httpx": ("TransportError",),
|
|
90
|
+
"openai": ("APITimeoutError", "APIConnectionError"),
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def error_family(code: str) -> str:
|
|
95
|
+
"""Normalise a provider error code: letters only, lower case, no exception suffix.
|
|
96
|
+
|
|
97
|
+
`ThrottlingException`, `throttling_exception` and `Throttling` all become `throttling`.
|
|
98
|
+
"""
|
|
99
|
+
family = re.sub(r"[^a-z]", "", code.lower())
|
|
100
|
+
for suffix in ("exception", "error", "fault"):
|
|
101
|
+
if family.endswith(suffix) and len(family) > len(suffix):
|
|
102
|
+
return family[: -len(suffix)]
|
|
103
|
+
return family
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def kind_for_code(code: str, status: int | None = None) -> ErrorKind | None:
|
|
107
|
+
"""Classify a provider error code by family, falling back to the HTTP status.
|
|
108
|
+
|
|
109
|
+
Returns None when neither the code nor the status says anything.
|
|
110
|
+
"""
|
|
111
|
+
family = error_family(code)
|
|
112
|
+
if family:
|
|
113
|
+
for prefixes, kind in (
|
|
114
|
+
(_THROTTLE_PREFIXES, ErrorKind.THROTTLE),
|
|
115
|
+
(_RETRYABLE_PREFIXES, ErrorKind.RETRYABLE),
|
|
116
|
+
(_FATAL_PREFIXES, ErrorKind.FATAL),
|
|
117
|
+
):
|
|
118
|
+
if family.startswith(prefixes):
|
|
119
|
+
return kind
|
|
120
|
+
return kind_for_status(status)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def kind_for_status(status: int | None) -> ErrorKind | None:
|
|
124
|
+
"""Classify by HTTP status alone: 429 throttle, 408/409/5xx retryable, other 4xx fatal."""
|
|
125
|
+
if status is None:
|
|
126
|
+
return None
|
|
127
|
+
if status == 429:
|
|
128
|
+
return ErrorKind.THROTTLE
|
|
129
|
+
if status in (408, 409) or status >= 500:
|
|
130
|
+
return ErrorKind.RETRYABLE
|
|
131
|
+
if 400 <= status < 500:
|
|
132
|
+
return ErrorKind.FATAL
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def parse_retry_after(
|
|
137
|
+
headers: Mapping[str, Any] | None, *, now: float | None = None
|
|
138
|
+
) -> float | None:
|
|
139
|
+
"""Seconds to wait from `retry-after-ms`, else `Retry-After` (seconds or an HTTP date).
|
|
140
|
+
|
|
141
|
+
Header names are matched case-blind; unparsable values are ignored and past dates give 0.
|
|
142
|
+
"""
|
|
143
|
+
if not headers:
|
|
144
|
+
return None
|
|
145
|
+
lowered = {str(k).lower(): str(v).strip() for k, v in headers.items()}
|
|
146
|
+
if (raw_ms := lowered.get("retry-after-ms")) is not None:
|
|
147
|
+
try:
|
|
148
|
+
return max(float(raw_ms) / 1000.0, 0.0)
|
|
149
|
+
except ValueError:
|
|
150
|
+
pass
|
|
151
|
+
raw = lowered.get("retry-after")
|
|
152
|
+
if raw is None:
|
|
153
|
+
return None
|
|
154
|
+
try:
|
|
155
|
+
return max(float(raw), 0.0)
|
|
156
|
+
except ValueError:
|
|
157
|
+
pass
|
|
158
|
+
try:
|
|
159
|
+
when = parsedate_to_datetime(raw)
|
|
160
|
+
except (TypeError, ValueError, IndexError):
|
|
161
|
+
return None
|
|
162
|
+
current = time.time() if now is None else now
|
|
163
|
+
return max(when.timestamp() - current, 0.0)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def is_transport_error(exc: BaseException) -> bool:
|
|
167
|
+
"""True for timeouts and connection failures from asyncio, botocore, aiohttp, httpx, openai."""
|
|
168
|
+
if isinstance(exc, TimeoutError | ConnectionError):
|
|
169
|
+
return True
|
|
170
|
+
for module_name, names in _TRANSPORT_TYPES.items():
|
|
171
|
+
module = sys.modules.get(module_name)
|
|
172
|
+
if module is None:
|
|
173
|
+
continue
|
|
174
|
+
types = tuple(t for n in names if isinstance(t := getattr(module, n, None), type))
|
|
175
|
+
if types and isinstance(exc, types):
|
|
176
|
+
return True
|
|
177
|
+
return False
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def is_target_error(exc: BaseException, info: ErrorInfo) -> bool:
|
|
181
|
+
"""True when a FATAL error is about the target, so `run_call` tries the next fallback step.
|
|
182
|
+
|
|
183
|
+
Model or profile not found / not in this region, access denied for the model, unknown
|
|
184
|
+
endpoint (404). Bad payloads, too-long input and credentials errors end the call.
|
|
185
|
+
"""
|
|
186
|
+
if info.kind is not ErrorKind.FATAL:
|
|
187
|
+
return False
|
|
188
|
+
message = _error_message(exc)
|
|
189
|
+
if _REQUEST_MESSAGE.search(message):
|
|
190
|
+
return False
|
|
191
|
+
family = error_family(info.code)
|
|
192
|
+
if family.startswith(_TARGET_FAMILIES) or _error_status(exc) == 404:
|
|
193
|
+
return True
|
|
194
|
+
request_family = family.startswith(("validation", "badrequest", "invalid"))
|
|
195
|
+
return (request_family or _error_status(exc) == 400) and bool(_TARGET_MESSAGE.search(message))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _error_message(exc: BaseException) -> str:
|
|
199
|
+
"""A botocore `Error.Message`, else the exception text (openai puts the body there)."""
|
|
200
|
+
response = getattr(exc, "response", None)
|
|
201
|
+
if isinstance(response, Mapping) and isinstance(response.get("Error"), Mapping):
|
|
202
|
+
return str(response["Error"].get("Message") or "")
|
|
203
|
+
return str(exc)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _error_status(exc: BaseException) -> int | None:
|
|
207
|
+
"""HTTP status of a botocore `ClientError` or an openai `APIStatusError`, if any."""
|
|
208
|
+
status = getattr(exc, "status_code", None)
|
|
209
|
+
response = getattr(exc, "response", None)
|
|
210
|
+
if status is None and isinstance(response, Mapping):
|
|
211
|
+
status = (response.get("ResponseMetadata") or {}).get("HTTPStatusCode")
|
|
212
|
+
return status if isinstance(status, int) else None
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def classify_bedrock(exc: BaseException) -> ErrorInfo:
|
|
216
|
+
"""Classify a botocore `ClientError` (or transport error) from a Bedrock runtime call.
|
|
217
|
+
|
|
218
|
+
Unknown codes fall back to the HTTP status; anything else that is not a transport error is
|
|
219
|
+
fatal, so a bug in the request never loops through retries.
|
|
220
|
+
"""
|
|
221
|
+
response = getattr(exc, "response", None)
|
|
222
|
+
if isinstance(response, Mapping) and isinstance(response.get("Error"), Mapping):
|
|
223
|
+
code = str(response["Error"].get("Code") or "")
|
|
224
|
+
meta = response.get("ResponseMetadata") or {}
|
|
225
|
+
status = meta.get("HTTPStatusCode")
|
|
226
|
+
kind = kind_for_code(code, status if isinstance(status, int) else None)
|
|
227
|
+
retry_after = parse_retry_after(meta.get("HTTPHeaders"))
|
|
228
|
+
return ErrorInfo(kind=kind or ErrorKind.FATAL, code=code, retry_after_secs=retry_after)
|
|
229
|
+
if is_transport_error(exc):
|
|
230
|
+
return ErrorInfo(kind=ErrorKind.RETRYABLE, code=type(exc).__name__)
|
|
231
|
+
return ErrorInfo(kind=ErrorKind.FATAL, code=type(exc).__name__)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def classify_openai(exc: BaseException) -> ErrorInfo:
|
|
235
|
+
"""Classify an openai SDK exception (or transport error) from an OpenAI-compatible server.
|
|
236
|
+
|
|
237
|
+
429 is a throttle carrying the server's advised wait, except `insufficient_quota`, which
|
|
238
|
+
no wait fixes. 408/409/5xx, timeouts and connection errors are retryable; other 4xx fatal.
|
|
239
|
+
"""
|
|
240
|
+
if is_transport_error(exc):
|
|
241
|
+
return ErrorInfo(kind=ErrorKind.RETRYABLE, code=type(exc).__name__)
|
|
242
|
+
status = getattr(exc, "status_code", None)
|
|
243
|
+
if not isinstance(status, int):
|
|
244
|
+
return ErrorInfo(kind=ErrorKind.FATAL, code=type(exc).__name__)
|
|
245
|
+
code = str(getattr(exc, "code", None) or getattr(exc, "type", None) or type(exc).__name__)
|
|
246
|
+
response = getattr(exc, "response", None)
|
|
247
|
+
retry_after = parse_retry_after(getattr(response, "headers", None))
|
|
248
|
+
kind = kind_for_status(status) or ErrorKind.FATAL
|
|
249
|
+
if kind is ErrorKind.THROTTLE and code == "insufficient_quota":
|
|
250
|
+
kind = ErrorKind.FATAL
|
|
251
|
+
return ErrorInfo(kind=kind, code=code, retry_after_secs=retry_after)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Embedding batching limits and Bedrock request-body builders (Titan, Cohere).
|
|
2
|
+
|
|
3
|
+
An embed call is split into tasks of at most `max_batch_size` texts: Titan takes one
|
|
4
|
+
text per request, Cohere v3 up to 96, OpenAI `chunk_size` (2048 by default).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
10
|
+
from typing import Any, Literal, TypeVar
|
|
11
|
+
|
|
12
|
+
_T = TypeVar("_T") # PEP 695 syntax needs 3.12; the package supports 3.11
|
|
13
|
+
|
|
14
|
+
TITAN_MAX_BATCH = 1
|
|
15
|
+
COHERE_MAX_BATCH = 96
|
|
16
|
+
OPENAI_DEFAULT_CHUNK_SIZE = 2048
|
|
17
|
+
|
|
18
|
+
CohereInputType = Literal["search_document", "search_query", "classification", "clustering"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def max_batch_size(provider: str, model_id: str, *, chunk_size: int | None = None) -> int:
|
|
22
|
+
"""Largest number of texts one embedding request may carry for this provider and model.
|
|
23
|
+
|
|
24
|
+
Bedrock models other than Cohere are treated like Titan (one text per request).
|
|
25
|
+
"""
|
|
26
|
+
if provider == "openai":
|
|
27
|
+
return chunk_size or OPENAI_DEFAULT_CHUNK_SIZE
|
|
28
|
+
return COHERE_MAX_BATCH if "cohere" in model_id.lower() else TITAN_MAX_BATCH
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def batches(items: Sequence[_T], size: int) -> Iterator[list[_T]]:
|
|
32
|
+
"""Yield consecutive slices of at most `size` items, preserving order."""
|
|
33
|
+
if size < 1:
|
|
34
|
+
raise ValueError("batch size must be >= 1")
|
|
35
|
+
for start in range(0, len(items), size):
|
|
36
|
+
yield list(items[start : start + size])
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def titan_body(
|
|
40
|
+
text: str, *, dimensions: int | None = None, normalize: bool | None = None
|
|
41
|
+
) -> dict[str, Any]:
|
|
42
|
+
"""InvokeModel body for Amazon Titan text embeddings (v2 accepts dimensions/normalize)."""
|
|
43
|
+
body: dict[str, Any] = {"inputText": text}
|
|
44
|
+
if dimensions is not None:
|
|
45
|
+
body["dimensions"] = dimensions
|
|
46
|
+
if normalize is not None:
|
|
47
|
+
body["normalize"] = normalize
|
|
48
|
+
return body
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def cohere_body(
|
|
52
|
+
texts: Sequence[str],
|
|
53
|
+
*,
|
|
54
|
+
input_type: CohereInputType = "search_document",
|
|
55
|
+
truncate: Literal["NONE", "START", "END"] | None = None,
|
|
56
|
+
) -> dict[str, Any]:
|
|
57
|
+
"""InvokeModel body for Cohere embed v3 on Bedrock (at most 96 texts)."""
|
|
58
|
+
if len(texts) > COHERE_MAX_BATCH:
|
|
59
|
+
raise ValueError(f"Cohere takes at most {COHERE_MAX_BATCH} texts, got {len(texts)}")
|
|
60
|
+
body: dict[str, Any] = {"texts": list(texts), "input_type": input_type}
|
|
61
|
+
if truncate is not None:
|
|
62
|
+
body["truncate"] = truncate
|
|
63
|
+
return body
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def embedding_body(model_id: str, texts: Sequence[str], **options: Any) -> dict[str, Any]:
|
|
67
|
+
"""Pick the Titan or Cohere body builder by model id; Titan takes exactly one text."""
|
|
68
|
+
if "cohere" in model_id.lower():
|
|
69
|
+
return cohere_body(texts, **options)
|
|
70
|
+
if len(texts) != TITAN_MAX_BATCH:
|
|
71
|
+
raise ValueError(f"Titan takes exactly one text per request, got {len(texts)}")
|
|
72
|
+
return titan_body(texts[0], **options)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def parse_embeddings(decoded: Mapping[str, Any]) -> list[list[float]]:
|
|
76
|
+
"""Vectors from a decoded Titan (`embedding`) or Cohere (`embeddings`) response body.
|
|
77
|
+
|
|
78
|
+
Cohere's typed form (`{"embeddings": {"float": [...]}}`) is accepted too.
|
|
79
|
+
"""
|
|
80
|
+
if "embedding" in decoded:
|
|
81
|
+
return [list(decoded["embedding"])]
|
|
82
|
+
vectors = decoded.get("embeddings")
|
|
83
|
+
if isinstance(vectors, Mapping):
|
|
84
|
+
vectors = vectors.get("float")
|
|
85
|
+
if not isinstance(vectors, list):
|
|
86
|
+
raise ValueError("response has no 'embedding' or 'embeddings'")
|
|
87
|
+
return [list(v) for v in vectors]
|
llm_queue/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""The model registry: one document (`LLMQ_REGISTRY`) for accounts, models and their settings.
|
|
2
|
+
|
|
3
|
+
`load_registry()` parses and checks it and returns a `ResolvedRegistry` that answers "which
|
|
4
|
+
deployments does this model have" and "what is this setting here, and where did it come from".
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from llm_queue.registry.resolve import Deployment, Effective, ResolvedRegistry
|
|
10
|
+
from llm_queue.registry.schema import (
|
|
11
|
+
Account,
|
|
12
|
+
ModelEntry,
|
|
13
|
+
RegionEntry,
|
|
14
|
+
Registry,
|
|
15
|
+
Routing,
|
|
16
|
+
register_account_kind,
|
|
17
|
+
)
|
|
18
|
+
from llm_queue.registry.settings import SETTINGS, Spec
|
|
19
|
+
from llm_queue.registry.validate import RegistryError, check, load_registry
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"SETTINGS",
|
|
23
|
+
"Account",
|
|
24
|
+
"Deployment",
|
|
25
|
+
"Effective",
|
|
26
|
+
"ModelEntry",
|
|
27
|
+
"RegionEntry",
|
|
28
|
+
"Registry",
|
|
29
|
+
"RegistryError",
|
|
30
|
+
"ResolvedRegistry",
|
|
31
|
+
"Routing",
|
|
32
|
+
"Spec",
|
|
33
|
+
"check",
|
|
34
|
+
"load_registry",
|
|
35
|
+
"register_account_kind",
|
|
36
|
+
]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Lua scripts of the registry store: publish by version rule, and compare-and-set live edits.
|
|
2
|
+
|
|
3
|
+
Both write the document, its version record, the audit entry and each model's resolved
|
|
4
|
+
numbers in one script call, so a reader never sees half an update. The state store runs as one
|
|
5
|
+
primary (cluster mode is not supported), so the keys need not share a hash tag.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
_COMMON = """
|
|
11
|
+
local t = redis.call('TIME')
|
|
12
|
+
local now = tonumber(t[1]) + tonumber(t[2]) / 1000000
|
|
13
|
+
local now_ms = math.floor(now * 1000)
|
|
14
|
+
local function audit(actor, op, version, diff, retention_ms)
|
|
15
|
+
redis.call('XADD', KEYS[3], '*', 'actor', actor, 'op', op, 'version', version,
|
|
16
|
+
'diff', diff, 'ts', tostring(now))
|
|
17
|
+
redis.call('XTRIM', KEYS[3], 'MINID', tostring(now_ms - tonumber(retention_ms)))
|
|
18
|
+
end
|
|
19
|
+
local function write_models(first_key, first_arg, changes)
|
|
20
|
+
for i = first_key, #KEYS do
|
|
21
|
+
redis.call('HSET', KEYS[i], 'changes', changes, 'numbers', ARGV[first_arg + i - first_key])
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
# KEYS: doc, meta (hash), audit (stream), history (hash version -> doc), then one numbers key
|
|
27
|
+
# per model. ARGV: doc, version, config hash, actor, diff JSON, audit retention ms, history size,
|
|
28
|
+
# then one numbers JSON per model key.
|
|
29
|
+
# Reply: {action, stored version, stored doc or ''} with action published | kept | not_bumped |
|
|
30
|
+
# ignored. Rule (plan §3.8): higher replaces, equal keeps the stored one (and its live edits),
|
|
31
|
+
# equal with another content hash keeps it and reports not_bumped, lower is ignored.
|
|
32
|
+
PUBLISH = (
|
|
33
|
+
_COMMON
|
|
34
|
+
+ """
|
|
35
|
+
local version = tonumber(ARGV[2])
|
|
36
|
+
local stored_version = tonumber(redis.call('HGET', KEYS[2], 'version') or '0')
|
|
37
|
+
local stored_doc = redis.call('GET', KEYS[1])
|
|
38
|
+
if stored_doc and stored_version >= version then
|
|
39
|
+
if stored_version > version then return {'ignored', stored_version, stored_doc} end
|
|
40
|
+
if redis.call('HGET', KEYS[2], 'config_hash') == ARGV[3] then
|
|
41
|
+
return {'kept', stored_version, stored_doc}
|
|
42
|
+
end
|
|
43
|
+
return {'not_bumped', stored_version, stored_doc}
|
|
44
|
+
end
|
|
45
|
+
local changes = redis.call('HINCRBY', KEYS[2], 'changes', 1)
|
|
46
|
+
redis.call('SET', KEYS[1], ARGV[1])
|
|
47
|
+
redis.call('HSET', KEYS[2], 'version', version, 'config_hash', ARGV[3], 'published', tostring(now))
|
|
48
|
+
redis.call('HSET', KEYS[4], tostring(version), ARGV[1])
|
|
49
|
+
local keep = tonumber(ARGV[7])
|
|
50
|
+
local versions = redis.call('HKEYS', KEYS[4])
|
|
51
|
+
if #versions > keep then
|
|
52
|
+
table.sort(versions, function(a, b) return tonumber(a) < tonumber(b) end)
|
|
53
|
+
for i = 1, #versions - keep do redis.call('HDEL', KEYS[4], versions[i]) end
|
|
54
|
+
end
|
|
55
|
+
audit(ARGV[4], 'publish', version, ARGV[5], ARGV[6])
|
|
56
|
+
write_models(5, 8, changes)
|
|
57
|
+
return {'published', version, ''}
|
|
58
|
+
"""
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# KEYS: doc, meta, audit, history, then numbers keys. ARGV: doc, expected changes, actor, diff
|
|
62
|
+
# JSON, audit retention ms, then numbers JSONs. Reply: {'ok', changes} or {'conflict', changes}
|
|
63
|
+
# when someone else changed the registry since it was read.
|
|
64
|
+
EDIT = (
|
|
65
|
+
_COMMON
|
|
66
|
+
+ """
|
|
67
|
+
local changes = tonumber(redis.call('HGET', KEYS[2], 'changes') or '0')
|
|
68
|
+
if changes ~= tonumber(ARGV[2]) or not redis.call('GET', KEYS[1]) then
|
|
69
|
+
return {'conflict', changes}
|
|
70
|
+
end
|
|
71
|
+
changes = redis.call('HINCRBY', KEYS[2], 'changes', 1)
|
|
72
|
+
redis.call('SET', KEYS[1], ARGV[1])
|
|
73
|
+
local version = redis.call('HGET', KEYS[2], 'version') or '0'
|
|
74
|
+
audit(ARGV[3], 'edit', version, ARGV[4], ARGV[5])
|
|
75
|
+
write_models(5, 6, changes)
|
|
76
|
+
return {'ok', changes}
|
|
77
|
+
"""
|
|
78
|
+
)
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Resolve a `Registry`: the deployments of each model and every setting's effective value.
|
|
2
|
+
|
|
3
|
+
A setting is taken from the most specific level that sets it: deployment → model → account →
|
|
4
|
+
`defaults` → library default. Object settings marked `merge` combine key by key. Used by the
|
|
5
|
+
validator, the admission script inputs and `llm-queue settings get`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from llm_queue.registry.schema import ModelEntry, Registry
|
|
15
|
+
from llm_queue.registry.settings import (
|
|
16
|
+
CHAT_INFERENCE,
|
|
17
|
+
EMBED_INFERENCE,
|
|
18
|
+
FALLBACK_AFTER_SECS,
|
|
19
|
+
MODEL_FALLBACK_AFTER_SECS,
|
|
20
|
+
SETTINGS,
|
|
21
|
+
A,
|
|
22
|
+
D,
|
|
23
|
+
M,
|
|
24
|
+
R,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
LIBRARY = "library"
|
|
28
|
+
_LEVEL_LABEL = {R: "deployment", M: "model", A: "account", D: "defaults"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True, slots=True)
|
|
32
|
+
class Deployment:
|
|
33
|
+
"""One model in one account and region (or inference profile): what has a budget.
|
|
34
|
+
|
|
35
|
+
`key` (`account/name`) names it in keys, metrics and status.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
model: str
|
|
39
|
+
account: str
|
|
40
|
+
name: str
|
|
41
|
+
group: str
|
|
42
|
+
region: str
|
|
43
|
+
profile_id: str | None = None
|
|
44
|
+
routes_to: tuple[str, ...] = ()
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def key(self) -> str:
|
|
48
|
+
"""`account/name`, unique inside a model."""
|
|
49
|
+
return f"{self.account}/{self.name}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True, slots=True)
|
|
53
|
+
class Effective:
|
|
54
|
+
"""A setting's value and the level it came from (`deployment` … `defaults`, `library`)."""
|
|
55
|
+
|
|
56
|
+
value: Any
|
|
57
|
+
level: str
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ResolvedRegistry:
|
|
61
|
+
"""Read view over a `Registry`: deployments per model and effective settings.
|
|
62
|
+
|
|
63
|
+
Pure and cheap to build; rebuilt on every registry reload.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def __init__(self, registry: Registry) -> None:
|
|
67
|
+
"""Index the registry's models and deployments."""
|
|
68
|
+
self.registry = registry
|
|
69
|
+
self._deployments: dict[str, tuple[Deployment, ...]] = {}
|
|
70
|
+
self._entries: dict[tuple[str, str], Mapping[str, Any]] = {}
|
|
71
|
+
for alias, model in registry.models.items():
|
|
72
|
+
found: list[Deployment] = []
|
|
73
|
+
for group in ("confirmed", "fallback"):
|
|
74
|
+
for account, entries in model.deployments.items():
|
|
75
|
+
for entry in entries:
|
|
76
|
+
if entry.group != group:
|
|
77
|
+
continue
|
|
78
|
+
deployment = Deployment(
|
|
79
|
+
model=alias,
|
|
80
|
+
account=account,
|
|
81
|
+
name=entry.name,
|
|
82
|
+
group=entry.group,
|
|
83
|
+
region=entry.region,
|
|
84
|
+
profile_id=entry.profile_id,
|
|
85
|
+
routes_to=entry.routes_to,
|
|
86
|
+
)
|
|
87
|
+
found.append(deployment)
|
|
88
|
+
self._entries[(alias, deployment.key)] = entry.settings
|
|
89
|
+
self._deployments[alias] = tuple(found)
|
|
90
|
+
|
|
91
|
+
def model(self, model: str) -> ModelEntry:
|
|
92
|
+
"""The entry of a model named by alias or provider model id."""
|
|
93
|
+
return self.registry.models[self.registry.alias_of(model)]
|
|
94
|
+
|
|
95
|
+
def deployments(self, model: str, group: str | None = None) -> tuple[Deployment, ...]:
|
|
96
|
+
"""Deployments of a model, confirmed first, in config order; optionally one group."""
|
|
97
|
+
found = self._deployments[self.registry.alias_of(model)]
|
|
98
|
+
return found if group is None else tuple(d for d in found if d.group == group)
|
|
99
|
+
|
|
100
|
+
def _layers(self, alias: str, deployment: str | None) -> list[tuple[str, Mapping[str, Any]]]:
|
|
101
|
+
"""The settings of each level, most specific first."""
|
|
102
|
+
model = self.registry.models[alias]
|
|
103
|
+
layers: list[tuple[str, Mapping[str, Any]]] = []
|
|
104
|
+
account = None
|
|
105
|
+
if deployment is not None:
|
|
106
|
+
if (alias, deployment) not in self._entries:
|
|
107
|
+
raise KeyError(f"{alias} has no deployment {deployment}")
|
|
108
|
+
layers.append((R, self._entries[(alias, deployment)]))
|
|
109
|
+
account = deployment.split("/", 1)[0]
|
|
110
|
+
layers.append((M, model.settings))
|
|
111
|
+
if account is not None:
|
|
112
|
+
layers.append((A, self.registry.accounts[account].settings))
|
|
113
|
+
layers.append((D, self.registry.defaults))
|
|
114
|
+
return layers
|
|
115
|
+
|
|
116
|
+
def effective(self, model: str, name: str, deployment: str | None = None) -> Effective:
|
|
117
|
+
"""The value of setting `name` for a model (and one deployment), with its level.
|
|
118
|
+
|
|
119
|
+
Without `deployment`, the deployment and account levels are skipped.
|
|
120
|
+
"""
|
|
121
|
+
spec = SETTINGS[name]
|
|
122
|
+
alias = self.registry.alias_of(model)
|
|
123
|
+
entry = self.registry.models[alias]
|
|
124
|
+
layers = self._layers(alias, deployment)
|
|
125
|
+
if spec.merge:
|
|
126
|
+
merged: dict[str, Any] = dict(spec.default or {})
|
|
127
|
+
level = LIBRARY
|
|
128
|
+
for code, values in reversed(layers):
|
|
129
|
+
if name in values:
|
|
130
|
+
part = values[name]
|
|
131
|
+
if name == "inference" and code in (D, A):
|
|
132
|
+
# Shared defaults apply only to models of the matching type.
|
|
133
|
+
own = CHAT_INFERENCE if entry.type == "chat" else EMBED_INFERENCE
|
|
134
|
+
part = {k: v for k, v in part.items() if k in own}
|
|
135
|
+
if not part:
|
|
136
|
+
continue
|
|
137
|
+
merged.update(part)
|
|
138
|
+
level = _LEVEL_LABEL[code]
|
|
139
|
+
return Effective(_complete(name, merged), level)
|
|
140
|
+
for code, values in layers:
|
|
141
|
+
if name in values:
|
|
142
|
+
return Effective(_complete(name, values[name]), _LEVEL_LABEL[code])
|
|
143
|
+
return Effective(self._library_default(entry, name, deployment), LIBRARY)
|
|
144
|
+
|
|
145
|
+
def value(self, model: str, name: str, deployment: str | None = None) -> Any:
|
|
146
|
+
"""Shortcut for `effective(...).value`."""
|
|
147
|
+
return self.effective(model, name, deployment).value
|
|
148
|
+
|
|
149
|
+
def _library_default(self, entry: ModelEntry, name: str, deployment: str | None) -> Any:
|
|
150
|
+
spec = SETTINGS[name]
|
|
151
|
+
if name == "rate_limits" and entry.type == "embedding":
|
|
152
|
+
# A `tpm` on an embedding model turns token limiting on (plan §3.19).
|
|
153
|
+
has_tpm = self.effective(entry.alias, "tpm", deployment).value is not None
|
|
154
|
+
return "both" if has_tpm else "rpm"
|
|
155
|
+
if spec.by_type is not None:
|
|
156
|
+
return spec.by_type(entry.type)
|
|
157
|
+
return spec.default
|
|
158
|
+
|
|
159
|
+
def table(self, model: str, deployment: str | None = None) -> dict[str, Effective]:
|
|
160
|
+
"""Every setting of a model (and deployment) with its value and level (CLI view)."""
|
|
161
|
+
alias = self.registry.alias_of(model)
|
|
162
|
+
return {name: self.effective(alias, name, deployment) for name in sorted(SETTINGS)}
|
|
163
|
+
|
|
164
|
+
def queue_name(self, model: str) -> str:
|
|
165
|
+
"""The model's normal queue name, without the environment prefix."""
|
|
166
|
+
alias = self.registry.alias_of(model)
|
|
167
|
+
named = self.registry.models[alias].settings.get("queue", {}).get("name")
|
|
168
|
+
return str(named) if named else f"llm-{alias}"
|
|
169
|
+
|
|
170
|
+
def residency(self, model: str) -> tuple[str, ...] | None:
|
|
171
|
+
"""The regions a model may reach (its own list, else the top-level one), None = any."""
|
|
172
|
+
entry = self.model(model)
|
|
173
|
+
return entry.residency if entry.residency is not None else self.registry.residency
|
|
174
|
+
|
|
175
|
+
def outside_residency(self, deployment: Deployment) -> bool:
|
|
176
|
+
"""True when a deployment may not be used under the residency list."""
|
|
177
|
+
allowed = self.residency(deployment.model)
|
|
178
|
+
if allowed is None:
|
|
179
|
+
return False
|
|
180
|
+
if deployment.profile_id is not None:
|
|
181
|
+
return not deployment.routes_to or not set(deployment.routes_to) <= set(allowed)
|
|
182
|
+
return deployment.region not in allowed
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _complete(name: str, value: Any) -> Any:
|
|
186
|
+
"""Fill the optional fields of the two fallback rules and `sticky`."""
|
|
187
|
+
if name == "fallback" and isinstance(value, dict):
|
|
188
|
+
return {"after_secs": FALLBACK_AFTER_SECS, "regions": "account", **value}
|
|
189
|
+
if name == "model_fallback" and isinstance(value, dict):
|
|
190
|
+
models = [
|
|
191
|
+
{"model": m, "require_headroom": False}
|
|
192
|
+
if isinstance(m, str)
|
|
193
|
+
else {"require_headroom": False, **m}
|
|
194
|
+
for m in value.get("models", [])
|
|
195
|
+
]
|
|
196
|
+
return {"after_secs": MODEL_FALLBACK_AFTER_SECS, **value, "models": models}
|
|
197
|
+
if name == "sticky" and isinstance(value, dict):
|
|
198
|
+
return {"idle_ttl_secs": 1800, **value}
|
|
199
|
+
return value
|