llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,251 @@
1
+ """Map Bedrock and OpenAI SDK exceptions to `ErrorInfo` (throttle / retryable / fatal).
2
+
3
+ Decisions use error *families*, not exact class names or codes, because real services and
4
+ fakes spell the same failure differently (`ThrottlingException`, `Throttling`, `throttled`).
5
+ Used by `BedrockProvider.classify` and `OpenAICompatProvider.classify`; never imports an SDK.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ import sys
12
+ import time
13
+ from collections.abc import Mapping
14
+ from email.utils import parsedate_to_datetime
15
+ from typing import Any
16
+
17
+ from llm_queue.errors import ErrorInfo, ErrorKind
18
+
19
+ # Prefixes of normalised codes (lower-case letters only, "exception"/"error" suffix dropped).
20
+ _THROTTLE_PREFIXES = (
21
+ "throttl",
22
+ "toomanyrequests",
23
+ "modelnotready",
24
+ "servicequotaexceeded",
25
+ "requestlimitexceeded",
26
+ "provisionedthroughputexceeded",
27
+ "ratelimit",
28
+ "slowdown",
29
+ )
30
+ _RETRYABLE_PREFIXES = (
31
+ "serviceunavailable",
32
+ "internal",
33
+ "modeltimeout",
34
+ "modelerror",
35
+ "modelstreamerror",
36
+ "requesttimeout",
37
+ "timeout",
38
+ "serviceerror",
39
+ "unavailable",
40
+ )
41
+ _FATAL_PREFIXES = (
42
+ "validation",
43
+ "accessdenied",
44
+ "resourcenotfound",
45
+ "unrecognizedclient",
46
+ "notauthorized",
47
+ "unauthorized",
48
+ "expiredtoken",
49
+ "invalid",
50
+ "badrequest",
51
+ )
52
+ # Fatal families that name one target (model, profile, region, endpoint or the key for that
53
+ # URL), not the request: the next target may serve it. Credentials errors are not here.
54
+ _TARGET_FAMILIES = (
55
+ "resourcenotfound",
56
+ "accessdenied",
57
+ "permissiondenied",
58
+ "modelnotfound",
59
+ "notfound",
60
+ "endpointnotfound",
61
+ "unknownoperation",
62
+ )
63
+ # A validation / bad-request family counts as target-specific only when its message says so.
64
+ _TARGET_MESSAGE = re.compile(
65
+ r"model identifier is invalid|invalid model identifier|on-demand throughput"
66
+ r"|inference profile|model_not_found|unknown model|no such model|end of (?:its )?life"
67
+ r"|not (?:available|supported|enabled|accessible) (?:in|for) (?:this|the|your) region"
68
+ r"|\b(?:model|endpoint)\b[^.]{0,80}?\b(?:not found|does not exist|doesn't exist"
69
+ r"|not available|unavailable|is not supported|isn't supported)",
70
+ re.IGNORECASE,
71
+ )
72
+ # Request-fatal however the code is spelled: a different target would refuse it the same way.
73
+ _REQUEST_MESSAGE = re.compile(
74
+ r"too long|too large|too many (?:input )?tokens|context (?:length|window)"
75
+ r"|maximum (?:input|context|token)|malformed",
76
+ re.IGNORECASE,
77
+ )
78
+ # Built lazily from whichever modules are already imported; an SDK that was never imported
79
+ # cannot have raised the exception being classified.
80
+ _TRANSPORT_TYPES: dict[str, tuple[str, ...]] = {
81
+ "botocore.exceptions": (
82
+ "ReadTimeoutError",
83
+ "ConnectTimeoutError",
84
+ "EndpointConnectionError",
85
+ "ConnectionClosedError",
86
+ "HTTPClientError",
87
+ ),
88
+ "aiohttp": ("ClientError",),
89
+ "httpx": ("TransportError",),
90
+ "openai": ("APITimeoutError", "APIConnectionError"),
91
+ }
92
+
93
+
94
+ def error_family(code: str) -> str:
95
+ """Normalise a provider error code: letters only, lower case, no exception suffix.
96
+
97
+ `ThrottlingException`, `throttling_exception` and `Throttling` all become `throttling`.
98
+ """
99
+ family = re.sub(r"[^a-z]", "", code.lower())
100
+ for suffix in ("exception", "error", "fault"):
101
+ if family.endswith(suffix) and len(family) > len(suffix):
102
+ return family[: -len(suffix)]
103
+ return family
104
+
105
+
106
+ def kind_for_code(code: str, status: int | None = None) -> ErrorKind | None:
107
+ """Classify a provider error code by family, falling back to the HTTP status.
108
+
109
+ Returns None when neither the code nor the status says anything.
110
+ """
111
+ family = error_family(code)
112
+ if family:
113
+ for prefixes, kind in (
114
+ (_THROTTLE_PREFIXES, ErrorKind.THROTTLE),
115
+ (_RETRYABLE_PREFIXES, ErrorKind.RETRYABLE),
116
+ (_FATAL_PREFIXES, ErrorKind.FATAL),
117
+ ):
118
+ if family.startswith(prefixes):
119
+ return kind
120
+ return kind_for_status(status)
121
+
122
+
123
+ def kind_for_status(status: int | None) -> ErrorKind | None:
124
+ """Classify by HTTP status alone: 429 throttle, 408/409/5xx retryable, other 4xx fatal."""
125
+ if status is None:
126
+ return None
127
+ if status == 429:
128
+ return ErrorKind.THROTTLE
129
+ if status in (408, 409) or status >= 500:
130
+ return ErrorKind.RETRYABLE
131
+ if 400 <= status < 500:
132
+ return ErrorKind.FATAL
133
+ return None
134
+
135
+
136
+ def parse_retry_after(
137
+ headers: Mapping[str, Any] | None, *, now: float | None = None
138
+ ) -> float | None:
139
+ """Seconds to wait from `retry-after-ms`, else `Retry-After` (seconds or an HTTP date).
140
+
141
+ Header names are matched case-blind; unparsable values are ignored and past dates give 0.
142
+ """
143
+ if not headers:
144
+ return None
145
+ lowered = {str(k).lower(): str(v).strip() for k, v in headers.items()}
146
+ if (raw_ms := lowered.get("retry-after-ms")) is not None:
147
+ try:
148
+ return max(float(raw_ms) / 1000.0, 0.0)
149
+ except ValueError:
150
+ pass
151
+ raw = lowered.get("retry-after")
152
+ if raw is None:
153
+ return None
154
+ try:
155
+ return max(float(raw), 0.0)
156
+ except ValueError:
157
+ pass
158
+ try:
159
+ when = parsedate_to_datetime(raw)
160
+ except (TypeError, ValueError, IndexError):
161
+ return None
162
+ current = time.time() if now is None else now
163
+ return max(when.timestamp() - current, 0.0)
164
+
165
+
166
+ def is_transport_error(exc: BaseException) -> bool:
167
+ """True for timeouts and connection failures from asyncio, botocore, aiohttp, httpx, openai."""
168
+ if isinstance(exc, TimeoutError | ConnectionError):
169
+ return True
170
+ for module_name, names in _TRANSPORT_TYPES.items():
171
+ module = sys.modules.get(module_name)
172
+ if module is None:
173
+ continue
174
+ types = tuple(t for n in names if isinstance(t := getattr(module, n, None), type))
175
+ if types and isinstance(exc, types):
176
+ return True
177
+ return False
178
+
179
+
180
+ def is_target_error(exc: BaseException, info: ErrorInfo) -> bool:
181
+ """True when a FATAL error is about the target, so `run_call` tries the next fallback step.
182
+
183
+ Model or profile not found / not in this region, access denied for the model, unknown
184
+ endpoint (404). Bad payloads, too-long input and credentials errors end the call.
185
+ """
186
+ if info.kind is not ErrorKind.FATAL:
187
+ return False
188
+ message = _error_message(exc)
189
+ if _REQUEST_MESSAGE.search(message):
190
+ return False
191
+ family = error_family(info.code)
192
+ if family.startswith(_TARGET_FAMILIES) or _error_status(exc) == 404:
193
+ return True
194
+ request_family = family.startswith(("validation", "badrequest", "invalid"))
195
+ return (request_family or _error_status(exc) == 400) and bool(_TARGET_MESSAGE.search(message))
196
+
197
+
198
+ def _error_message(exc: BaseException) -> str:
199
+ """A botocore `Error.Message`, else the exception text (openai puts the body there)."""
200
+ response = getattr(exc, "response", None)
201
+ if isinstance(response, Mapping) and isinstance(response.get("Error"), Mapping):
202
+ return str(response["Error"].get("Message") or "")
203
+ return str(exc)
204
+
205
+
206
+ def _error_status(exc: BaseException) -> int | None:
207
+ """HTTP status of a botocore `ClientError` or an openai `APIStatusError`, if any."""
208
+ status = getattr(exc, "status_code", None)
209
+ response = getattr(exc, "response", None)
210
+ if status is None and isinstance(response, Mapping):
211
+ status = (response.get("ResponseMetadata") or {}).get("HTTPStatusCode")
212
+ return status if isinstance(status, int) else None
213
+
214
+
215
+ def classify_bedrock(exc: BaseException) -> ErrorInfo:
216
+ """Classify a botocore `ClientError` (or transport error) from a Bedrock runtime call.
217
+
218
+ Unknown codes fall back to the HTTP status; anything else that is not a transport error is
219
+ fatal, so a bug in the request never loops through retries.
220
+ """
221
+ response = getattr(exc, "response", None)
222
+ if isinstance(response, Mapping) and isinstance(response.get("Error"), Mapping):
223
+ code = str(response["Error"].get("Code") or "")
224
+ meta = response.get("ResponseMetadata") or {}
225
+ status = meta.get("HTTPStatusCode")
226
+ kind = kind_for_code(code, status if isinstance(status, int) else None)
227
+ retry_after = parse_retry_after(meta.get("HTTPHeaders"))
228
+ return ErrorInfo(kind=kind or ErrorKind.FATAL, code=code, retry_after_secs=retry_after)
229
+ if is_transport_error(exc):
230
+ return ErrorInfo(kind=ErrorKind.RETRYABLE, code=type(exc).__name__)
231
+ return ErrorInfo(kind=ErrorKind.FATAL, code=type(exc).__name__)
232
+
233
+
234
+ def classify_openai(exc: BaseException) -> ErrorInfo:
235
+ """Classify an openai SDK exception (or transport error) from an OpenAI-compatible server.
236
+
237
+ 429 is a throttle carrying the server's advised wait, except `insufficient_quota`, which
238
+ no wait fixes. 408/409/5xx, timeouts and connection errors are retryable; other 4xx fatal.
239
+ """
240
+ if is_transport_error(exc):
241
+ return ErrorInfo(kind=ErrorKind.RETRYABLE, code=type(exc).__name__)
242
+ status = getattr(exc, "status_code", None)
243
+ if not isinstance(status, int):
244
+ return ErrorInfo(kind=ErrorKind.FATAL, code=type(exc).__name__)
245
+ code = str(getattr(exc, "code", None) or getattr(exc, "type", None) or type(exc).__name__)
246
+ response = getattr(exc, "response", None)
247
+ retry_after = parse_retry_after(getattr(response, "headers", None))
248
+ kind = kind_for_status(status) or ErrorKind.FATAL
249
+ if kind is ErrorKind.THROTTLE and code == "insufficient_quota":
250
+ kind = ErrorKind.FATAL
251
+ return ErrorInfo(kind=kind, code=code, retry_after_secs=retry_after)
@@ -0,0 +1,87 @@
1
+ """Embedding batching limits and Bedrock request-body builders (Titan, Cohere).
2
+
3
+ An embed call is split into tasks of at most `max_batch_size` texts: Titan takes one
4
+ text per request, Cohere v3 up to 96, OpenAI `chunk_size` (2048 by default).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Iterator, Mapping, Sequence
10
+ from typing import Any, Literal, TypeVar
11
+
12
+ _T = TypeVar("_T") # PEP 695 syntax needs 3.12; the package supports 3.11
13
+
14
+ TITAN_MAX_BATCH = 1
15
+ COHERE_MAX_BATCH = 96
16
+ OPENAI_DEFAULT_CHUNK_SIZE = 2048
17
+
18
+ CohereInputType = Literal["search_document", "search_query", "classification", "clustering"]
19
+
20
+
21
+ def max_batch_size(provider: str, model_id: str, *, chunk_size: int | None = None) -> int:
22
+ """Largest number of texts one embedding request may carry for this provider and model.
23
+
24
+ Bedrock models other than Cohere are treated like Titan (one text per request).
25
+ """
26
+ if provider == "openai":
27
+ return chunk_size or OPENAI_DEFAULT_CHUNK_SIZE
28
+ return COHERE_MAX_BATCH if "cohere" in model_id.lower() else TITAN_MAX_BATCH
29
+
30
+
31
+ def batches(items: Sequence[_T], size: int) -> Iterator[list[_T]]:
32
+ """Yield consecutive slices of at most `size` items, preserving order."""
33
+ if size < 1:
34
+ raise ValueError("batch size must be >= 1")
35
+ for start in range(0, len(items), size):
36
+ yield list(items[start : start + size])
37
+
38
+
39
+ def titan_body(
40
+ text: str, *, dimensions: int | None = None, normalize: bool | None = None
41
+ ) -> dict[str, Any]:
42
+ """InvokeModel body for Amazon Titan text embeddings (v2 accepts dimensions/normalize)."""
43
+ body: dict[str, Any] = {"inputText": text}
44
+ if dimensions is not None:
45
+ body["dimensions"] = dimensions
46
+ if normalize is not None:
47
+ body["normalize"] = normalize
48
+ return body
49
+
50
+
51
+ def cohere_body(
52
+ texts: Sequence[str],
53
+ *,
54
+ input_type: CohereInputType = "search_document",
55
+ truncate: Literal["NONE", "START", "END"] | None = None,
56
+ ) -> dict[str, Any]:
57
+ """InvokeModel body for Cohere embed v3 on Bedrock (at most 96 texts)."""
58
+ if len(texts) > COHERE_MAX_BATCH:
59
+ raise ValueError(f"Cohere takes at most {COHERE_MAX_BATCH} texts, got {len(texts)}")
60
+ body: dict[str, Any] = {"texts": list(texts), "input_type": input_type}
61
+ if truncate is not None:
62
+ body["truncate"] = truncate
63
+ return body
64
+
65
+
66
+ def embedding_body(model_id: str, texts: Sequence[str], **options: Any) -> dict[str, Any]:
67
+ """Pick the Titan or Cohere body builder by model id; Titan takes exactly one text."""
68
+ if "cohere" in model_id.lower():
69
+ return cohere_body(texts, **options)
70
+ if len(texts) != TITAN_MAX_BATCH:
71
+ raise ValueError(f"Titan takes exactly one text per request, got {len(texts)}")
72
+ return titan_body(texts[0], **options)
73
+
74
+
75
+ def parse_embeddings(decoded: Mapping[str, Any]) -> list[list[float]]:
76
+ """Vectors from a decoded Titan (`embedding`) or Cohere (`embeddings`) response body.
77
+
78
+ Cohere's typed form (`{"embeddings": {"float": [...]}}`) is accepted too.
79
+ """
80
+ if "embedding" in decoded:
81
+ return [list(decoded["embedding"])]
82
+ vectors = decoded.get("embeddings")
83
+ if isinstance(vectors, Mapping):
84
+ vectors = vectors.get("float")
85
+ if not isinstance(vectors, list):
86
+ raise ValueError("response has no 'embedding' or 'embeddings'")
87
+ return [list(v) for v in vectors]
llm_queue/py.typed ADDED
File without changes
@@ -0,0 +1,36 @@
1
+ """The model registry: one document (`LLMQ_REGISTRY`) for accounts, models and their settings.
2
+
3
+ `load_registry()` parses and checks it and returns a `ResolvedRegistry` that answers "which
4
+ deployments does this model have" and "what is this setting here, and where did it come from".
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from llm_queue.registry.resolve import Deployment, Effective, ResolvedRegistry
10
+ from llm_queue.registry.schema import (
11
+ Account,
12
+ ModelEntry,
13
+ RegionEntry,
14
+ Registry,
15
+ Routing,
16
+ register_account_kind,
17
+ )
18
+ from llm_queue.registry.settings import SETTINGS, Spec
19
+ from llm_queue.registry.validate import RegistryError, check, load_registry
20
+
21
+ __all__ = [
22
+ "SETTINGS",
23
+ "Account",
24
+ "Deployment",
25
+ "Effective",
26
+ "ModelEntry",
27
+ "RegionEntry",
28
+ "Registry",
29
+ "RegistryError",
30
+ "ResolvedRegistry",
31
+ "Routing",
32
+ "Spec",
33
+ "check",
34
+ "load_registry",
35
+ "register_account_kind",
36
+ ]
@@ -0,0 +1,78 @@
1
+ """Lua scripts of the registry store: publish by version rule, and compare-and-set live edits.
2
+
3
+ Both write the document, its version record, the audit entry and each model's resolved
4
+ numbers in one script call, so a reader never sees half an update. The state store runs as one
5
+ primary (cluster mode is not supported), so the keys need not share a hash tag.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ _COMMON = """
11
+ local t = redis.call('TIME')
12
+ local now = tonumber(t[1]) + tonumber(t[2]) / 1000000
13
+ local now_ms = math.floor(now * 1000)
14
+ local function audit(actor, op, version, diff, retention_ms)
15
+ redis.call('XADD', KEYS[3], '*', 'actor', actor, 'op', op, 'version', version,
16
+ 'diff', diff, 'ts', tostring(now))
17
+ redis.call('XTRIM', KEYS[3], 'MINID', tostring(now_ms - tonumber(retention_ms)))
18
+ end
19
+ local function write_models(first_key, first_arg, changes)
20
+ for i = first_key, #KEYS do
21
+ redis.call('HSET', KEYS[i], 'changes', changes, 'numbers', ARGV[first_arg + i - first_key])
22
+ end
23
+ end
24
+ """
25
+
26
+ # KEYS: doc, meta (hash), audit (stream), history (hash version -> doc), then one numbers key
27
+ # per model. ARGV: doc, version, config hash, actor, diff JSON, audit retention ms, history size,
28
+ # then one numbers JSON per model key.
29
+ # Reply: {action, stored version, stored doc or ''} with action published | kept | not_bumped |
30
+ # ignored. Rule (plan §3.8): higher replaces, equal keeps the stored one (and its live edits),
31
+ # equal with another content hash keeps it and reports not_bumped, lower is ignored.
32
+ PUBLISH = (
33
+ _COMMON
34
+ + """
35
+ local version = tonumber(ARGV[2])
36
+ local stored_version = tonumber(redis.call('HGET', KEYS[2], 'version') or '0')
37
+ local stored_doc = redis.call('GET', KEYS[1])
38
+ if stored_doc and stored_version >= version then
39
+ if stored_version > version then return {'ignored', stored_version, stored_doc} end
40
+ if redis.call('HGET', KEYS[2], 'config_hash') == ARGV[3] then
41
+ return {'kept', stored_version, stored_doc}
42
+ end
43
+ return {'not_bumped', stored_version, stored_doc}
44
+ end
45
+ local changes = redis.call('HINCRBY', KEYS[2], 'changes', 1)
46
+ redis.call('SET', KEYS[1], ARGV[1])
47
+ redis.call('HSET', KEYS[2], 'version', version, 'config_hash', ARGV[3], 'published', tostring(now))
48
+ redis.call('HSET', KEYS[4], tostring(version), ARGV[1])
49
+ local keep = tonumber(ARGV[7])
50
+ local versions = redis.call('HKEYS', KEYS[4])
51
+ if #versions > keep then
52
+ table.sort(versions, function(a, b) return tonumber(a) < tonumber(b) end)
53
+ for i = 1, #versions - keep do redis.call('HDEL', KEYS[4], versions[i]) end
54
+ end
55
+ audit(ARGV[4], 'publish', version, ARGV[5], ARGV[6])
56
+ write_models(5, 8, changes)
57
+ return {'published', version, ''}
58
+ """
59
+ )
60
+
61
+ # KEYS: doc, meta, audit, history, then numbers keys. ARGV: doc, expected changes, actor, diff
62
+ # JSON, audit retention ms, then numbers JSONs. Reply: {'ok', changes} or {'conflict', changes}
63
+ # when someone else changed the registry since it was read.
64
+ EDIT = (
65
+ _COMMON
66
+ + """
67
+ local changes = tonumber(redis.call('HGET', KEYS[2], 'changes') or '0')
68
+ if changes ~= tonumber(ARGV[2]) or not redis.call('GET', KEYS[1]) then
69
+ return {'conflict', changes}
70
+ end
71
+ changes = redis.call('HINCRBY', KEYS[2], 'changes', 1)
72
+ redis.call('SET', KEYS[1], ARGV[1])
73
+ local version = redis.call('HGET', KEYS[2], 'version') or '0'
74
+ audit(ARGV[3], 'edit', version, ARGV[4], ARGV[5])
75
+ write_models(5, 6, changes)
76
+ return {'ok', changes}
77
+ """
78
+ )
@@ -0,0 +1,199 @@
1
+ """Resolve a `Registry`: the deployments of each model and every setting's effective value.
2
+
3
+ A setting is taken from the most specific level that sets it: deployment → model → account →
4
+ `defaults` → library default. Object settings marked `merge` combine key by key. Used by the
5
+ validator, the admission script inputs and `llm-queue settings get`.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Mapping
11
+ from dataclasses import dataclass
12
+ from typing import Any
13
+
14
+ from llm_queue.registry.schema import ModelEntry, Registry
15
+ from llm_queue.registry.settings import (
16
+ CHAT_INFERENCE,
17
+ EMBED_INFERENCE,
18
+ FALLBACK_AFTER_SECS,
19
+ MODEL_FALLBACK_AFTER_SECS,
20
+ SETTINGS,
21
+ A,
22
+ D,
23
+ M,
24
+ R,
25
+ )
26
+
27
+ LIBRARY = "library"
28
+ _LEVEL_LABEL = {R: "deployment", M: "model", A: "account", D: "defaults"}
29
+
30
+
31
+ @dataclass(frozen=True, slots=True)
32
+ class Deployment:
33
+ """One model in one account and region (or inference profile): what has a budget.
34
+
35
+ `key` (`account/name`) names it in keys, metrics and status.
36
+ """
37
+
38
+ model: str
39
+ account: str
40
+ name: str
41
+ group: str
42
+ region: str
43
+ profile_id: str | None = None
44
+ routes_to: tuple[str, ...] = ()
45
+
46
+ @property
47
+ def key(self) -> str:
48
+ """`account/name`, unique inside a model."""
49
+ return f"{self.account}/{self.name}"
50
+
51
+
52
+ @dataclass(frozen=True, slots=True)
53
+ class Effective:
54
+ """A setting's value and the level it came from (`deployment` … `defaults`, `library`)."""
55
+
56
+ value: Any
57
+ level: str
58
+
59
+
60
+ class ResolvedRegistry:
61
+ """Read view over a `Registry`: deployments per model and effective settings.
62
+
63
+ Pure and cheap to build; rebuilt on every registry reload.
64
+ """
65
+
66
+ def __init__(self, registry: Registry) -> None:
67
+ """Index the registry's models and deployments."""
68
+ self.registry = registry
69
+ self._deployments: dict[str, tuple[Deployment, ...]] = {}
70
+ self._entries: dict[tuple[str, str], Mapping[str, Any]] = {}
71
+ for alias, model in registry.models.items():
72
+ found: list[Deployment] = []
73
+ for group in ("confirmed", "fallback"):
74
+ for account, entries in model.deployments.items():
75
+ for entry in entries:
76
+ if entry.group != group:
77
+ continue
78
+ deployment = Deployment(
79
+ model=alias,
80
+ account=account,
81
+ name=entry.name,
82
+ group=entry.group,
83
+ region=entry.region,
84
+ profile_id=entry.profile_id,
85
+ routes_to=entry.routes_to,
86
+ )
87
+ found.append(deployment)
88
+ self._entries[(alias, deployment.key)] = entry.settings
89
+ self._deployments[alias] = tuple(found)
90
+
91
+ def model(self, model: str) -> ModelEntry:
92
+ """The entry of a model named by alias or provider model id."""
93
+ return self.registry.models[self.registry.alias_of(model)]
94
+
95
+ def deployments(self, model: str, group: str | None = None) -> tuple[Deployment, ...]:
96
+ """Deployments of a model, confirmed first, in config order; optionally one group."""
97
+ found = self._deployments[self.registry.alias_of(model)]
98
+ return found if group is None else tuple(d for d in found if d.group == group)
99
+
100
+ def _layers(self, alias: str, deployment: str | None) -> list[tuple[str, Mapping[str, Any]]]:
101
+ """The settings of each level, most specific first."""
102
+ model = self.registry.models[alias]
103
+ layers: list[tuple[str, Mapping[str, Any]]] = []
104
+ account = None
105
+ if deployment is not None:
106
+ if (alias, deployment) not in self._entries:
107
+ raise KeyError(f"{alias} has no deployment {deployment}")
108
+ layers.append((R, self._entries[(alias, deployment)]))
109
+ account = deployment.split("/", 1)[0]
110
+ layers.append((M, model.settings))
111
+ if account is not None:
112
+ layers.append((A, self.registry.accounts[account].settings))
113
+ layers.append((D, self.registry.defaults))
114
+ return layers
115
+
116
+ def effective(self, model: str, name: str, deployment: str | None = None) -> Effective:
117
+ """The value of setting `name` for a model (and one deployment), with its level.
118
+
119
+ Without `deployment`, the deployment and account levels are skipped.
120
+ """
121
+ spec = SETTINGS[name]
122
+ alias = self.registry.alias_of(model)
123
+ entry = self.registry.models[alias]
124
+ layers = self._layers(alias, deployment)
125
+ if spec.merge:
126
+ merged: dict[str, Any] = dict(spec.default or {})
127
+ level = LIBRARY
128
+ for code, values in reversed(layers):
129
+ if name in values:
130
+ part = values[name]
131
+ if name == "inference" and code in (D, A):
132
+ # Shared defaults apply only to models of the matching type.
133
+ own = CHAT_INFERENCE if entry.type == "chat" else EMBED_INFERENCE
134
+ part = {k: v for k, v in part.items() if k in own}
135
+ if not part:
136
+ continue
137
+ merged.update(part)
138
+ level = _LEVEL_LABEL[code]
139
+ return Effective(_complete(name, merged), level)
140
+ for code, values in layers:
141
+ if name in values:
142
+ return Effective(_complete(name, values[name]), _LEVEL_LABEL[code])
143
+ return Effective(self._library_default(entry, name, deployment), LIBRARY)
144
+
145
+ def value(self, model: str, name: str, deployment: str | None = None) -> Any:
146
+ """Shortcut for `effective(...).value`."""
147
+ return self.effective(model, name, deployment).value
148
+
149
+ def _library_default(self, entry: ModelEntry, name: str, deployment: str | None) -> Any:
150
+ spec = SETTINGS[name]
151
+ if name == "rate_limits" and entry.type == "embedding":
152
+ # A `tpm` on an embedding model turns token limiting on (plan §3.19).
153
+ has_tpm = self.effective(entry.alias, "tpm", deployment).value is not None
154
+ return "both" if has_tpm else "rpm"
155
+ if spec.by_type is not None:
156
+ return spec.by_type(entry.type)
157
+ return spec.default
158
+
159
+ def table(self, model: str, deployment: str | None = None) -> dict[str, Effective]:
160
+ """Every setting of a model (and deployment) with its value and level (CLI view)."""
161
+ alias = self.registry.alias_of(model)
162
+ return {name: self.effective(alias, name, deployment) for name in sorted(SETTINGS)}
163
+
164
+ def queue_name(self, model: str) -> str:
165
+ """The model's normal queue name, without the environment prefix."""
166
+ alias = self.registry.alias_of(model)
167
+ named = self.registry.models[alias].settings.get("queue", {}).get("name")
168
+ return str(named) if named else f"llm-{alias}"
169
+
170
+ def residency(self, model: str) -> tuple[str, ...] | None:
171
+ """The regions a model may reach (its own list, else the top-level one), None = any."""
172
+ entry = self.model(model)
173
+ return entry.residency if entry.residency is not None else self.registry.residency
174
+
175
+ def outside_residency(self, deployment: Deployment) -> bool:
176
+ """True when a deployment may not be used under the residency list."""
177
+ allowed = self.residency(deployment.model)
178
+ if allowed is None:
179
+ return False
180
+ if deployment.profile_id is not None:
181
+ return not deployment.routes_to or not set(deployment.routes_to) <= set(allowed)
182
+ return deployment.region not in allowed
183
+
184
+
185
+ def _complete(name: str, value: Any) -> Any:
186
+ """Fill the optional fields of the two fallback rules and `sticky`."""
187
+ if name == "fallback" and isinstance(value, dict):
188
+ return {"after_secs": FALLBACK_AFTER_SECS, "regions": "account", **value}
189
+ if name == "model_fallback" and isinstance(value, dict):
190
+ models = [
191
+ {"model": m, "require_headroom": False}
192
+ if isinstance(m, str)
193
+ else {"require_headroom": False, **m}
194
+ for m in value.get("models", [])
195
+ ]
196
+ return {"after_secs": MODEL_FALLBACK_AFTER_SECS, **value, "models": models}
197
+ if name == "sticky" and isinstance(value, dict):
198
+ return {"idle_ttl_secs": 1800, **value}
199
+ return value