llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,614 @@
|
|
|
1
|
+
"""The registry in the state store: publish by version, live edits, audit, rollback, reload.
|
|
2
|
+
|
|
3
|
+
Every write is one Lua call that stores the document, its version record, an audit entry (path
|
|
4
|
+
diffs, secrets redacted) and each model's resolved numbers together; a live edit is a
|
|
5
|
+
compare-and-set on the change counter. Readers keep the last good registry when the stored one is
|
|
6
|
+
unreadable, and the reload loop republishes the config when the stored registry was lost.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import hashlib
|
|
13
|
+
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
import orjson
|
|
18
|
+
from redis.exceptions import RedisError
|
|
19
|
+
|
|
20
|
+
from llm_queue._logging import get_logger
|
|
21
|
+
from llm_queue.engine.store_guard import store_op
|
|
22
|
+
from llm_queue.errors import ConfigError
|
|
23
|
+
from llm_queue.registry import _lua
|
|
24
|
+
from llm_queue.registry.resolve import ResolvedRegistry
|
|
25
|
+
from llm_queue.registry.validate import RegistryError, check
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
import redis.asyncio
|
|
29
|
+
|
|
30
|
+
logger = get_logger(__name__)
|
|
31
|
+
|
|
32
|
+
AUDIT_RETENTION_DAYS = 90
|
|
33
|
+
LIVE_EDIT_MODES = ("safe", "all", "off")
|
|
34
|
+
_SECRET_FIELDS = frozenset({"role_arn", "external_id_ref", "api_key_ref"})
|
|
35
|
+
# Settings whose value goes into a model's resolved numbers (read by the admission script).
|
|
36
|
+
ADMISSION_SETTINGS = (
|
|
37
|
+
"enabled",
|
|
38
|
+
"rpm",
|
|
39
|
+
"tpm",
|
|
40
|
+
"rate_limits",
|
|
41
|
+
"rate_window",
|
|
42
|
+
"rate_burst",
|
|
43
|
+
"tpm_output_weight",
|
|
44
|
+
"max_concurrency",
|
|
45
|
+
"weight",
|
|
46
|
+
"attempt_timeout_secs",
|
|
47
|
+
"breaker_threshold",
|
|
48
|
+
"breaker_cooldown_secs",
|
|
49
|
+
"breaker_window_secs",
|
|
50
|
+
"breaker_probe_secs",
|
|
51
|
+
)
|
|
52
|
+
MODEL_ADMISSION_SETTINGS = (
|
|
53
|
+
"balance",
|
|
54
|
+
"balance_priority",
|
|
55
|
+
"balance_opts",
|
|
56
|
+
"priority_reserve",
|
|
57
|
+
"normal_min_share",
|
|
58
|
+
"require_headroom_pct",
|
|
59
|
+
"max_waiting",
|
|
60
|
+
"health",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class RegistryConflict(ConfigError):
|
|
65
|
+
"""A live edit lost a race with another edit three times in a row."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class LiveEditRefused(ConfigError):
|
|
69
|
+
"""A live edit touched fields that change only by deploy (or live edits are off)."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
Path = tuple[str, ...]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True, slots=True)
|
|
76
|
+
class PublishResult:
|
|
77
|
+
"""What a publish did: `published`, `kept`, `not_bumped` or `ignored`, and the version live."""
|
|
78
|
+
|
|
79
|
+
action: str
|
|
80
|
+
version: int
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def content_hash(doc: Mapping[str, Any]) -> str:
|
|
84
|
+
"""Stable hash of a document (sorted keys), to spot a config change without a version bump."""
|
|
85
|
+
return hashlib.sha256(orjson.dumps(doc, option=orjson.OPT_SORT_KEYS)).hexdigest()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def diff(before: Any, after: Any, path: Path = ()) -> list[tuple[Path, Any, Any]]:
|
|
89
|
+
"""Leaf changes between two documents as (path, before, after); lists compare whole."""
|
|
90
|
+
if isinstance(before, dict) and after is None:
|
|
91
|
+
after = {}
|
|
92
|
+
if isinstance(after, dict) and before is None:
|
|
93
|
+
before = {}
|
|
94
|
+
if isinstance(before, dict) and isinstance(after, dict):
|
|
95
|
+
out: list[tuple[Path, Any, Any]] = []
|
|
96
|
+
for key in sorted(set(before) | set(after)):
|
|
97
|
+
out += diff(before.get(key), after.get(key), (*path, key))
|
|
98
|
+
return out
|
|
99
|
+
return [] if before == after else [(path, before, after)]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _redact(path: Path, value: Any) -> Any:
|
|
103
|
+
if value is not None and path and path[-1] in _SECRET_FIELDS:
|
|
104
|
+
return "[redacted]"
|
|
105
|
+
if isinstance(value, dict):
|
|
106
|
+
return {k: _redact((*path, k), v) for k, v in value.items()}
|
|
107
|
+
return value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def audit_diff(before: Mapping[str, Any], after: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|
111
|
+
"""The audit form of a change: one entry per changed path, secrets redacted."""
|
|
112
|
+
return [
|
|
113
|
+
{"path": ".".join(p), "before": _redact(p, b), "after": _redact(p, a)}
|
|
114
|
+
for p, b, a in diff(dict(before), dict(after))
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _deploy_only(path: Path) -> str | None:
|
|
119
|
+
"""Why a changed path needs a deploy (a new `registry_version`), or None."""
|
|
120
|
+
if path[:1] in (("registry_version",), ("min_lib_version",)):
|
|
121
|
+
return "changes with a deploy"
|
|
122
|
+
if path[:1] == ("models",) and len(path) == 2:
|
|
123
|
+
return "adding or removing a model needs a new registry_version"
|
|
124
|
+
if path[:1] == ("models",) and len(path) >= 3:
|
|
125
|
+
if path[2] in ("type", "provider", "model_id", "worker_slots"):
|
|
126
|
+
return f"`{path[2]}` changes only with a deploy"
|
|
127
|
+
if path[2:4] == ("queue", "name"):
|
|
128
|
+
return "a queue name changes only with a deploy"
|
|
129
|
+
if path[:2] == ("defaults", "worker_slots"):
|
|
130
|
+
return "`worker_slots` changes only with a deploy"
|
|
131
|
+
return None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _protected(path: Path) -> str | None:
|
|
135
|
+
"""Why a changed path is protected under `registry_live_edit=safe`, or None."""
|
|
136
|
+
if "residency" in path:
|
|
137
|
+
return "residency changes only with a deploy"
|
|
138
|
+
if path[-1:] == ("debug_capture",):
|
|
139
|
+
return "debug_capture (stores prompt text) changes only with a deploy"
|
|
140
|
+
if path[:1] == ("accounts",):
|
|
141
|
+
if len(path) == 2:
|
|
142
|
+
return "adding or removing an account changes only with a deploy"
|
|
143
|
+
if len(path) >= 3 and path[2] in ("kind", "base_url", *_SECRET_FIELDS):
|
|
144
|
+
return f"account `{path[2]}` changes only with a deploy"
|
|
145
|
+
if path[:1] == ("models",) and len(path) >= 4 and path[2] == "deployments":
|
|
146
|
+
if len(path) == 4 or path[4] in ("confirmed", "fallback"):
|
|
147
|
+
return "region lists change only with a deploy"
|
|
148
|
+
if "params" in path:
|
|
149
|
+
return "deployment params change only with a deploy"
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def refused_edits(before: Mapping[str, Any], after: Mapping[str, Any], mode: str) -> list[str]:
|
|
154
|
+
"""Reasons a live edit from `before` to `after` is refused under `registry_live_edit=mode`."""
|
|
155
|
+
if mode not in LIVE_EDIT_MODES:
|
|
156
|
+
raise ValueError(f"registry_live_edit must be one of {', '.join(LIVE_EDIT_MODES)}")
|
|
157
|
+
changes = diff(dict(before), dict(after))
|
|
158
|
+
if mode == "off" and changes:
|
|
159
|
+
return ["live edits are off (registry_live_edit=off)"]
|
|
160
|
+
reasons = []
|
|
161
|
+
whole = []
|
|
162
|
+
for section, what in (("models", "a model"), ("accounts", "an account")):
|
|
163
|
+
old, new = set(before.get(section, {})), set(after.get(section, {}))
|
|
164
|
+
for name in sorted(old ^ new):
|
|
165
|
+
whole.append((section, name))
|
|
166
|
+
reasons.append(
|
|
167
|
+
f"{section}.{name}: adding or removing {what} needs a new registry_version"
|
|
168
|
+
)
|
|
169
|
+
for path, _, _ in changes:
|
|
170
|
+
if path[:2] in whole:
|
|
171
|
+
continue
|
|
172
|
+
why = _deploy_only(path) or (_protected(path) if mode == "safe" else None)
|
|
173
|
+
if why:
|
|
174
|
+
reasons.append(f"{'.'.join(path)}: {why}")
|
|
175
|
+
return reasons
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _set_path(doc: dict[str, Any], path: Sequence[str], value: Any) -> None:
|
|
179
|
+
"""Set (or delete, value None) one path, creating objects on the way."""
|
|
180
|
+
node = doc
|
|
181
|
+
for key in path[:-1]:
|
|
182
|
+
child = node.get(key)
|
|
183
|
+
if not isinstance(child, dict):
|
|
184
|
+
child = {}
|
|
185
|
+
node[key] = child
|
|
186
|
+
node = child
|
|
187
|
+
if value is None:
|
|
188
|
+
node.pop(path[-1], None)
|
|
189
|
+
else:
|
|
190
|
+
node[path[-1]] = value
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def split_path(doc: Mapping[str, Any], dotted: str) -> list[str]:
|
|
194
|
+
"""Split a dotted path, keeping names that contain dots (`deployments.aws-main.x.y`) whole.
|
|
195
|
+
|
|
196
|
+
At each level the longest run of segments that names an existing key wins.
|
|
197
|
+
"""
|
|
198
|
+
parts = dotted.split(".")
|
|
199
|
+
out: list[str] = []
|
|
200
|
+
node: Any = doc
|
|
201
|
+
i = 0
|
|
202
|
+
while i < len(parts):
|
|
203
|
+
chosen = 1
|
|
204
|
+
if isinstance(node, Mapping):
|
|
205
|
+
for j in range(len(parts), i, -1):
|
|
206
|
+
if ".".join(parts[i:j]) in node:
|
|
207
|
+
chosen = j - i
|
|
208
|
+
break
|
|
209
|
+
key = ".".join(parts[i : i + chosen])
|
|
210
|
+
out.append(key)
|
|
211
|
+
node = node.get(key) if isinstance(node, Mapping) else None
|
|
212
|
+
i += chosen
|
|
213
|
+
return out
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def admission_numbers(view: ResolvedRegistry, alias: str) -> dict[str, Any]:
|
|
217
|
+
"""A model's resolved numbers for the admission script: model values, one row per deployment."""
|
|
218
|
+
model = {name: view.value(alias, name) for name in MODEL_ADMISSION_SETTINGS}
|
|
219
|
+
deployments = []
|
|
220
|
+
for dep in view.deployments(alias):
|
|
221
|
+
row = {name: view.value(alias, name, dep.key) for name in ADMISSION_SETTINGS}
|
|
222
|
+
# A model disabled as a whole still serves its waiting calls (drain, plan §3.18).
|
|
223
|
+
if view.effective(alias, "enabled", dep.key).level == "model":
|
|
224
|
+
row["enabled"] = True
|
|
225
|
+
account = view.registry.accounts[dep.account]
|
|
226
|
+
routing = view.registry.routing
|
|
227
|
+
row.update(
|
|
228
|
+
key=dep.key,
|
|
229
|
+
account=dep.account,
|
|
230
|
+
group=dep.group,
|
|
231
|
+
region=dep.region,
|
|
232
|
+
outside_residency=view.outside_residency(dep),
|
|
233
|
+
account_enabled=account.enabled,
|
|
234
|
+
blocked=dep.region in routing.blocked_regions
|
|
235
|
+
or dep.account in routing.blocked_accounts,
|
|
236
|
+
shared_rpm=account.settings.get("shared_rpm"),
|
|
237
|
+
shared_tpm=account.settings.get("shared_tpm"),
|
|
238
|
+
account_priority_reserve=account.settings.get("priority_reserve", 0),
|
|
239
|
+
)
|
|
240
|
+
deployments.append(row)
|
|
241
|
+
return {"model": model, "deployments": deployments}
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class RegistryStore:
|
|
245
|
+
"""The registry of one environment (key prefix) in the state store.
|
|
246
|
+
|
|
247
|
+
`current` is the last good `ResolvedRegistry`; hot paths read it without I/O. `config` is
|
|
248
|
+
this process's own document, used by `boot()` and to restore a lost registry.
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
def __init__(
|
|
252
|
+
self,
|
|
253
|
+
client: redis.asyncio.Redis,
|
|
254
|
+
prefix: str,
|
|
255
|
+
*,
|
|
256
|
+
config: Mapping[str, Any] | None = None,
|
|
257
|
+
live_edit: str = "safe",
|
|
258
|
+
allowed_hosts: tuple[str, ...] = (),
|
|
259
|
+
audit_retention_days: int = AUDIT_RETENTION_DAYS,
|
|
260
|
+
history_size: int = 20,
|
|
261
|
+
edit_retries: int = 3,
|
|
262
|
+
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
|
263
|
+
) -> None:
|
|
264
|
+
"""Keep the client and settings; nothing is read or written until `boot` or `load`."""
|
|
265
|
+
if live_edit not in LIVE_EDIT_MODES:
|
|
266
|
+
raise ConfigError(f"registry_live_edit must be one of {', '.join(LIVE_EDIT_MODES)}")
|
|
267
|
+
self._client = client
|
|
268
|
+
self.prefix = prefix
|
|
269
|
+
self._history_size = history_size # `Tuning.registry`
|
|
270
|
+
self._edit_retries = edit_retries
|
|
271
|
+
self.config = dict(config) if config is not None else None
|
|
272
|
+
self.live_edit = live_edit
|
|
273
|
+
self._hosts = allowed_hosts
|
|
274
|
+
self._retention_ms = audit_retention_days * 86_400_000
|
|
275
|
+
self._sleep = sleep
|
|
276
|
+
self._publish = client.register_script(_lua.PUBLISH)
|
|
277
|
+
self._edit = client.register_script(_lua.EDIT)
|
|
278
|
+
self._current: ResolvedRegistry | None = None
|
|
279
|
+
self._changes = -1
|
|
280
|
+
self._version = 0
|
|
281
|
+
self.invalid_count = 0
|
|
282
|
+
self._warned: set[str] = set()
|
|
283
|
+
|
|
284
|
+
# ---------------------------------------------------------------- keys
|
|
285
|
+
|
|
286
|
+
@property
|
|
287
|
+
def doc_key(self) -> str:
|
|
288
|
+
"""The stored document (JSON string)."""
|
|
289
|
+
return f"{self.prefix}:{{reg}}:doc"
|
|
290
|
+
|
|
291
|
+
@property
|
|
292
|
+
def meta_key(self) -> str:
|
|
293
|
+
"""Version record: `version`, `config_hash`, `changes`, `published`."""
|
|
294
|
+
return f"{self.prefix}:{{reg}}:meta"
|
|
295
|
+
|
|
296
|
+
@property
|
|
297
|
+
def audit_key(self) -> str:
|
|
298
|
+
"""Audit stream: one entry per publish or edit, path diffs with secrets redacted."""
|
|
299
|
+
return f"{self.prefix}:{{reg}}:audit"
|
|
300
|
+
|
|
301
|
+
@property
|
|
302
|
+
def history_key(self) -> str:
|
|
303
|
+
"""Published versions kept for rollback (hash version -> document)."""
|
|
304
|
+
return f"{self.prefix}:{{reg}}:history"
|
|
305
|
+
|
|
306
|
+
def numbers_key(self, alias: str) -> str:
|
|
307
|
+
"""A model's resolved numbers and the change counter they belong to."""
|
|
308
|
+
return f"{self.prefix}:{{m:{alias}}}:cfg"
|
|
309
|
+
|
|
310
|
+
# ---------------------------------------------------------------- reading
|
|
311
|
+
|
|
312
|
+
@property
|
|
313
|
+
def current(self) -> ResolvedRegistry:
|
|
314
|
+
"""Last good registry; raises `ConfigError` before the first successful load."""
|
|
315
|
+
if self._current is None:
|
|
316
|
+
raise ConfigError("the registry has not been loaded yet")
|
|
317
|
+
return self._current
|
|
318
|
+
|
|
319
|
+
@property
|
|
320
|
+
def changes(self) -> int:
|
|
321
|
+
"""Change counter of the loaded registry (-1 before the first load)."""
|
|
322
|
+
return self._changes
|
|
323
|
+
|
|
324
|
+
async def _read(self) -> tuple[dict[str, Any] | None, dict[str, str]]:
|
|
325
|
+
async with self._client.pipeline(transaction=False) as pipe:
|
|
326
|
+
pipe.get(self.doc_key)
|
|
327
|
+
pipe.hgetall(self.meta_key)
|
|
328
|
+
raw, meta = await pipe.execute()
|
|
329
|
+
meta_text = {_text(k): _text(v) for k, v in (meta or {}).items()}
|
|
330
|
+
if raw is None:
|
|
331
|
+
return None, meta_text
|
|
332
|
+
try:
|
|
333
|
+
doc = orjson.loads(raw)
|
|
334
|
+
except orjson.JSONDecodeError:
|
|
335
|
+
return {"__invalid__": _text(raw)[:200]}, meta_text
|
|
336
|
+
return doc, meta_text
|
|
337
|
+
|
|
338
|
+
def _accept(self, doc: Mapping[str, Any], meta: Mapping[str, str]) -> bool:
|
|
339
|
+
registry, errors, _ = check(doc, strict=False, allowed_hosts=self._hosts)
|
|
340
|
+
if registry is None:
|
|
341
|
+
self.invalid_count += 1
|
|
342
|
+
key = "|".join(errors)
|
|
343
|
+
if key not in self._warned:
|
|
344
|
+
self._warned.add(key)
|
|
345
|
+
logger.warning("registry_invalid", problems=errors[:20], count=len(errors))
|
|
346
|
+
return False
|
|
347
|
+
for warning in registry.warnings:
|
|
348
|
+
if warning not in self._warned:
|
|
349
|
+
self._warned.add(warning)
|
|
350
|
+
logger.warning("registry_warning", warning=warning)
|
|
351
|
+
self._current = ResolvedRegistry(registry)
|
|
352
|
+
self._changes = int(meta.get("changes", "0"))
|
|
353
|
+
self._version = registry.version
|
|
354
|
+
return True
|
|
355
|
+
|
|
356
|
+
async def load(self) -> ResolvedRegistry | None:
|
|
357
|
+
"""Read the stored registry; keep the last good one when it is unreadable or invalid.
|
|
358
|
+
|
|
359
|
+
Returns None when there is no stored registry (nothing published yet, or it was lost).
|
|
360
|
+
"""
|
|
361
|
+
try:
|
|
362
|
+
doc, meta = await self._read()
|
|
363
|
+
except (RedisError, OSError) as exc:
|
|
364
|
+
logger.warning("registry_unreachable", error=str(exc))
|
|
365
|
+
return self._current
|
|
366
|
+
if doc is None:
|
|
367
|
+
return None
|
|
368
|
+
self._accept(doc, meta)
|
|
369
|
+
return self._current
|
|
370
|
+
|
|
371
|
+
# ---------------------------------------------------------------- publishing
|
|
372
|
+
|
|
373
|
+
def _numbers(self, view: ResolvedRegistry) -> tuple[list[str], list[str]]:
|
|
374
|
+
aliases = sorted(view.registry.models)
|
|
375
|
+
keys = [self.numbers_key(a) for a in aliases]
|
|
376
|
+
values = [orjson.dumps(admission_numbers(view, a)).decode() for a in aliases]
|
|
377
|
+
return keys, values
|
|
378
|
+
|
|
379
|
+
@store_op
|
|
380
|
+
async def publish(self, doc: Mapping[str, Any], *, actor: str) -> PublishResult:
|
|
381
|
+
"""Publish a document by the version rule; raise `RegistryError` when it is invalid."""
|
|
382
|
+
registry, errors, warnings = check(doc, strict=True, allowed_hosts=self._hosts)
|
|
383
|
+
if registry is None:
|
|
384
|
+
raise RegistryError(errors, warnings)
|
|
385
|
+
view = ResolvedRegistry(registry)
|
|
386
|
+
stored, _ = await self._read()
|
|
387
|
+
changes = orjson.dumps(audit_diff(stored or {}, doc)).decode()
|
|
388
|
+
keys, values = self._numbers(view)
|
|
389
|
+
reply = await self._publish(
|
|
390
|
+
[self.doc_key, self.meta_key, self.audit_key, self.history_key, *keys],
|
|
391
|
+
[
|
|
392
|
+
orjson.dumps(doc).decode(),
|
|
393
|
+
registry.version,
|
|
394
|
+
content_hash(doc),
|
|
395
|
+
actor,
|
|
396
|
+
changes,
|
|
397
|
+
self._retention_ms,
|
|
398
|
+
self._history_size,
|
|
399
|
+
*values,
|
|
400
|
+
],
|
|
401
|
+
)
|
|
402
|
+
action, version = _text(reply[0]), int(reply[1])
|
|
403
|
+
if action == "not_bumped":
|
|
404
|
+
logger.warning("registry_version_not_bumped", version=version)
|
|
405
|
+
elif action == "ignored":
|
|
406
|
+
logger.info("registry_older_ignored", stored=version, config=registry.version)
|
|
407
|
+
elif action == "published":
|
|
408
|
+
logger.info("registry_published", version=version, actor=actor)
|
|
409
|
+
logger.info("llmq_registry_audit", actor=actor, op="publish", diff=changes)
|
|
410
|
+
await self.load()
|
|
411
|
+
return PublishResult(action, version)
|
|
412
|
+
|
|
413
|
+
async def boot(self, *, actor: str = "boot") -> ResolvedRegistry:
|
|
414
|
+
"""Publish this process's config by the version rule, then load what is live.
|
|
415
|
+
|
|
416
|
+
When the store cannot be reached the process starts on its own config (plan §3.18).
|
|
417
|
+
"""
|
|
418
|
+
if self.config is None:
|
|
419
|
+
raise ConfigError("RegistryStore.boot needs the process's own registry (config)")
|
|
420
|
+
try:
|
|
421
|
+
await self.publish(self.config, actor=actor)
|
|
422
|
+
except (RedisError, OSError) as exc:
|
|
423
|
+
logger.warning("registry_store_unreachable_at_boot", error=str(exc))
|
|
424
|
+
registry, errors, warnings = check(self.config, allowed_hosts=self._hosts)
|
|
425
|
+
if registry is None:
|
|
426
|
+
raise RegistryError(errors, warnings) from exc
|
|
427
|
+
self._current = ResolvedRegistry(registry)
|
|
428
|
+
return self.current
|
|
429
|
+
|
|
430
|
+
# ---------------------------------------------------------------- live edits
|
|
431
|
+
|
|
432
|
+
@store_op
|
|
433
|
+
async def edit(
|
|
434
|
+
self, change: Callable[[dict[str, Any]], None], *, actor: str
|
|
435
|
+
) -> ResolvedRegistry:
|
|
436
|
+
"""Apply `change` to the stored document as a live edit: validated, checked, audited.
|
|
437
|
+
|
|
438
|
+
Compare-and-set on the change counter; on a race the document is read again and the
|
|
439
|
+
change re-applied (up to three times).
|
|
440
|
+
"""
|
|
441
|
+
for _ in range(self._edit_retries):
|
|
442
|
+
doc, meta = await self._read()
|
|
443
|
+
if doc is None:
|
|
444
|
+
raise ConfigError("no registry is stored; publish one first")
|
|
445
|
+
after = orjson.loads(orjson.dumps(doc))
|
|
446
|
+
change(after)
|
|
447
|
+
reasons = refused_edits(doc, after, self.live_edit)
|
|
448
|
+
if reasons:
|
|
449
|
+
logger.warning("registry_edit_refused", actor=actor, reasons=reasons)
|
|
450
|
+
raise LiveEditRefused("; ".join(reasons))
|
|
451
|
+
registry, errors, warnings = check(after, strict=False, allowed_hosts=self._hosts)
|
|
452
|
+
if registry is None:
|
|
453
|
+
raise RegistryError(errors, warnings)
|
|
454
|
+
changes = orjson.dumps(audit_diff(doc, after)).decode()
|
|
455
|
+
keys, values = self._numbers(ResolvedRegistry(registry))
|
|
456
|
+
reply = await self._edit(
|
|
457
|
+
[self.doc_key, self.meta_key, self.audit_key, self.history_key, *keys],
|
|
458
|
+
[
|
|
459
|
+
orjson.dumps(after).decode(),
|
|
460
|
+
int(meta.get("changes", "0")),
|
|
461
|
+
actor,
|
|
462
|
+
changes,
|
|
463
|
+
self._retention_ms,
|
|
464
|
+
*values,
|
|
465
|
+
],
|
|
466
|
+
)
|
|
467
|
+
if _text(reply[0]) == "ok":
|
|
468
|
+
logger.info("llmq_registry_audit", actor=actor, op="edit", diff=changes)
|
|
469
|
+
loaded = await self.load()
|
|
470
|
+
assert loaded is not None
|
|
471
|
+
return loaded
|
|
472
|
+
raise RegistryConflict("the registry kept changing; try again")
|
|
473
|
+
|
|
474
|
+
@store_op
|
|
475
|
+
async def set(
|
|
476
|
+
self, values: Mapping[str, Any], *, actor: str, model: str | None = None
|
|
477
|
+
) -> ResolvedRegistry:
|
|
478
|
+
"""Set dotted paths (`{"rpm": 400}` with `model=`, or from the root); None deletes."""
|
|
479
|
+
|
|
480
|
+
def change(doc: dict[str, Any]) -> None:
|
|
481
|
+
base = doc
|
|
482
|
+
prefix: list[str] = []
|
|
483
|
+
if model is not None:
|
|
484
|
+
alias = self.current.registry.alias_of(model) if self._current else model
|
|
485
|
+
prefix = ["models", alias]
|
|
486
|
+
base = doc.get("models", {}).get(alias, {})
|
|
487
|
+
for dotted, value in values.items():
|
|
488
|
+
_set_path(doc, [*prefix, *split_path(base, dotted)], value)
|
|
489
|
+
|
|
490
|
+
return await self.edit(change, actor=actor)
|
|
491
|
+
|
|
492
|
+
@store_op
|
|
493
|
+
async def set_enabled(
|
|
494
|
+
self, model: str, deployment: str, enabled: bool, *, actor: str
|
|
495
|
+
) -> ResolvedRegistry:
|
|
496
|
+
"""Enable or disable one deployment (`account/name`) of a model."""
|
|
497
|
+
account, _, name = deployment.partition("/")
|
|
498
|
+
alias = self.current.registry.alias_of(model)
|
|
499
|
+
path = ["models", alias, "deployments", account, "overrides", name, "enabled"]
|
|
500
|
+
|
|
501
|
+
def change(doc: dict[str, Any]) -> None:
|
|
502
|
+
_set_path(doc, path, None if enabled else False)
|
|
503
|
+
|
|
504
|
+
return await self.edit(change, actor=actor)
|
|
505
|
+
|
|
506
|
+
@store_op
|
|
507
|
+
async def block(
|
|
508
|
+
self, *, region: str | None = None, account: str | None = None, on: bool, actor: str
|
|
509
|
+
) -> ResolvedRegistry:
|
|
510
|
+
"""Block (or unblock) a region or an account for every model.
|
|
511
|
+
|
|
512
|
+
`ConfigError` when neither is given, or when blocking a name the registry does not use
|
|
513
|
+
(a typo would otherwise block nothing and look done).
|
|
514
|
+
"""
|
|
515
|
+
if not region and not account:
|
|
516
|
+
raise ConfigError("give a region or an account to block")
|
|
517
|
+
if on:
|
|
518
|
+
view = self.current
|
|
519
|
+
accounts = set(view.registry.raw.get("accounts", {}))
|
|
520
|
+
regions = {d.region for alias in view.registry.models for d in view.deployments(alias)}
|
|
521
|
+
if account and account not in accounts:
|
|
522
|
+
raise ConfigError(f"unknown account {account!r}")
|
|
523
|
+
if region and region not in regions:
|
|
524
|
+
raise ConfigError(f"no deployment uses region {region!r}")
|
|
525
|
+
|
|
526
|
+
def change(doc: dict[str, Any]) -> None:
|
|
527
|
+
routing = doc.setdefault("routing", {})
|
|
528
|
+
for field, value in (("blocked_regions", region), ("blocked_accounts", account)):
|
|
529
|
+
if value is None:
|
|
530
|
+
continue
|
|
531
|
+
listed = [v for v in routing.get(field, []) if v != value]
|
|
532
|
+
routing[field] = [*listed, value] if on else listed
|
|
533
|
+
|
|
534
|
+
return await self.edit(change, actor=actor)
|
|
535
|
+
|
|
536
|
+
# ---------------------------------------------------------------- history, rollback, diff
|
|
537
|
+
|
|
538
|
+
@store_op
|
|
539
|
+
async def log(self, count: int = 20, path_prefix: str = "") -> list[dict[str, Any]]:
|
|
540
|
+
"""Latest audit entries, newest first; `path_prefix` keeps entries that touch it."""
|
|
541
|
+
entries = await self._client.xrevrange(self.audit_key, count=max(count * 5, count))
|
|
542
|
+
out: list[dict[str, Any]] = []
|
|
543
|
+
for entry_id, fields in entries or ():
|
|
544
|
+
row = {_text(k): _text(v) for k, v in (fields or {}).items()}
|
|
545
|
+
changes = orjson.loads(row.get("diff", "[]"))
|
|
546
|
+
if path_prefix and not any(c["path"].startswith(path_prefix) for c in changes):
|
|
547
|
+
continue
|
|
548
|
+
out.append({"id": _text(entry_id), **row, "diff": changes})
|
|
549
|
+
if len(out) == count:
|
|
550
|
+
break
|
|
551
|
+
return out
|
|
552
|
+
|
|
553
|
+
@store_op
|
|
554
|
+
async def versions(self) -> list[int]:
|
|
555
|
+
"""Published versions still kept for rollback, oldest first."""
|
|
556
|
+
return sorted(int(_text(v)) for v in await self._client.hkeys(self.history_key))
|
|
557
|
+
|
|
558
|
+
@store_op
|
|
559
|
+
async def rollback(self, version: int, *, actor: str) -> PublishResult:
|
|
560
|
+
"""Publish the content of an earlier version as a new, higher version."""
|
|
561
|
+
raw = await self._client.hget(self.history_key, str(version))
|
|
562
|
+
if raw is None:
|
|
563
|
+
raise ConfigError(f"version {version} is not kept any more")
|
|
564
|
+
doc = orjson.loads(raw)
|
|
565
|
+
_, meta = await self._read()
|
|
566
|
+
doc["registry_version"] = int(meta.get("version", "0")) + 1
|
|
567
|
+
return await self.publish(doc, actor=f"{actor} (rollback to {version})")
|
|
568
|
+
|
|
569
|
+
@store_op
|
|
570
|
+
async def diff_with_config(self) -> list[dict[str, Any]]:
|
|
571
|
+
"""What is live but not in this process's config (live edits a deploy would drop)."""
|
|
572
|
+
doc, _ = await self._read()
|
|
573
|
+
if self.config is None or doc is None:
|
|
574
|
+
return []
|
|
575
|
+
return [
|
|
576
|
+
change
|
|
577
|
+
for change in audit_diff(self.config, doc)
|
|
578
|
+
if change["path"] != "registry_version"
|
|
579
|
+
]
|
|
580
|
+
|
|
581
|
+
# ---------------------------------------------------------------- reload loop
|
|
582
|
+
|
|
583
|
+
async def refresh(self) -> ResolvedRegistry | None:
|
|
584
|
+
"""One reload: load, and restore the config when the stored registry was lost.
|
|
585
|
+
|
|
586
|
+
A missing or older stored registry while this process holds a newer one means the store
|
|
587
|
+
was flushed or failed over: republish by the boot rule and alert (live edits are lost).
|
|
588
|
+
"""
|
|
589
|
+
held = self._version
|
|
590
|
+
loaded = await self.load()
|
|
591
|
+
if self.config is None:
|
|
592
|
+
return loaded
|
|
593
|
+
try:
|
|
594
|
+
doc, meta = await self._read()
|
|
595
|
+
except (RedisError, OSError):
|
|
596
|
+
return self._current
|
|
597
|
+
stored = int(meta.get("version", "0")) if doc is not None else 0
|
|
598
|
+
if doc is None or stored < held:
|
|
599
|
+
logger.error("registry_reset_live_edits_lost", stored=stored, held=held)
|
|
600
|
+
await self.publish(self.config, actor="restore")
|
|
601
|
+
return self._current
|
|
602
|
+
|
|
603
|
+
async def run(self, interval_secs: float) -> None:
|
|
604
|
+
"""Reload every `interval_secs` until cancelled; errors never end the loop."""
|
|
605
|
+
while True:
|
|
606
|
+
try:
|
|
607
|
+
await self.refresh()
|
|
608
|
+
except (RedisError, OSError, ConfigError) as exc:
|
|
609
|
+
logger.warning("registry_reload_failed", error=str(exc))
|
|
610
|
+
await self._sleep(interval_secs)
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _text(value: Any) -> str:
|
|
614
|
+
return value.decode() if isinstance(value, bytes) else str(value)
|