llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,2357 @@
1
+ """The engine of one process: caller and worker over the registry, queues and admission.
2
+
3
+ `Engine.call` registers a call (duplicate ids attach to the first run), sends it to its model's
4
+ queue and waits for the result. The worker side reads each model's priority queue first, admits
5
+ calls with one store call each, holds a refused call (and stops reading its class) until it is
6
+ granted or expires, runs granted calls with the attempt timers, retries by the refund rules and
7
+ falls back to fallback regions and other models by the registry's rules (plan §3.3 to §3.18).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ import contextlib
14
+ import hashlib
15
+ import hmac
16
+ import random
17
+ import time
18
+ from collections.abc import AsyncIterator, Mapping, Sequence
19
+ from dataclasses import dataclass, field
20
+ from typing import TYPE_CHECKING, Any
21
+
22
+ from redis.exceptions import ConnectionError as StoreConnectionError
23
+ from redis.exceptions import OutOfMemoryError, ResponseError
24
+ from redis.exceptions import ReadOnlyError as StoreReadOnlyError
25
+ from redis.exceptions import TimeoutError as StoreTimeoutError
26
+
27
+ from llm_queue._logging import get_logger
28
+ from llm_queue.clients.pool import CredentialsUnavailable, route_for
29
+ from llm_queue.clients.providers import Providers
30
+ from llm_queue.clients.timing import AttemptTimeout, Timing
31
+ from llm_queue.engine import hooks, inference, tokens
32
+ from llm_queue.engine.admission import (
33
+ Admission,
34
+ Ask,
35
+ Candidate,
36
+ Decision,
37
+ owner_token,
38
+ position_of,
39
+ )
40
+ from llm_queue.engine.envelope import Envelope, Meta, Reply, params_hash, parse
41
+ from llm_queue.engine.errors import CallFailed
42
+ from llm_queue.engine.health import Health
43
+ from llm_queue.engine.keys import Keys
44
+ from llm_queue.engine.metrics import Metrics, NullSink
45
+ from llm_queue.engine.offload import ResultOffload
46
+ from llm_queue.engine.queue import (
47
+ Message,
48
+ NotReplicated,
49
+ QueueFull,
50
+ QueueSettings,
51
+ StreamQueues,
52
+ )
53
+ from llm_queue.engine.requests import RequestStore
54
+ from llm_queue.engine.status import AlertEvaluator, StatusReader
55
+ from llm_queue.engine.streaming import ChunkReader, ChunkWriter, StreamTooLarge
56
+ from llm_queue.errors import ErrorKind, UnsupportedEnvelopeError
57
+ from llm_queue.ids import new_id
58
+ from llm_queue.providers.classify import is_target_error
59
+ from llm_queue.tuning import EngineTimings
60
+
61
+ if TYPE_CHECKING:
62
+ import redis.asyncio
63
+
64
+ from llm_queue.registry import Deployment, ResolvedRegistry
65
+ from llm_queue.registry.store import RegistryStore
66
+
67
+ logger = get_logger(__name__)
68
+ OPS = {("bedrock", "chat"): "converse", ("bedrock", "embedding"): "invoke_model"}
69
+
70
+
71
+ def _probe_params(provider: str, kind: str, model_id: str) -> dict[str, Any]:
72
+ """The default probe request: one short word, one output token."""
73
+ if kind == "embedding":
74
+ if provider == "openai":
75
+ return {"input": "ping"}
76
+ if "cohere" in model_id:
77
+ return {"body": {"texts": ["ping"], "input_type": "search_query"}}
78
+ return {"body": {"inputText": "ping"}}
79
+ if provider == "openai":
80
+ return {"messages": [{"role": "user", "content": "ping"}], "max_tokens": 1}
81
+ return {
82
+ "messages": [{"role": "user", "content": [{"text": "ping"}]}],
83
+ "inferenceConfig": {"maxTokens": 1},
84
+ }
85
+
86
+
87
+ def _auth_error(code: str) -> bool:
88
+ """Credential errors: drop the cached client so the secret is read again."""
89
+ lowered = code.lower()
90
+ return any(
91
+ part in lowered
92
+ for part in ("unrecognizedclient", "expiredtoken", "authentication", "invalid_api_key")
93
+ )
94
+
95
+
96
+ @dataclass(frozen=True, slots=True)
97
+ class EngineSettings:
98
+ """Process-level settings (plan §3.18 infrastructure table)."""
99
+
100
+ queue_prefix: str = ""
101
+ lease_secs: float = 60.0
102
+ waiter_secs: float = 2.0
103
+ result_ttl_secs: int = 300
104
+ result_dedupe_secs: int = 0
105
+ status_ttl_secs: int = 1800
106
+ caller_ttl_secs: int = 30
107
+ poll_secs: float = 5.0 # safety-net result check (the wake-up is published with the result)
108
+ sweep_secs: float = 10.0
109
+ receive_batch: int = 10
110
+ caller_margin_secs: float = 5.0
111
+ local_limit_cap: int = 10
112
+ drain_secs: float = 45.0
113
+ id_hash_salt: str = "" # LLMQ_ID_HASH_SALT: the key of every user / session id hash
114
+ cfg_reload_secs: float = 30.0 # LLMQ_CFG_RELOAD_SECS: how often the registry is re-read
115
+ queue_failover_secs: float = 5.0 # queue-store errors this long before calls run direct
116
+ queue_wait_replicas: int = 0 # a send waits for this many replicas (0: off)
117
+ queue_wait_ms: int = 100 # how long it waits for them
118
+ debug_capture_allowed: bool = False # LLMQ_DEBUG_CAPTURE_FROM_CONFIG (never in prod)
119
+ timings: EngineTimings = field(default_factory=EngineTimings) # `Tuning.engine`
120
+
121
+
122
+ # ARGV: key prefix, request ids... Reply: the 0-based indexes of the ids with a stored result.
123
+ _RESULTS_EXIST = """
124
+ local out = {}
125
+ for i = 2, #ARGV do
126
+ if redis.call('EXISTS', ARGV[1] .. ':{r:' .. ARGV[i] .. '}:res') == 1 then
127
+ table.insert(out, i - 2)
128
+ end
129
+ end
130
+ return out
131
+ """
132
+
133
+
134
+ def _refusal(exc: BaseException) -> str:
135
+ """A caller-facing message for a store that answered with a refusal."""
136
+ if isinstance(exc, OutOfMemoryError) or "OOM" in str(exc):
137
+ return "the state store is out of memory" # redis-py strips the "OOM" prefix
138
+ return f"the state store refused the call: {exc}"
139
+
140
+
141
+ # READONLY: a failover left this connection on the old primary (now a replica).
142
+ STORE_ERRORS = (
143
+ StoreConnectionError,
144
+ StoreTimeoutError,
145
+ StoreReadOnlyError,
146
+ ConnectionError,
147
+ OSError,
148
+ )
149
+
150
+
151
+ @dataclass(slots=True)
152
+ class _Held:
153
+ """A call this process holds (waiting for budget, or running).
154
+
155
+ `message` is None on the direct path (kill switch, `queue_pct`, queue-store failover); its
156
+ position is then the store's time at registration.
157
+ """
158
+
159
+ message: Message | None
160
+ envelope: Envelope
161
+ first_seen: float
162
+ position_ms: int
163
+ tag: str
164
+ grants: set[tuple[str, str]] = field(default_factory=set)
165
+ meta: dict[str, dict[str, Any]] = field(default_factory=dict)
166
+ broken: bool = False
167
+ cancelled: bool = False
168
+ target_failed: list[tuple[str, str]] = field(default_factory=list)
169
+ run: asyncio.Task[None] | None = None
170
+ # Model-level request and estimate, built once per registry view (a deep copy of the
171
+ # prompt each time: ~0.7 ms for 200 KB, and it was rebuilt in _ask, _decide and _attempt).
172
+ charged: tuple[Any, str, tokens.Estimate] | None = None
173
+
174
+ @property
175
+ def position(self) -> float:
176
+ """Queue position as a number (direct calls: their registration time)."""
177
+ if self.message is not None:
178
+ return position_of(self.message.id)
179
+ return float(self.position_ms * 1000)
180
+
181
+
182
+ class _QueueDown(Exception):
183
+ """The queue store refused a send while the state store works: run the call direct."""
184
+
185
+
186
+ class _SendNotReplicated(Exception): # internal signal, never raised to callers
187
+ """The send reached the primary but no replica in time: the caller runs it direct."""
188
+
189
+
190
+ class StoreClock:
191
+ """The state store's clock, from one `TIME` read and the local monotonic clock."""
192
+
193
+ def __init__(self) -> None:
194
+ """No offset until `sync`."""
195
+ self._offset = 0.0
196
+
197
+ async def sync(self, client: redis.asyncio.Redis) -> None:
198
+ """Measure the offset between the store's clock and this process."""
199
+ before = time.monotonic()
200
+ secs, micros = await client.time()
201
+ after = time.monotonic()
202
+ self._offset = (secs + micros / 1e6) - (before + after) / 2
203
+
204
+ def now(self) -> float:
205
+ """Seconds on the store's clock."""
206
+ return time.monotonic() + self._offset
207
+
208
+
209
+ class Engine:
210
+ """Caller and worker of one process (plan E72: workers run in the app pods)."""
211
+
212
+ def __init__(
213
+ self,
214
+ state: redis.asyncio.Redis,
215
+ registry: RegistryStore,
216
+ providers: Providers,
217
+ *,
218
+ queue_store: redis.asyncio.Redis | None = None,
219
+ settings: EngineSettings | None = None,
220
+ worker_id: str | None = None,
221
+ metrics: Metrics | None = None,
222
+ offload: ResultOffload | None = None,
223
+ offload_bytes: int | None = None,
224
+ ) -> None:
225
+ """Wire the stores; nothing runs until `start`."""
226
+ self.settings = settings or EngineSettings()
227
+ self.keys = Keys(registry.prefix)
228
+ self.state = state
229
+ self.registry = registry
230
+ self.providers = providers
231
+ self._queue_store = queue_store
232
+ self.queues = StreamQueues(queue_store or state, self.keys)
233
+ self.requests = RequestStore(
234
+ state,
235
+ self.keys,
236
+ result_ttl_secs=self.settings.result_ttl_secs,
237
+ result_dedupe_secs=self.settings.result_dedupe_secs,
238
+ status_ttl_secs=self.settings.status_ttl_secs,
239
+ offload=offload,
240
+ offload_bytes=offload_bytes,
241
+ )
242
+ self.admission = Admission(
243
+ state,
244
+ self.keys,
245
+ lease_secs=self.settings.lease_secs,
246
+ waiter_secs=self.settings.waiter_secs,
247
+ timings=self.settings.timings,
248
+ )
249
+ self.worker_id = worker_id or new_id()[-14:] # the random tail: the head is time
250
+ self.clock = StoreClock()
251
+ self._tasks: list[asyncio.Task[None]] = []
252
+ self._running: set[asyncio.Task[None]] = set()
253
+ self._held: dict[str, _Held] = {}
254
+ self._held_class: set[tuple[str, str]] = set()
255
+ self._slots: dict[str, asyncio.Semaphore] = {}
256
+ self._waiters: dict[str, asyncio.Future[None]] = {}
257
+ self._pubsub: Any = None
258
+ self._cancels: Any = None
259
+ self._stopping = False
260
+ self._paused = False
261
+ self._local_callers: dict[str, int] = {} # request id -> coroutines waiting on it here
262
+ self._wake: dict[str, asyncio.Event] = {} # model -> wakes its receive loop
263
+ self._beat_ok_at = time.monotonic() # last heartbeat that reached the store
264
+ self._store_trouble_at = -1e9 # monotonic time this process last saw the store fail
265
+ self._results_exist = state.register_script(_RESULTS_EXIST)
266
+ self.ready = False
267
+ self._direct_locks: dict[tuple[str, str], asyncio.Lock] = {}
268
+ self.health = Health(state, self.keys)
269
+ self.metrics: Metrics = metrics or NullSink()
270
+ self.status_reader = StatusReader(state, self.keys, self.queues, self.settings.queue_prefix)
271
+ self.alerts = AlertEvaluator(state, self.keys)
272
+ # Credentials that failed in this process, per account and region: (account, region)
273
+ # -> (until, backoff). Per region: a token can be refused in one region only (an
274
+ # opt-in region such as ap-south-2 refuses v1 STS tokens), and keying by account took
275
+ # every region of it down (scenario P5).
276
+ self._bad_accounts: dict[tuple[str, str], tuple[float, float]] = {}
277
+ self._idle = asyncio.Event()
278
+ self._static: dict[str, asyncio.Semaphore] = {}
279
+ self._rng = random.Random()
280
+ self.chunks = ChunkReader(state, block_ms=self.settings.timings.stream_read_block_ms)
281
+
282
+ # ================================================================ lifecycle
283
+
284
+ async def start(self, *, worker: bool = True, caller: bool = True) -> None:
285
+ """Boot the registry, then start the worker loops and the result reader."""
286
+ await self.registry.boot() # runs on its own config when the store is down
287
+ try:
288
+ await self.clock.sync(self.state)
289
+ await self.state.set(self.keys.epoch, str(self.clock.now()), nx=True)
290
+ except STORE_ERRORS as exc:
291
+ # Start anyway (plan §3.18): calls follow each model's `store_down` rule and
292
+ # the background loops keep retrying until the store answers.
293
+ logger.error("state_store_unreachable_at_start", error=str(exc))
294
+ if caller:
295
+ self._pubsub = self.state.pubsub()
296
+ self._tasks.append(asyncio.create_task(self._result_reader()))
297
+ self._tasks.append(asyncio.create_task(self._result_sweep()))
298
+ if worker:
299
+ for alias in self.registry.current.registry.models:
300
+ slots = int(self.registry.current.value(alias, "worker_slots"))
301
+ self._slots[alias] = asyncio.Semaphore(slots)
302
+ self._tasks.append(asyncio.create_task(self._model_loop(alias)))
303
+ self._tasks.append(asyncio.create_task(self._sweeper()))
304
+ self._tasks.append(asyncio.create_task(self._prober()))
305
+ self._tasks.append(asyncio.create_task(self._alert_loop()))
306
+ self._cancels = self.state.pubsub()
307
+ self._tasks.append(asyncio.create_task(self._cancel_reader()))
308
+ self._tasks.append(asyncio.create_task(self._warm_clients()))
309
+ self._tasks.append(asyncio.create_task(self._heartbeat()))
310
+ self._tasks.append(asyncio.create_task(self.registry.run(self.settings.cfg_reload_secs)))
311
+ self.ready = True
312
+
313
+ def pause_receiving(self) -> None:
314
+ """Not ready, and take no new calls from the queues; running and held calls go on.
315
+
316
+ The first step of a shutdown, for a preStop hook: the load balancer stops sending
317
+ requests while this pod finishes its work. `stop()` follows.
318
+ """
319
+ self.ready = False
320
+ self._paused = True
321
+
322
+ async def stop(self, drain_secs: float | None = None) -> None:
323
+ """Shut down in order (plan §3.18): not ready, stop receiving, finish running calls.
324
+
325
+ Held calls go back to the queue; results keep being read for this pod's own callers
326
+ until they finish or the drain ends; then everything stops.
327
+ """
328
+ drain = self.settings.drain_secs if drain_secs is None else drain_secs
329
+ loop = asyncio.get_running_loop()
330
+ until = loop.time() + drain
331
+ self.ready = False
332
+ self._stopping = True
333
+ for held in list(self._held.values()):
334
+ if not held.grants and held.run is None and held.message is not None:
335
+ with contextlib.suppress(*STORE_ERRORS):
336
+ await self.queues.return_(held.message)
337
+ self._drop(held) # else the heartbeat hides it again for the whole drain
338
+ if self._running:
339
+ await asyncio.wait(set(self._running), timeout=max(0.0, until - loop.time()))
340
+ if self._waiters:
341
+ self._idle.clear()
342
+ with contextlib.suppress(TimeoutError):
343
+ await asyncio.wait_for(self._idle.wait(), max(0.0, until - loop.time()))
344
+ for task in [*self._tasks, *self._running]:
345
+ task.cancel()
346
+ for task in [*self._tasks, *self._running]:
347
+ with contextlib.suppress(asyncio.CancelledError, Exception):
348
+ await task
349
+ await self.chunks.aclose()
350
+ for pubsub in (self._pubsub, self._cancels):
351
+ if pubsub is not None:
352
+ with contextlib.suppress(Exception):
353
+ await pubsub.aclose()
354
+
355
+ async def aclose(self) -> None:
356
+ """Close the provider clients and both store clients (after `stop()`)."""
357
+ await self.providers.aclose()
358
+ if self._queue_store is not None:
359
+ await self._queue_store.aclose()
360
+ await self.state.aclose()
361
+
362
+ # ================================================================ caller
363
+
364
+ def _queue_name(self, view: ResolvedRegistry, alias: str, priority: bool) -> str:
365
+ name = self.settings.queue_prefix + view.queue_name(alias)
366
+ return f"{name}-priority" if priority else name
367
+
368
+ async def call(
369
+ self,
370
+ model: str,
371
+ params: Mapping[str, Any],
372
+ *,
373
+ priority: bool = False,
374
+ timeout_secs: float | None = None,
375
+ max_wait_secs: float | None = None,
376
+ request_id: str | None = None,
377
+ session_id: str | None = None,
378
+ user_id: str | None = None,
379
+ tenant_id: str | None = None,
380
+ allow_fallback: str = "all",
381
+ allowed_regions: Sequence[str] | None = None,
382
+ allowed_accounts: Sequence[str] | None = None,
383
+ force_direct: bool = False,
384
+ stream: bool = False,
385
+ stream_id: str | None = None,
386
+ ) -> Reply:
387
+ """Run one call through the queue and return its reply; `CallFailed` on failure.
388
+
389
+ `force_direct` runs it in this process (sync calls), with the same admission.
390
+ `stream` has the worker write events to the chunk stream of `stream_id` (`stream`).
391
+ """
392
+ view = self.registry.current
393
+ alias = view.registry.alias_of(model)
394
+ if view.effective(alias, "enabled").level == "model" and not view.value(alias, "enabled"):
395
+ rule = view.value(alias, "model_fallback")
396
+ if rule and allow_fallback == "all" and rule["models"]:
397
+ alias = rule["models"][0]["model"]
398
+ else:
399
+ raise CallFailed("model_disabled", f"{alias} is disabled")
400
+ entry = view.registry.models[alias]
401
+ op = OPS.get((entry.provider, entry.type), "chat" if entry.type == "chat" else "embeddings")
402
+ timeout = float(timeout_secs or view.value(alias, "timeout_secs"))
403
+ timeout = min(timeout, float(view.value(alias, "timeout_max_secs")))
404
+ rid = request_id or stream_id or new_id()
405
+ envelope = Envelope(
406
+ request_id=rid,
407
+ op=op,
408
+ model=alias,
409
+ params=dict(params),
410
+ priority=priority,
411
+ caller=self.worker_id,
412
+ timeout_secs=timeout,
413
+ max_wait_secs=max_wait_secs or view.value(alias, "max_wait_secs"),
414
+ allow_fallback=allow_fallback,
415
+ allowed_regions=tuple(allowed_regions) if allowed_regions else None,
416
+ allowed_accounts=tuple(allowed_accounts) if allowed_accounts else None,
417
+ session_id=session_id,
418
+ user_id=user_id,
419
+ tenant_id=tenant_id,
420
+ app_request_id=request_id is not None,
421
+ stream=stream,
422
+ )
423
+ hook_name = view.value(alias, "admission")
424
+ hook = hooks.get("admission", hook_name)
425
+ if hook is not None:
426
+ ctx = hooks.CallContext(alias, rid, priority, session_id, user_id, tenant_id)
427
+ verdict = hooks.call_safely("admission", hook_name, hook, ctx, default=None)
428
+ if isinstance(verdict, hooks.Reject):
429
+ raise CallFailed("rejected", verdict.reason, request_id=rid)
430
+ direct = force_direct or self._direct_route(view, alias, rid, user_id)
431
+ self._local_callers[rid] = self._local_callers.get(rid, 0) + 1
432
+ # One deadline for the whole call, whatever path it takes: a store outage sent calls
433
+ # down the store_down path with a fresh timeout, and the soak saw 135 answers come
434
+ # after their deadline (20 s store block, 30 s calls).
435
+ deadline = asyncio.get_running_loop().time() + timeout + self.settings.caller_margin_secs
436
+ try:
437
+ async with asyncio.timeout_at(deadline):
438
+ try:
439
+ return await self._call_once(
440
+ view, alias, envelope, params, priority, timeout, request_id, direct
441
+ )
442
+ except _QueueDown:
443
+ return await self._call_once(
444
+ view, alias, envelope, params, priority, timeout, request_id, True
445
+ )
446
+ except STORE_ERRORS as exc:
447
+ if stream: # chunks travel through the state store
448
+ message = "the state store is unreachable"
449
+ raise CallFailed("queue_unavailable", message) from exc
450
+ return await self._store_down(view, alias, envelope, exc, deadline)
451
+ except ResponseError as exc:
452
+ # The store answered but refused (out of memory, a script error): still one
453
+ # error family for callers. A refused registration raised redis-py's
454
+ # OutOfMemoryError straight to the app (scenario S9).
455
+ raise CallFailed("queue_unavailable", _refusal(exc), request_id=rid) from exc
456
+ except TimeoutError as exc:
457
+ raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid) from exc
458
+ except asyncio.CancelledError:
459
+ # The awaiting task was cancelled: cancel the call unless others wait on its id,
460
+ # in other processes or in this one (they share this process's caller entry).
461
+ with contextlib.suppress(Exception):
462
+ callers = await asyncio.shield(self.requests.callers(rid))
463
+ alone_here = self._local_callers.get(rid, 0) <= 1
464
+ if alone_here and set(callers) <= {self.worker_id}:
465
+ await asyncio.shield(self.cancel(rid))
466
+ raise
467
+ finally:
468
+ left = self._local_callers.get(rid, 1) - 1
469
+ if left:
470
+ self._local_callers[rid] = left
471
+ else:
472
+ self._local_callers.pop(rid, None)
473
+
474
+ async def stream(
475
+ self, model: str, params: Mapping[str, Any], **call: Any
476
+ ) -> AsyncIterator[dict[str, Any]]:
477
+ """Run a chat call and yield the provider's events as they arrive (plan §3.13, F6).
478
+
479
+ Takes `call`'s options. The events are ConverseStream events (Bedrock) or chat
480
+ completion chunks (OpenAI-compatible). A failure after events were yielded raises
481
+ `CallFailed` with `partial_text`; no switch to another deployment happens then.
482
+ """
483
+ view = self.registry.current
484
+ alias = view.registry.alias_of(model)
485
+ idle = float(view.value(alias, "stream_idle_secs"))
486
+ rid = call.pop("request_id", None) or new_id()
487
+ key = self.keys.request(rid, "chunks")
488
+ queue = self.chunks.open(key)
489
+ # The call's own deadline, as `call` computes it: the idle wait between chunks
490
+ # (stream_idle_secs, 30) once let a stream whose worker died answer after it (scenario
491
+ # L2 at 300 %: 38.8 s for a 15 s deadline).
492
+ timeout = float(call.get("timeout_secs") or view.value(alias, "timeout_secs"))
493
+ timeout = min(timeout, float(view.value(alias, "timeout_max_secs")))
494
+ loop = asyncio.get_running_loop()
495
+ deadline = loop.time() + timeout + self.settings.caller_margin_secs
496
+ task = asyncio.create_task(self.call(model, params, stream=True, stream_id=rid, **call))
497
+ expected, started = 1, False
498
+ try:
499
+ while True:
500
+ left = deadline - loop.time()
501
+ if left <= 0:
502
+ raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid)
503
+ getter = asyncio.ensure_future(queue.get())
504
+ finished = task.done()
505
+ done, _ = await asyncio.wait(
506
+ {getter} if finished else {getter, task},
507
+ timeout=min(idle, left) if started or finished else left,
508
+ return_when=asyncio.FIRST_COMPLETED,
509
+ )
510
+ if getter not in done:
511
+ getter.cancel()
512
+ if not done and deadline - loop.time() <= 0: # the deadline, not idleness
513
+ raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid)
514
+ if task in done:
515
+ if not started:
516
+ # The store may be down too: then nothing can be replayed, and
517
+ # the call's own failure is the answer (scenario L2: this read
518
+ # raised redis-py's ConnectionError to the app).
519
+ try:
520
+ pending = await self.state.xlen(key)
521
+ except STORE_ERRORS:
522
+ task.result() # the call's failure, if it failed
523
+ raise CallFailed(
524
+ "queue_unavailable",
525
+ "the store is unreachable; the stream cannot be read",
526
+ request_id=rid,
527
+ ) from None
528
+ if not pending:
529
+ task.result() # raises the call's failure (nothing streamed)
530
+ return # a finished call whose chunks expired
531
+ continue # its last entries are still on the way
532
+ if finished:
533
+ task.result()
534
+ raise CallFailed(
535
+ "worker_lost", "the stream ended without its end entry", request_id=rid
536
+ )
537
+ raise CallFailed(
538
+ "deadline_exceeded",
539
+ f"no chunk for {idle:g} s (stream_idle_secs)",
540
+ request_id=rid,
541
+ )
542
+ entry = getter.result()
543
+ if entry["seq"] != expected:
544
+ raise CallFailed(
545
+ "worker_lost", f"gap in the stream at {expected}", request_id=rid
546
+ )
547
+ expected += 1
548
+ started = True
549
+ if entry["t"] == "chunk":
550
+ for event in entry["d"]:
551
+ yield event
552
+ elif entry["t"] == "end":
553
+ await task
554
+ return
555
+ else:
556
+ data = entry["d"]
557
+ raise CallFailed(
558
+ data.get("kind", "provider_error"),
559
+ data.get("message", ""),
560
+ partial_text=data.get("partial") or None,
561
+ request_id=rid,
562
+ )
563
+ finally:
564
+ self.chunks.close(key)
565
+ if not task.done():
566
+ task.cancel() # the call's own handler cancels the request
567
+ with contextlib.suppress(BaseException):
568
+ await task
569
+
570
+ def hash_id(self, value: str) -> str:
571
+ """Keyed hash (HMAC-SHA256, `LLMQ_ID_HASH_SALT`) of a user or session id, 16 hex chars.
572
+
573
+ Every id outside the message itself is stored and logged only as this (plan §3.17):
574
+ status records, sticky keys, logs and `routing.allow_users`. Unkeyed, a low-entropy id
575
+ (a client code, a phone number) is brute-forced back from the hash.
576
+ """
577
+ key = self.settings.id_hash_salt.encode()
578
+ return hmac.new(key, value.encode(), hashlib.sha256).hexdigest()[:16]
579
+
580
+ def _direct_route(
581
+ self, view: ResolvedRegistry, alias: str, rid: str, user_id: str | None
582
+ ) -> bool:
583
+ """True when this call runs direct: kill switch, or outside `queue_pct` (canary)."""
584
+ routing = view.registry.routing
585
+ if user_id and self.hash_id(user_id) in routing.allow_users:
586
+ return False
587
+ if routing.kill_switch:
588
+ return True
589
+ pct = int(view.value(alias, "queue_pct"))
590
+ return pct < 100 and int(hashlib.sha256(rid.encode()).hexdigest()[:8], 16) % 100 >= pct
591
+
592
+ async def _call_once(
593
+ self,
594
+ view: ResolvedRegistry,
595
+ alias: str,
596
+ envelope: Envelope,
597
+ params: Mapping[str, Any],
598
+ priority: bool,
599
+ call_secs: float,
600
+ request_id: str | None,
601
+ direct: bool,
602
+ ) -> Reply:
603
+ rid = envelope.request_id
604
+ user_id, session_id = envelope.user_id, envelope.session_id
605
+ user_hash = self.hash_id(user_id or session_id or "")
606
+ found = await self.requests.register(
607
+ rid,
608
+ caller=self.worker_id,
609
+ user_hash=user_hash,
610
+ params_hash=params_hash({"m": alias, "p": params}),
611
+ model=alias,
612
+ priority=priority,
613
+ app_id=request_id is not None,
614
+ )
615
+ if found.outcome == "conflict":
616
+ raise CallFailed(
617
+ "request_id_conflict", "the id was used with other params", request_id=rid
618
+ )
619
+ if found.outcome == "expired":
620
+ raise CallFailed("result_expired", "the id finished earlier", request_id=rid)
621
+ waiter = self._waiters.setdefault(rid, asyncio.get_running_loop().create_future())
622
+ loop = asyncio.get_running_loop()
623
+ until = loop.time() + call_secs + self.settings.caller_margin_secs
624
+ outcome = found.outcome
625
+ try:
626
+ while True:
627
+ if outcome in ("new", "takeover"):
628
+ if direct:
629
+ await self.requests.confirm(rid, keep_secs=call_secs)
630
+ task = asyncio.create_task(self._direct(envelope))
631
+ self._running.add(task)
632
+ task.add_done_callback(self._running.discard)
633
+ else:
634
+ try:
635
+ await self._send(rid, envelope, view, alias, priority, call_secs)
636
+ except _SendNotReplicated:
637
+ await self.requests.confirm(rid, keep_secs=call_secs)
638
+ task = asyncio.create_task(self._direct(envelope))
639
+ self._running.add(task)
640
+ task.add_done_callback(self._running.discard)
641
+ reply = await self._wait_result(
642
+ rid, waiter, until - loop.time(), recheck=outcome == "attached"
643
+ )
644
+ if reply is not None:
645
+ return reply
646
+ # The send this caller attached to never happened: register again.
647
+ again = await self.requests.register(
648
+ rid,
649
+ caller=self.worker_id,
650
+ user_hash=user_hash,
651
+ params_hash=params_hash({"m": alias, "p": params}),
652
+ model=alias,
653
+ priority=priority,
654
+ app_id=request_id is not None,
655
+ )
656
+ outcome = again.outcome
657
+ if outcome == "conflict":
658
+ raise CallFailed("request_id_conflict", "the id was used with other params")
659
+ finally:
660
+ # Other coroutines here may wait on the same id and share its wake-up: removing
661
+ # it left them asleep until their deadline's last read (one waiter cancelled).
662
+ if self._local_callers.get(rid, 1) <= 1:
663
+ self._waiters.pop(rid, None)
664
+ if not self._waiters:
665
+ self._idle.set()
666
+
667
+ async def embed(
668
+ self,
669
+ model: str,
670
+ texts: Sequence[str],
671
+ *,
672
+ priority: bool = False,
673
+ timeout_secs: float | None = None,
674
+ input_type: str | None = None,
675
+ **call: Any,
676
+ ) -> list[list[float]]:
677
+ """One vector per text, in order (batched by the model's `embed_batch_size`)."""
678
+ from llm_queue.engine.embed import embed
679
+
680
+ return await embed(
681
+ self,
682
+ model,
683
+ texts,
684
+ priority=priority,
685
+ timeout_secs=timeout_secs,
686
+ input_type=input_type,
687
+ **call,
688
+ )
689
+
690
+ async def _send(
691
+ self,
692
+ rid: str,
693
+ envelope: Envelope,
694
+ view: ResolvedRegistry,
695
+ alias: str,
696
+ priority: bool,
697
+ keep_secs: float,
698
+ ) -> None:
699
+ """Put the message on its queue and confirm the record; undo the record on failure.
700
+
701
+ A separate queue store that keeps failing for `queue_failover_secs` sends the call
702
+ down the direct path; shorter blips are retried. With `queue_wait_replicas`, a send
703
+ that no replica acknowledged in time is a failed send (plan B23) and the call runs
704
+ direct.
705
+ """
706
+ settings = QueueSettings(max_waiting=int(view.value(alias, "max_waiting")))
707
+ loop = asyncio.get_running_loop()
708
+ give_up = loop.time() + self.settings.queue_failover_secs
709
+ body = envelope.dumps()
710
+ while True:
711
+ try:
712
+ await self.queues.send(
713
+ self._queue_name(view, alias, priority),
714
+ body,
715
+ settings,
716
+ wait_replicas=self.settings.queue_wait_replicas,
717
+ wait_ms=self.settings.queue_wait_ms,
718
+ )
719
+ break
720
+ except QueueFull as exc:
721
+ await self.requests.abandon(rid, caller=self.worker_id)
722
+ raise CallFailed("queue_full", str(exc), retry_after_secs=5) from exc
723
+ except NotReplicated as exc:
724
+ # A failed send (plan B23): the call runs direct now, under this same record
725
+ # (re-registering raced the unreplicated copy's claim into a conflict). Not
726
+ # re-sent: each try would add a copy; the copy meets a claimed job and stops.
727
+ logger.warning("send_not_replicated", request_id=rid, error=str(exc))
728
+ raise _SendNotReplicated from exc
729
+ except ResponseError as exc:
730
+ await self.requests.abandon(rid, caller=self.worker_id)
731
+ if isinstance(exc, OutOfMemoryError) or "OOM" in str(exc):
732
+ raise CallFailed("queue_unavailable", _refusal(exc)) from exc
733
+ raise
734
+ except STORE_ERRORS as exc:
735
+ if not self.queues_shared and loop.time() < give_up:
736
+ await asyncio.sleep(0.25)
737
+ continue
738
+ await self.requests.abandon(rid, caller=self.worker_id)
739
+ if self.queues_shared:
740
+ raise
741
+ raise _QueueDown from exc
742
+ except BaseException:
743
+ await self.requests.abandon(rid, caller=self.worker_id)
744
+ raise
745
+ try:
746
+ await self.requests.confirm(rid, keep_secs=keep_secs)
747
+ except STORE_ERRORS as exc: # the message is on the queue: never run it a second time
748
+ logger.warning("confirm_failed", request_id=rid, error=str(exc))
749
+
750
+ @property
751
+ def queues_shared(self) -> bool:
752
+ """True when queues live in the state store (one store for both)."""
753
+ return self._queue_store is None
754
+
755
+ async def cancel(self, request_id: str) -> None:
756
+ """Cancel a call: a waiting one never starts, a running one is stopped."""
757
+ await self.state.set(
758
+ self.keys.request(request_id, "cancel"), "1", ex=self.settings.status_ttl_secs
759
+ )
760
+ await self.state.publish(self.keys.cancel_channel, request_id)
761
+
762
+ async def _cancel_reader(self) -> None:
763
+ """Stop running attempts whose call was cancelled (by any process)."""
764
+ subscribed = False
765
+ while True:
766
+ try:
767
+ if not subscribed:
768
+ await self._cancels.subscribe(self.keys.cancel_channel)
769
+ subscribed = True
770
+ message = await self._cancels.get_message(
771
+ ignore_subscribe_messages=True, timeout=1.0
772
+ )
773
+ except Exception as exc:
774
+ logger.warning("cancel_reader_error", error=str(exc))
775
+ subscribed = False
776
+ with contextlib.suppress(Exception):
777
+ await self._cancels.reset()
778
+ await asyncio.sleep(1)
779
+ continue
780
+ if not message:
781
+ continue
782
+ data = message["data"]
783
+ rid = data.decode() if isinstance(data, bytes) else str(data)
784
+ held = self._held.get(rid)
785
+ if held is not None and held.run is not None and not held.run.done():
786
+ held.cancelled = True
787
+ held.run.cancel()
788
+
789
+ async def _store_down(
790
+ self,
791
+ view: ResolvedRegistry,
792
+ alias: str,
793
+ envelope: Envelope,
794
+ exc: BaseException,
795
+ deadline: float,
796
+ ) -> Reply:
797
+ """The state store is unreachable: the model's `store_down` rule (plan §3.11).
798
+
799
+ `deadline` is the call's own (loop time, margin included): `pause` waits only until
800
+ then, never a fresh timeout from the moment the store failed.
801
+ """
802
+ mode = view.value(alias, "store_down")
803
+ logger.warning("store_down", model=alias, mode=mode, error=str(exc))
804
+ if mode == "static_cap":
805
+ return await self._static_cap(view, alias, envelope)
806
+ if mode == "pause":
807
+ loop = asyncio.get_running_loop()
808
+ until = deadline - self.settings.caller_margin_secs
809
+ while loop.time() < until:
810
+ await asyncio.sleep(1.0)
811
+ try:
812
+ await self.state.ping()
813
+ except STORE_ERRORS:
814
+ continue
815
+ return await self.call(
816
+ alias,
817
+ envelope.params or {},
818
+ priority=envelope.priority,
819
+ timeout_secs=max(1.0, until - loop.time()),
820
+ request_id=envelope.request_id if envelope.app_request_id else None,
821
+ session_id=envelope.session_id,
822
+ user_id=envelope.user_id,
823
+ tenant_id=envelope.tenant_id,
824
+ )
825
+ raise CallFailed("queue_unavailable", "the state store is unreachable") from exc
826
+
827
+ async def _static_cap(self, view: ResolvedRegistry, alias: str, envelope: Envelope) -> Reply:
828
+ """Run without the store: at most `local_limit_cap` calls per process and model."""
829
+ entry = view.registry.models[alias]
830
+ cap = self._static.setdefault(alias, asyncio.Semaphore(self.settings.local_limit_cap))
831
+ provider = self.providers.get(entry.provider)
832
+ error: BaseException | None = None
833
+ async with cap:
834
+ for dep in view.deployments(alias, "confirmed"):
835
+ if not view.value(alias, "enabled", dep.key) or view.outside_residency(dep):
836
+ continue
837
+ route = route_for(view, dep)
838
+ params = self._params(view, alias, envelope, dep.key)
839
+ timing = Timing(
840
+ float(view.value(alias, "attempt_timeout_secs", dep.key)),
841
+ float(view.value(alias, "stream_idle_secs")),
842
+ )
843
+ try:
844
+ result = await provider.call(envelope.op, params, route, timing)
845
+ except Exception as exc:
846
+ error = exc
847
+ continue
848
+ meta = Meta(served_model=alias, deployment=dep.key, attempts=1)
849
+ return Reply(
850
+ request_id=envelope.request_id, ok=True, response=result.response, meta=meta
851
+ )
852
+ raise CallFailed("provider_error", str(error or "no usable deployment"))
853
+
854
+ async def _direct(self, env: Envelope) -> None:
855
+ """Run a call in this process (no queue), with the same admission and waiter order.
856
+
857
+ One waiter per process, model and class registers in the store; the others wait here.
858
+ """
859
+ held = _Held(
860
+ message=None,
861
+ envelope=env,
862
+ first_seen=self.clock.now(),
863
+ position_ms=int(self.clock.now() * 1000),
864
+ tag=f"d{new_id()[-10:]}", # random tail (a v7 id starts with the time)
865
+ )
866
+ self._held[env.request_id] = held
867
+ alias = env.model
868
+ cls = "p" if env.priority else "n"
869
+ lock = self._direct_locks.setdefault((alias, cls), asyncio.Lock())
870
+ try:
871
+ async with lock:
872
+ decision = await self._direct_wait(held)
873
+ if decision is not None:
874
+ await self._run(held, decision, 1, alias)
875
+ finally:
876
+ if not held.grants:
877
+ self._drop(held)
878
+
879
+ async def _direct_wait(self, held: _Held) -> Decision | None:
880
+ """Admission for a direct call until granted (the decision) or settled (None)."""
881
+ env = held.envelope
882
+ alias = env.model
883
+ while True:
884
+ view = self.registry.current
885
+ now = self.clock.now()
886
+ if now >= self._deadline(held):
887
+ await self._finish_call(
888
+ held, "rate_limited", "no budget before the deadline", final="expired"
889
+ )
890
+ return None
891
+ waited = now - held.position_ms / 1000.0
892
+ if env.max_wait_secs is not None and waited >= env.max_wait_secs:
893
+ await self._finish_call(
894
+ held, "rate_limited", "max_wait_secs passed", final="expired"
895
+ )
896
+ return None
897
+ decision = await self._decide(view, alias, held, 1, frozenset())
898
+ if decision is None:
899
+ return None
900
+ if decision.kind == "granted":
901
+ return decision
902
+ if await self._model_fallback(view, alias, held, 1, waited):
903
+ return None
904
+ await asyncio.sleep(min(max(decision.wait_secs, 0.01), 1.0))
905
+
906
+ def _rearm(self, rid: str, done: asyncio.Future[None]) -> asyncio.Future[None]:
907
+ """The id's next wake-up after `done` fired: made by the first waiter to see it, shared.
908
+
909
+ Each waiter making its own replaced the others', so all but the last slept until a
910
+ recheck (two coroutines waiting on one id in one process).
911
+ """
912
+ current = self._waiters.get(rid)
913
+ if current is None or current is done:
914
+ current = asyncio.get_running_loop().create_future()
915
+ self._waiters[rid] = current
916
+ return current
917
+
918
+ async def _wait_result(
919
+ self, rid: str, waiter: asyncio.Future[None], secs: float, *, recheck: bool = False
920
+ ) -> Reply | None:
921
+ """The reply; None (only with `recheck`) when the call's record disappeared."""
922
+ loop = asyncio.get_running_loop()
923
+ until = loop.time() + secs
924
+ if not recheck:
925
+ # Just sent: the reply cannot be there yet. Wait for the wake-up (or one poll)
926
+ # before the first read; reading at once cost a round trip on every call.
927
+ with contextlib.suppress(TimeoutError):
928
+ await asyncio.wait_for(asyncio.shield(waiter), min(secs, self.settings.poll_secs))
929
+ if waiter.done():
930
+ waiter = self._rearm(rid, waiter)
931
+ while True:
932
+ try:
933
+ reply = await self.requests.take_result(rid)
934
+ gone = reply is None and recheck and not await self.requests.job(rid)
935
+ except STORE_ERRORS as exc:
936
+ # The call is already sent or running: wait for the store, never re-run it
937
+ # on the store_down path (that ran it twice, once outside the budget).
938
+ if until - loop.time() <= 0:
939
+ raise CallFailed(
940
+ "deadline_exceeded", "the store did not answer in time", request_id=rid
941
+ ) from exc
942
+ await asyncio.sleep(min(0.5, max(0.0, until - loop.time())))
943
+ continue
944
+ if gone:
945
+ return None
946
+ if reply is not None:
947
+ if reply.ok:
948
+ return reply
949
+ raise CallFailed(
950
+ reply.error or "provider_error",
951
+ reply.message,
952
+ retry_after_secs=reply.retry_after_secs,
953
+ partial_text=reply.partial_text,
954
+ request_id=rid,
955
+ )
956
+ left = until - loop.time()
957
+ if left <= 0:
958
+ raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid)
959
+ # The wake-up comes by pub/sub (published with the result), or from this process's
960
+ # one result sweep (`_result_sweep`); a read per waiting caller made store load
961
+ # grow with the backlog (B56). The timeout is only a last-resort recheck.
962
+ with contextlib.suppress(TimeoutError):
963
+ await asyncio.wait_for(
964
+ asyncio.shield(waiter), min(left, self.settings.timings.result_recheck_secs)
965
+ )
966
+ if waiter.done():
967
+ waiter = self._rearm(rid, waiter)
968
+
969
+ async def _warm_clients(self) -> None:
970
+ """Create every Bedrock deployment's client now, not on its first call.
971
+
972
+ A client takes about a second to build (measured: the first call of a fresh worker took
973
+ 1,032 ms against ~140 ms after; B92 asks for <= 500 ms of first-call overhead).
974
+ Failures are ignored here: a call meets them again, and handles them.
975
+ """
976
+ view = self.registry.current
977
+ for alias, entry in view.registry.models.items():
978
+ if entry.provider != "bedrock":
979
+ continue
980
+ for deployment in view.deployments(alias):
981
+ with contextlib.suppress(Exception):
982
+ await self.providers.pool.bedrock(route_for(view, deployment))
983
+
984
+ async def _result_sweep(self) -> None:
985
+ """Every `poll_secs`, one store call finds results of this process's waiting callers.
986
+
987
+ The safety net behind the pub/sub wake-up: one script per process per period, however
988
+ many calls wait (a read per waiting caller grew with the backlog, plan B56).
989
+ """
990
+ while True:
991
+ await asyncio.sleep(self.settings.poll_secs)
992
+ waiting = [rid for rid, w in self._waiters.items() if not w.done()]
993
+ for start in range(0, len(waiting), 500):
994
+ chunk = waiting[start : start + 500]
995
+ try:
996
+ found = await self._results_exist([], [self.keys.prefix, *chunk])
997
+ except Exception as exc: # the sweep must survive store hiccups
998
+ logger.warning("result_sweep_failed", error=str(exc))
999
+ break
1000
+ for index in found or ():
1001
+ waiter = self._waiters.get(chunk[int(index)])
1002
+ if waiter is not None and not waiter.done():
1003
+ waiter.set_result(None)
1004
+
1005
+ async def _result_reader(self) -> None:
1006
+ """Wake waiting callers when their result is published (re-subscribes after errors)."""
1007
+ subscribed = False
1008
+ while True:
1009
+ try:
1010
+ if not subscribed:
1011
+ await self._pubsub.subscribe(self.keys.done_channel(self.worker_id))
1012
+ subscribed = True
1013
+ # Results published while unsubscribed were missed: every waiter re-reads.
1014
+ for pending in list(self._waiters.values()):
1015
+ if not pending.done():
1016
+ pending.set_result(None)
1017
+ message = await self._pubsub.get_message(
1018
+ ignore_subscribe_messages=True, timeout=1.0
1019
+ )
1020
+ except Exception as exc: # the reader must survive store hiccups
1021
+ logger.warning("result_reader_error", error=str(exc))
1022
+ subscribed = False
1023
+ with contextlib.suppress(Exception):
1024
+ await self._pubsub.reset()
1025
+ await asyncio.sleep(1)
1026
+ continue
1027
+ if not message:
1028
+ continue
1029
+ rid = (
1030
+ message["data"].decode()
1031
+ if isinstance(message["data"], bytes)
1032
+ else str(message["data"])
1033
+ )
1034
+ waiter = self._waiters.get(rid)
1035
+ if waiter is not None and not waiter.done():
1036
+ waiter.set_result(None)
1037
+
1038
+ # ================================================================ worker: receiving
1039
+
1040
+ async def _model_loop(self, alias: str) -> None:
1041
+ """Receive this model's calls (priority first) and admit them in order.
1042
+
1043
+ Idle, it blocks on its queues' doorbell (a send wakes one worker at once) for 50 ms at a
1044
+ time, then 1 s after a second with nothing: polling idle loops made ~96 receive
1045
+ scripts/s per pod with 4 models (measured); a busy loop never waits.
1046
+ """
1047
+ empty = 0
1048
+ wake = self._wake.setdefault(alias, asyncio.Event())
1049
+ while not self._stopping:
1050
+ if self._paused:
1051
+ await asyncio.sleep(0.2)
1052
+ continue
1053
+ try:
1054
+ got = await self._receive_once(alias)
1055
+ except Exception as exc: # a store blip must not end the loop
1056
+ logger.warning("receive_failed", model=alias, error=str(exc))
1057
+ if isinstance(exc, STORE_ERRORS):
1058
+ # Not only the heartbeat (every lease/3): a block between two beats went
1059
+ # unseen, and callers whose keys lapsed in it were dropped as gone (soak).
1060
+ self._store_trouble_at = time.monotonic()
1061
+ await asyncio.sleep(1)
1062
+ continue
1063
+ held = (alias, "p") in self._held_class or (alias, "n") in self._held_class
1064
+ # Idle only when the queues were empty, not when a class waits on its held call.
1065
+ empty = 0 if got or held else empty + 1
1066
+ if not got:
1067
+ pace = self.settings.timings
1068
+ pause = (
1069
+ pace.idle_poll_secs if empty < pace.idle_after_empty else pace.idle_slow_secs
1070
+ )
1071
+ if held: # a held call ending (granted or settled) wakes it at once
1072
+ with contextlib.suppress(TimeoutError):
1073
+ await asyncio.wait_for(wake.wait(), pause)
1074
+ else: # a send rings the doorbell: one idle worker wakes per call
1075
+ view = self.registry.current
1076
+ names = [self._queue_name(view, alias, p) for p in (True, False)]
1077
+ try:
1078
+ await self.queues.wait_bell(names, pause)
1079
+ except STORE_ERRORS:
1080
+ self._store_trouble_at = time.monotonic() # (was swallowed unseen)
1081
+ wake.clear()
1082
+
1083
+ async def _receive_once(self, alias: str) -> bool:
1084
+ view = self.registry.current
1085
+ if alias not in view.registry.models:
1086
+ await asyncio.sleep(1)
1087
+ return False
1088
+ slots = self._slots[alias]
1089
+ free = slots._value # free worker slots of this model, read only
1090
+ if free <= 0:
1091
+ await asyncio.sleep(0.02)
1092
+ return False
1093
+ settings = self._queue_settings(view, alias)
1094
+ got_any = False
1095
+ for priority in (True, False):
1096
+ cls = "p" if priority else "n"
1097
+ if (alias, cls) in self._held_class:
1098
+ continue
1099
+ name = self._queue_name(view, alias, priority)
1100
+ messages, dead = await self.queues.receive(
1101
+ name, min(free, self.settings.receive_batch), settings
1102
+ )
1103
+ for letter in dead:
1104
+ await self._dead_lettered(letter.body)
1105
+ if not messages:
1106
+ continue
1107
+ got_any = True
1108
+ for index, message in enumerate(messages):
1109
+ try:
1110
+ held = await self._admit_new(alias, message)
1111
+ except Exception:
1112
+ for rest in messages[index + 1 :]: # not seen: back to the queue now
1113
+ with contextlib.suppress(Exception):
1114
+ await self.queues.return_(rest)
1115
+ raise
1116
+ if held:
1117
+ for rest in messages[index + 1 :]:
1118
+ await self.queues.return_(rest)
1119
+ break
1120
+ if got_any:
1121
+ break
1122
+ return got_any
1123
+
1124
+ def _queue_settings(self, view: ResolvedRegistry, alias: str) -> QueueSettings:
1125
+ queue = view.value(alias, "queue")
1126
+ return QueueSettings(
1127
+ visibility_secs=float(queue["visibility_secs"]),
1128
+ max_receives=int(queue["max_receives"]),
1129
+ retention_secs=int(queue["retention_secs"]),
1130
+ dlq_retention_secs=int(queue["dlq_retention_secs"]),
1131
+ max_waiting=int(view.value(alias, "max_waiting")),
1132
+ )
1133
+
1134
+ async def _dead_lettered(self, body: bytes) -> None:
1135
+ try:
1136
+ env = parse(body)
1137
+ except Exception:
1138
+ return
1139
+ await self._reply(env, "dead_lettered", "the call failed too many times", final="failed")
1140
+
1141
+ async def _admit_new(self, alias: str, message: Message) -> bool:
1142
+ """Admit a newly received message; True when it is now held (its class stops)."""
1143
+ try:
1144
+ env = parse(message.body)
1145
+ except UnsupportedEnvelopeError:
1146
+ await self.queues.return_(message, delay_secs=5)
1147
+ return False
1148
+ except Exception as exc:
1149
+ logger.warning("bad_message", queue=message.queue, error=str(exc))
1150
+ await self.queues.fail(message, QueueSettings())
1151
+ return False
1152
+ held = _Held(
1153
+ message,
1154
+ env,
1155
+ first_seen=self.clock.now(),
1156
+ position_ms=message.position_ms,
1157
+ tag=message.id,
1158
+ )
1159
+ self._held[env.request_id] = held
1160
+ try:
1161
+ if not await self._caller_alive(env):
1162
+ # A warning: a dropped call is worth seeing (the soak and scenario S5 found
1163
+ # calls dropped wrongly, and INFO logs alone did not show it).
1164
+ logger.warning("caller_gone", request_id=env.request_id, model=env.model)
1165
+ await self._finish_call(held, "cancelled", "caller_gone", final="cancelled")
1166
+ return False
1167
+ return await self._try(held, attempt=1)
1168
+ except Exception:
1169
+ await self._release(held)
1170
+ raise
1171
+
1172
+ async def _caller_alive(self, env: Envelope) -> bool:
1173
+ """False when every caller of this call is gone (plan §3.18 R24).
1174
+
1175
+ After a store outage the callers' liveness keys may have expired only because nobody
1176
+ could renew them: for a caller TTL and a heartbeat after this process last saw the
1177
+ store fail, callers count as alive (B95: a 45 s block dropped 429 of 500 calls).
1178
+
1179
+ Judged from this process's own heartbeat too: a half-open store hung every command for
1180
+ the read timeout, the doorbell wait swallowed its error, and calls were dropped the
1181
+ moment the store came back (scenario S5). A heartbeat that is late or just recovered
1182
+ from a gap means the store was out of reach here as well, so a lapsed key proves
1183
+ nothing.
1184
+ """
1185
+ beat = self.settings.lease_secs / 3
1186
+ grace = self.settings.caller_ttl_secs + beat + 5
1187
+ now = time.monotonic()
1188
+ if now - self._store_trouble_at < grace or now - self._beat_ok_at > 2 * beat + 1:
1189
+ return True
1190
+ try:
1191
+ return await self.requests.callers_alive(
1192
+ env.request_id, prefix=self.keys.prefix, caller_ttl=self.settings.caller_ttl_secs
1193
+ )
1194
+ except STORE_ERRORS:
1195
+ self._store_trouble_at = time.monotonic()
1196
+ return True
1197
+
1198
+ # ================================================================ worker: admission
1199
+
1200
+ def _deadline(self, held: _Held) -> float:
1201
+ return held.position_ms / 1000.0 + held.envelope.timeout_secs
1202
+
1203
+ def _candidates(
1204
+ self,
1205
+ view: ResolvedRegistry,
1206
+ alias: str,
1207
+ env: Envelope,
1208
+ age: float,
1209
+ failed: frozenset[str],
1210
+ cause: str,
1211
+ ) -> list[Candidate]:
1212
+ deps: list[Deployment] = list(view.deployments(alias))
1213
+ if env.allowed_regions:
1214
+ deps = [d for d in deps if d.region in env.allowed_regions]
1215
+ if env.allowed_accounts:
1216
+ deps = [d for d in deps if d.account in env.allowed_accounts]
1217
+ fallback_ok = self._may_spill(view, alias, env, age, cause, "fallback")
1218
+ now = time.monotonic()
1219
+ deps = [
1220
+ d for d in deps if self._bad_accounts.get((d.account, d.region), (0.0, 0.0))[0] <= now
1221
+ ]
1222
+ out = []
1223
+ for dep in deps:
1224
+ if dep.group == "fallback" and not fallback_ok:
1225
+ continue
1226
+ out.append(Candidate(dep.key, dep.group, last=dep.key in failed)) # type: ignore[arg-type]
1227
+ return out
1228
+
1229
+ def _ctx(self, alias: str, held: _Held, waited: float) -> hooks.CallContext:
1230
+ env = held.envelope
1231
+ return hooks.CallContext(
1232
+ alias, env.request_id, env.priority, env.session_id, env.user_id, env.tenant_id, waited
1233
+ )
1234
+
1235
+ def _may_spill(
1236
+ self,
1237
+ view: ResolvedRegistry,
1238
+ alias: str,
1239
+ env: Envelope,
1240
+ age: float,
1241
+ cause: str,
1242
+ rule_name: str,
1243
+ ) -> bool:
1244
+ """Whether a call may use fallback regions / models now: rule or `fallback_policy`."""
1245
+ allowed = ("all", "regions") if rule_name == "fallback" else ("all",)
1246
+ rule = view.value(alias, rule_name)
1247
+ if env.allow_fallback not in allowed or not rule:
1248
+ return False
1249
+ policy_name = view.value(alias, "fallback_policy")
1250
+ policy = hooks.get("fallback_policy", policy_name)
1251
+ if policy is not None:
1252
+ ctx = hooks.CallContext(
1253
+ alias, env.request_id, env.priority, env.session_id, env.user_id, env.tenant_id, age
1254
+ )
1255
+ default = cause in rule["on"] and age >= float(rule["after_secs"])
1256
+ return bool(
1257
+ hooks.call_safely(
1258
+ "fallback_policy", policy_name, policy, cause, age, rule, ctx, default=default
1259
+ )
1260
+ )
1261
+ return cause in rule["on"] and age >= float(rule["after_secs"])
1262
+
1263
+ def _ask(
1264
+ self,
1265
+ view: ResolvedRegistry,
1266
+ alias: str,
1267
+ held: _Held,
1268
+ attempt: int,
1269
+ failed: frozenset[str],
1270
+ *,
1271
+ foreign_from: str | None = None,
1272
+ require_pct: int = 0,
1273
+ ) -> Ask | None:
1274
+ env = held.envelope
1275
+ age = self.clock.now() - held.position_ms / 1000.0
1276
+ cause = "error" if failed else ("broken" if held.broken else "exhausted")
1277
+ candidates = self._candidates(view, alias, env, age, failed, cause)
1278
+ if not candidates:
1279
+ return None
1280
+ entry = view.registry.models[alias]
1281
+ estimate = self._charged(view, alias, held)
1282
+ strategy = view.value(alias, "balance")
1283
+ if env.priority and view.value(alias, "balance_priority"):
1284
+ strategy = view.value(alias, "balance_priority")
1285
+ if str(strategy).startswith("custom:"):
1286
+ candidates = self._custom_order(view, alias, held, candidates, str(strategy)[7:], age)
1287
+ strategy = "ordered"
1288
+ sticky = view.value(alias, "sticky")
1289
+ sticky_hash = None
1290
+ if sticky and entry.type == "chat":
1291
+ key: str | None = env.session_id
1292
+ if str(sticky["key"]).startswith("custom:"):
1293
+ name = str(sticky["key"])[7:]
1294
+ fn = hooks.get("sticky_key", name)
1295
+ ctx = self._ctx(alias, held, age)
1296
+ key = hooks.call_safely("sticky_key", name, fn, ctx, default=None) if fn else None
1297
+ if key:
1298
+ sticky_hash = self.hash_id(str(key))
1299
+ return Ask(
1300
+ alias=alias,
1301
+ request_id=env.request_id,
1302
+ worker=self.worker_id,
1303
+ attempt=attempt,
1304
+ cls="p" if env.priority else "n",
1305
+ position=held.position,
1306
+ candidates=candidates,
1307
+ strategy=str(strategy),
1308
+ pick=str((view.value(alias, "balance_opts") or {}).get("pick", "percent")),
1309
+ cost_r=1,
1310
+ cost_t=estimate.total,
1311
+ claim=foreign_from is None,
1312
+ sticky_hash=sticky_hash,
1313
+ sticky_ttl_secs=float(sticky["idle_ttl_secs"]) if sticky else 1800.0,
1314
+ sticky_move_secs=float(view.value(alias, "sticky_move_after_secs")),
1315
+ foreign_from=foreign_from,
1316
+ require_pct=require_pct,
1317
+ tag=held.tag,
1318
+ )
1319
+
1320
+ def _custom_order(
1321
+ self,
1322
+ view: ResolvedRegistry,
1323
+ alias: str,
1324
+ held: _Held,
1325
+ candidates: list[Candidate],
1326
+ name: str,
1327
+ age: float,
1328
+ ) -> list[Candidate]:
1329
+ """Candidates in the order a custom balancer prefers (unknown keys ignored)."""
1330
+ fn = hooks.get("balancer", name)
1331
+ if fn is None:
1332
+ return candidates
1333
+ deps = {d.key: d for d in view.deployments(alias)}
1334
+ views = [
1335
+ hooks.DeploymentView(
1336
+ key=c.key,
1337
+ account=deps[c.key].account,
1338
+ region=deps[c.key].region,
1339
+ group=c.tier,
1340
+ rpm=view.value(alias, "rpm", c.key),
1341
+ tpm=view.value(alias, "tpm", c.key),
1342
+ weight=float(view.value(alias, "weight", c.key)),
1343
+ tags=view.value(alias, "tags", c.key) or {},
1344
+ )
1345
+ for c in candidates
1346
+ ]
1347
+ ctx = self._ctx(alias, held, age)
1348
+ order = hooks.call_safely("balancer", name, fn, views, ctx, default=None)
1349
+ if not order:
1350
+ return candidates
1351
+ by_key = {c.key: c for c in candidates}
1352
+ ranked = [by_key[k] for k in order if k in by_key]
1353
+ return ranked + [c for c in candidates if c.key not in set(order)]
1354
+
1355
+ def _owner(self, held: _Held, attempt: int) -> str:
1356
+ """Owner token of an attempt; the message id tells two deliveries of one id apart."""
1357
+ worker = f"{self.worker_id}@{held.tag}"
1358
+ return owner_token(held.envelope.request_id, worker, attempt)
1359
+
1360
+ def _params(
1361
+ self, view: ResolvedRegistry, alias: str, env: Envelope, dep: str | None
1362
+ ) -> dict[str, Any]:
1363
+ entry = view.registry.models[alias]
1364
+ request = inference.apply(
1365
+ entry.provider,
1366
+ env.op,
1367
+ env.params or {},
1368
+ view.value(alias, "inference", dep) or {},
1369
+ view.value(alias, "inference_limits", dep) or {},
1370
+ )
1371
+ extra = view.value(alias, "params", dep) if dep else None
1372
+ return inference.merge_params(request, extra or {})
1373
+
1374
+ def _charged(self, view: ResolvedRegistry, alias: str, held: _Held) -> tokens.Estimate:
1375
+ """The token estimate admission charges: from model-level params, cached per view."""
1376
+ cached = held.charged
1377
+ if cached is not None and cached[0] is view and cached[1] == alias:
1378
+ return cached[2]
1379
+ params = self._params(view, alias, held.envelope, None)
1380
+ estimate = self._estimate(view, alias, held.envelope, params)
1381
+ held.charged = (view, alias, estimate)
1382
+ return estimate
1383
+
1384
+ def _estimate(
1385
+ self, view: ResolvedRegistry, alias: str, env: Envelope, params: Mapping[str, Any]
1386
+ ) -> tokens.Estimate:
1387
+ entry = view.registry.models[alias]
1388
+ return tokens.estimate(
1389
+ env.op,
1390
+ params,
1391
+ embedding=entry.type == "embedding",
1392
+ default_output_tokens=int(view.value(alias, "default_output_tokens")),
1393
+ output_weight=float(view.value(alias, "tpm_output_weight")),
1394
+ estimator=view.value(alias, "token_estimator"),
1395
+ )
1396
+
1397
+ async def _try(self, held: _Held, attempt: int, failed: frozenset[str] = frozenset()) -> bool:
1398
+ """Admit (or re-admit) a held call; start it when granted. True when it stays held."""
1399
+ view = self.registry.current
1400
+ env = held.envelope
1401
+ alias = env.model
1402
+ now = self.clock.now()
1403
+ deadline = self._deadline(held)
1404
+ if now >= deadline:
1405
+ await self._finish_call(
1406
+ held, "rate_limited", "no budget before the deadline", final="expired"
1407
+ )
1408
+ return False
1409
+ max_wait = env.max_wait_secs
1410
+ if max_wait is not None and attempt == 1 and now - held.position_ms / 1000.0 >= max_wait:
1411
+ # It already waited its limit in the queue, unreceived: refuse it now rather than
1412
+ # start it with no time left (it ran past its deadline and the caller gave up).
1413
+ await self._finish_call(
1414
+ held, "rate_limited", "max_wait_secs passed", final="expired", retry_after=1.0
1415
+ )
1416
+ return False
1417
+ decision = await self._decide(view, alias, held, attempt, failed)
1418
+ if decision is None:
1419
+ return False
1420
+ if decision.kind == "granted":
1421
+ self._start_run(held, decision, attempt)
1422
+ return False
1423
+ if decision.kind == "wait":
1424
+ max_wait = env.max_wait_secs
1425
+ waited = now - held.position_ms / 1000.0
1426
+ if max_wait is not None and attempt == 1 and waited >= max_wait:
1427
+ await self._finish_call(
1428
+ held,
1429
+ "rate_limited",
1430
+ "max_wait_secs passed",
1431
+ final="expired",
1432
+ retry_after=decision.wait_secs,
1433
+ )
1434
+ return False
1435
+ fallen = await self._model_fallback(view, alias, held, attempt, waited)
1436
+ if fallen:
1437
+ return False
1438
+ if attempt == 1:
1439
+ cls = "p" if env.priority else "n"
1440
+ self._held_class.add((alias, cls))
1441
+ task = asyncio.create_task(self._hold(held, decision.wait_secs))
1442
+ self._running.add(task)
1443
+ task.add_done_callback(self._running.discard)
1444
+ return True
1445
+ await asyncio.sleep(min(decision.wait_secs, 1.0))
1446
+ return await self._try(held, attempt, failed)
1447
+ return False
1448
+
1449
+ async def _decide(
1450
+ self,
1451
+ view: ResolvedRegistry,
1452
+ alias: str,
1453
+ held: _Held,
1454
+ attempt: int,
1455
+ failed: frozenset[str],
1456
+ ) -> Decision | None:
1457
+ """One admission call and the terminal outcomes; None when the call was settled."""
1458
+ ask = self._ask(view, alias, held, attempt, failed)
1459
+ if ask is None:
1460
+ if self._bad_accounts and any(
1461
+ (d.account, d.region) in self._bad_accounts for d in view.deployments(alias)
1462
+ ):
1463
+ await self._finish_call(
1464
+ held,
1465
+ "credentials_unavailable",
1466
+ "no account with working credentials",
1467
+ retry_after=60,
1468
+ )
1469
+ else:
1470
+ await self._finish_call(
1471
+ held, "no_deployment", "the call's filter leaves no deployment"
1472
+ )
1473
+ return None
1474
+ estimate = self._charged(view, alias, held)
1475
+ tpm = [view.value(alias, "tpm", c.key) for c in ask.candidates]
1476
+ if all(isinstance(v, int) for v in tpm) and tpm and estimate.input > max(tpm):
1477
+ await self._finish_call(held, "request_too_large", "input above the token budget")
1478
+ return None
1479
+ decision = await self.admission.pick_take(ask, self.registry.changes)
1480
+ if decision.kind == "wait":
1481
+ self.metrics.count("llmq_admission_refusals_total", model=alias, reason=decision.reason)
1482
+ if decision.kind == "stale_config":
1483
+ await self.registry.load()
1484
+ decision = await self.admission.pick_take(ask, self.registry.changes)
1485
+ if decision.kind == "no_config": # the store lost the numbers: restore, then once more
1486
+ await self.registry.refresh()
1487
+ decision = await self.admission.pick_take(ask, self.registry.changes)
1488
+ if decision.kind in ("no_config", "stale_config"):
1489
+ # Still no usable numbers. Before, the call stayed held and hidden for good.
1490
+ logger.error("admission_config_missing", model=alias, kind=decision.kind)
1491
+ if held.message is None:
1492
+ await self._release(held)
1493
+ raise CallFailed("queue_unavailable", f"{alias}: admission numbers are missing")
1494
+ await self._release(held, delay_secs=5.0)
1495
+ return None
1496
+ if decision.kind in ("done", "cancelled"):
1497
+ if decision.kind == "cancelled":
1498
+ await self._finish_call(held, "cancelled", "cancelled", final="cancelled")
1499
+ else:
1500
+ await self._ack(held)
1501
+ return None
1502
+ if decision.kind == "duplicate":
1503
+ self._drop(held)
1504
+ if held.message is not None:
1505
+ await self.queues.return_(held.message, delay_secs=self.settings.lease_secs / 2)
1506
+ return None
1507
+ if decision.kind == "none":
1508
+ has_fallback = any(d.group == "fallback" for d in view.deployments(alias))
1509
+ rule = view.value(alias, "fallback")
1510
+ if not held.broken and has_fallback and rule and "broken" in rule["on"]:
1511
+ held.broken = True
1512
+ age = self.clock.now() - held.position_ms / 1000.0
1513
+ if age < float(rule["after_secs"]):
1514
+ return Decision(
1515
+ "wait", wait_secs=float(rule["after_secs"]) - age, reason="broken"
1516
+ )
1517
+ return await self._decide(view, alias, held, attempt, failed)
1518
+ fell = await self._model_fallback(view, alias, held, attempt, 1e9, broken=True)
1519
+ if not fell:
1520
+ due = await self.health.disabled(alias)
1521
+ wait = max(1.0, min(due.values()) - self.clock.now()) if due else 60.0
1522
+ # Why no deployment was usable, per deployment: an operator needs to tell an
1523
+ # open breaker from a disabled deployment (the bare line said neither).
1524
+ why: dict[str, str] = {}
1525
+ for dep in view.deployments(alias):
1526
+ with contextlib.suppress(*STORE_ERRORS):
1527
+ cb = await self.state.hgetall(self.keys.model(alias, f"cb:{dep.key}"))
1528
+ raw = (cb or {}).get(b"state", b"closed")
1529
+ state = raw.decode() if isinstance(raw, bytes) else str(raw)
1530
+ disabled = " (disabled)" if dep.key in due else ""
1531
+ why[dep.key] = f"breaker {state}{disabled}"
1532
+ logger.error("model_unavailable", model=alias, deployments=why)
1533
+ await self._finish_call(
1534
+ held, "model_unavailable", "no usable deployment", retry_after=wait
1535
+ )
1536
+ return None
1537
+ return decision
1538
+
1539
+ async def _hold(self, held: _Held, wait: float) -> None:
1540
+ """Keep a refused call: retry admission until granted, settled or expired."""
1541
+ alias = held.envelope.model
1542
+ cls = "p" if held.envelope.priority else "n"
1543
+ try:
1544
+ while not self._stopping:
1545
+ await asyncio.sleep(min(max(wait, 0.01), 1.0))
1546
+ view = self.registry.current
1547
+ if held.message is not None:
1548
+ await self.queues.extend(
1549
+ held.message, float(view.value(alias, "queue")["visibility_secs"])
1550
+ )
1551
+ now = self.clock.now()
1552
+ if now >= self._deadline(held):
1553
+ await self._finish_call(
1554
+ held, "rate_limited", "no budget before the deadline", final="expired"
1555
+ )
1556
+ return
1557
+ waited = now - held.position_ms / 1000.0
1558
+ if (
1559
+ held.envelope.max_wait_secs is not None
1560
+ and waited >= held.envelope.max_wait_secs
1561
+ ):
1562
+ await self._finish_call(
1563
+ held,
1564
+ "rate_limited",
1565
+ "max_wait_secs passed",
1566
+ final="expired",
1567
+ retry_after=wait,
1568
+ )
1569
+ return
1570
+ if await self._model_fallback(view, alias, held, 1, waited):
1571
+ return
1572
+ decision = await self._decide(view, alias, held, 1, frozenset())
1573
+ if decision is None:
1574
+ return
1575
+ if decision.kind == "granted":
1576
+ self._start_run(held, decision, 1)
1577
+ return
1578
+ wait = decision.wait_secs
1579
+ except Exception as exc: # never leave the call held with nobody working on it
1580
+ if isinstance(exc, STORE_ERRORS):
1581
+ self._store_trouble_at = time.monotonic()
1582
+ logger.warning("hold_failed", request_id=held.envelope.request_id, error=str(exc))
1583
+ await self._release(held)
1584
+ finally:
1585
+ self._held_class.discard((alias, cls))
1586
+ if (event := self._wake.get(alias)) is not None:
1587
+ event.set() # the class is free: receive its next call now
1588
+
1589
+ async def _model_fallback(
1590
+ self,
1591
+ view: ResolvedRegistry,
1592
+ alias: str,
1593
+ held: _Held,
1594
+ attempt: int,
1595
+ waited: float,
1596
+ *,
1597
+ broken: bool = False,
1598
+ ) -> bool:
1599
+ """Try the model's fallback models when its rule allows; True when one granted."""
1600
+ rule = view.value(alias, "model_fallback")
1601
+ env = held.envelope
1602
+ cause = "broken" if broken else "exhausted"
1603
+ if not self._may_spill(view, alias, env, waited, cause, "model_fallback"):
1604
+ return False
1605
+ for item in rule["models"]:
1606
+ other = item["model"]
1607
+ if other not in view.registry.models:
1608
+ continue
1609
+ pct = int(view.value(other, "require_headroom_pct")) if item["require_headroom"] else 0
1610
+ ask = self._ask(
1611
+ view, other, held, attempt, frozenset(), foreign_from=alias, require_pct=pct
1612
+ )
1613
+ if ask is None:
1614
+ continue
1615
+ decision = await self.admission.pick_take(ask, self.registry.changes)
1616
+ if decision.kind == "granted":
1617
+ await self.requests.set_status(env.request_id, "in_progress", served_model=other)
1618
+ self._start_run(held, decision, attempt, served=other)
1619
+ return True
1620
+ return False
1621
+
1622
+ # ================================================================ worker: running
1623
+
1624
+ def _start_run(
1625
+ self, held: _Held, decision: Decision, attempt: int, served: str | None = None
1626
+ ) -> None:
1627
+ alias = served or held.envelope.model
1628
+ held.grants.add((alias, self._owner(held, attempt)))
1629
+ task = asyncio.create_task(self._run(held, decision, attempt, alias))
1630
+ self._running.add(task)
1631
+ task.add_done_callback(self._running.discard)
1632
+
1633
+ async def _run(self, held: _Held, decision: Decision, attempt: int, alias: str) -> None:
1634
+ slots = self._slots.get(held.envelope.model)
1635
+ held.run = asyncio.current_task()
1636
+ try:
1637
+ if slots is None:
1638
+ await self._attempts(held, decision, attempt, alias)
1639
+ else:
1640
+ async with slots:
1641
+ await self._attempts(held, decision, attempt, alias)
1642
+ except asyncio.CancelledError:
1643
+ if not held.cancelled:
1644
+ raise
1645
+ # Cancelled by a caller: give the grant back as a cut attempt, reply `cancelled`.
1646
+ try:
1647
+ await self._give_back(held)
1648
+ await self._finish_call(held, "cancelled", "cancelled", final="cancelled")
1649
+ except Exception as exc:
1650
+ logger.warning("cancel_failed", request_id=held.envelope.request_id, error=str(exc))
1651
+ await self._release(held)
1652
+ except Exception as exc:
1653
+ # A store outage, a store refusal (OOM) or a bug mid-call. Never leave the call held
1654
+ # with nobody working on it (the heartbeat would hide its message for good): give
1655
+ # the grants back and return the message, so it runs again (at least once; a stored
1656
+ # result is reused). A direct call has no message: its caller gets the error.
1657
+ if isinstance(exc, STORE_ERRORS):
1658
+ self._store_trouble_at = time.monotonic()
1659
+ logger.warning(
1660
+ "run_failed",
1661
+ request_id=held.envelope.request_id,
1662
+ error=str(exc),
1663
+ error_type=type(exc).__name__,
1664
+ )
1665
+ await self._release(held)
1666
+ if held.message is None:
1667
+ raise
1668
+
1669
+ async def _attempts(self, held: _Held, decision: Decision, attempt: int, alias: str) -> None:
1670
+ throttles = 0
1671
+ tries = 0
1672
+ failed: set[str] = set()
1673
+ retry_secs = 0.0
1674
+ while True:
1675
+ view = self.registry.current
1676
+ dep_key = decision.deployment or ""
1677
+ owner = self._owner(held, attempt)
1678
+ held.grants.add((alias, owner))
1679
+ outcome, reply_or_error = await self._attempt(
1680
+ view, alias, held, dep_key, owner, attempt
1681
+ )
1682
+ held.grants.discard((alias, owner))
1683
+ if outcome == "ok":
1684
+ return
1685
+ kind, info, exc = reply_or_error
1686
+ if outcome == "stream_failed": # never switched silently once chunks were sent
1687
+ await self._finish_call(held, kind, str(exc), partial=info)
1688
+ return
1689
+ entry = view.registry.models[alias]
1690
+ if outcome == "throttle":
1691
+ retry_secs = 0.0 # a throttle moves on at once (a 5xx's backoff is not reused)
1692
+ throttles += 1
1693
+ limit = int(view.value(alias, "throttle_retry_attempts", dep_key))
1694
+ if throttles > limit:
1695
+ await self._finish_call(
1696
+ held, "rate_limited", "the provider kept throttling", retry_after=info
1697
+ )
1698
+ return
1699
+ else:
1700
+ tries += 1
1701
+ if outcome == "fatal" or tries >= int(view.value(alias, "max_attempts")):
1702
+ await self._finish_call(held, "provider_error", str(exc))
1703
+ return
1704
+ # A move to a deployment this call has not tried is free: it is failover, not
1705
+ # a retry storm (bounded by the deployments and max_attempts). Counting it
1706
+ # failed calls as retry_budget_exhausted while another region was healthy
1707
+ # (scenario P3: 20 of 60 calls with one region hanging).
1708
+ tried = frozenset(failed | {dep_key})
1709
+ age = self.clock.now() - held.position_ms / 1000.0
1710
+ untried = any(
1711
+ not c.last
1712
+ for c in self._candidates(view, alias, held.envelope, age, tried, "error")
1713
+ )
1714
+ if not untried and not await self._retry_allowed(alias, view):
1715
+ await self._finish_call(held, "retry_budget_exhausted", str(exc))
1716
+ return
1717
+ retry_secs = self._backoff(view, alias, dep_key, tries)
1718
+ failed.add(dep_key)
1719
+ attempt += 1
1720
+ if retry_secs: # never past the deadline: the caller is told in time
1721
+ left = self._deadline(held) - self.clock.now()
1722
+ await asyncio.sleep(max(0.0, min(retry_secs, left)))
1723
+ while True:
1724
+ if self.clock.now() >= self._deadline(held):
1725
+ await self._finish_call(held, "deadline_exceeded", f"last error: {kind}")
1726
+ return
1727
+ next_decision = await self._decide(view, alias, held, attempt, frozenset(failed))
1728
+ if next_decision is None:
1729
+ return
1730
+ if next_decision.kind == "granted":
1731
+ decision = next_decision
1732
+ break
1733
+ if next_decision.kind == "busy":
1734
+ await self._finish_call(held, "rate_limited", "fallback model is busy")
1735
+ return
1736
+ await asyncio.sleep(min(next_decision.wait_secs, 1.0))
1737
+ del entry
1738
+
1739
+ def _backoff(self, view: ResolvedRegistry, alias: str, dep: str, tries: int) -> float:
1740
+ base = float(view.value(alias, "retry_base_secs", dep))
1741
+ factor = float(view.value(alias, "retry_multiplier", dep))
1742
+ spread = float(view.value(alias, "retry_spread", dep))
1743
+ low = base * factor ** (tries - 1)
1744
+ return min(
1745
+ low * (1 + (spread - 1) * self._rng.random()),
1746
+ float(view.value(alias, "retry_max_secs", dep)),
1747
+ )
1748
+
1749
+ async def _retry_allowed(self, alias: str, view: ResolvedRegistry) -> bool:
1750
+ minute = int(self.clock.now() // 60)
1751
+ key = self.keys.model(alias, f"retries:{minute}")
1752
+ async with self.state.pipeline(transaction=False) as pipe:
1753
+ pipe.hincrby(key, "retries", 1)
1754
+ pipe.hget(key, "first")
1755
+ pipe.expire(key, 120)
1756
+ retries, first, _ = await pipe.execute()
1757
+ first_n = int(first or 0)
1758
+ allowed = max(
1759
+ int(view.value(alias, "retry_budget_min")),
1760
+ first_n * int(view.value(alias, "retry_budget_pct")) // 100,
1761
+ )
1762
+ return int(retries) <= allowed
1763
+
1764
+ async def _attempt(
1765
+ self,
1766
+ view: ResolvedRegistry,
1767
+ alias: str,
1768
+ held: _Held,
1769
+ dep_key: str,
1770
+ owner: str,
1771
+ attempt: int,
1772
+ ) -> tuple[str, Any]:
1773
+ """One provider attempt and its settlement: ('ok', None) or (outcome, (kind, info, exc))."""
1774
+ env = held.envelope
1775
+ entry = view.registry.models[alias]
1776
+ deployment = next(d for d in view.deployments(alias) if d.key == dep_key)
1777
+ route = route_for(view, deployment)
1778
+ params = self._params(view, alias, env, dep_key)
1779
+ # Refunds settle what admission charged: `_ask`'s estimate from model-level params. One
1780
+ # from this deployment's params (e.g. a larger account max_tokens) refunded more than
1781
+ # was taken, and TPM stopped holding (found in the 0.3.0 review).
1782
+ estimate = self._charged(view, alias, held)
1783
+ remaining = max(0.1, self._deadline(held) - self.clock.now())
1784
+ attempt_budget = float(view.value(alias, "attempt_timeout_secs", dep_key))
1785
+ timing = Timing(
1786
+ first_response_secs=min(
1787
+ attempt_budget,
1788
+ remaining / 2 if attempt == 1 else remaining,
1789
+ ),
1790
+ idle_secs=float(view.value(alias, "stream_idle_secs")),
1791
+ )
1792
+ provider = self.providers.get(entry.provider)
1793
+ held.meta[owner] = {
1794
+ "alias": alias,
1795
+ "owner": owner,
1796
+ "request_id": env.request_id,
1797
+ "dep": dep_key,
1798
+ "cls": "p" if env.priority else "n",
1799
+ "cost_r": 1,
1800
+ "cost_t": estimate.total,
1801
+ }
1802
+ if attempt == 1:
1803
+ minute = int(self.clock.now() // 60)
1804
+ await self.state.hincrby(self.keys.model(env.model, f"retries:{minute}"), "first", 1)
1805
+ writer = None
1806
+ if env.stream and env.op in ("converse", "chat"):
1807
+ writer = ChunkWriter(
1808
+ self.state,
1809
+ self.keys.request(env.request_id, "chunks"),
1810
+ owner,
1811
+ flush_ms=int(view.value(alias, "stream_flush_ms", dep_key)),
1812
+ max_bytes=int(view.value(alias, "stream_max_bytes")),
1813
+ ttl_secs=self.settings.result_ttl_secs + int(env.timeout_secs),
1814
+ )
1815
+ if await self.state.xlen(writer.key):
1816
+ # Another worker already streamed part of this answer (plan R19).
1817
+ await self.admission.finish(
1818
+ alias,
1819
+ owner,
1820
+ env.request_id,
1821
+ refund_r=1,
1822
+ refund_t=estimate.total,
1823
+ outcome="none",
1824
+ latency_ms=-1,
1825
+ )
1826
+ lost = "the worker serving this stream was lost"
1827
+ await writer.error("worker_lost", lost)
1828
+ return "stream_failed", ("worker_lost", None, RuntimeError(lost))
1829
+ started = time.monotonic()
1830
+ try:
1831
+ result = await asyncio.wait_for(
1832
+ provider.call(env.op, params, route, timing, sink=writer.add if writer else None),
1833
+ remaining,
1834
+ )
1835
+ if writer is not None:
1836
+ await writer.end(
1837
+ {
1838
+ "inputTokens": result.usage.input_tokens,
1839
+ "outputTokens": result.usage.output_tokens,
1840
+ }
1841
+ )
1842
+ except Exception as exc:
1843
+ latency = (time.monotonic() - started) * 1000
1844
+ info = provider.classify(exc)
1845
+ if writer is not None and writer.sent:
1846
+ # Chunks reached the caller: end the stream with the partial text (plan §3.13).
1847
+ refund = tokens.refund(estimate, "cut")
1848
+ await self.admission.finish(
1849
+ alias,
1850
+ owner,
1851
+ env.request_id,
1852
+ refund_r=refund.requests,
1853
+ refund_t=refund.tokens,
1854
+ outcome="fail",
1855
+ latency_ms=latency,
1856
+ )
1857
+ kind = "rate_limited" if info.kind is ErrorKind.THROTTLE else "provider_error"
1858
+ message = (
1859
+ "stream above stream_max_bytes" if isinstance(exc, StreamTooLarge) else str(exc)
1860
+ )
1861
+ with contextlib.suppress(Exception):
1862
+ await writer.error(kind, message)
1863
+ return "stream_failed", (kind, writer.partial_text, exc)
1864
+ cut = isinstance(exc, AttemptTimeout) and "chunk" in str(exc)
1865
+ # The caller's own deadline cut this attempt (the outer timer, or a first-response
1866
+ # timer shortened below `attempt_timeout_secs`): not the provider's fault. Counted
1867
+ # as a failure, late grants under a rate limit opened the breaker and turned the
1868
+ # rest into model_unavailable (found by scripts/load.py).
1869
+ deadline_cut = isinstance(exc, TimeoutError) and (
1870
+ not isinstance(exc, AttemptTimeout)
1871
+ or ("first response" in str(exc) and timing.first_response_secs < attempt_budget)
1872
+ )
1873
+ if deadline_cut:
1874
+ await self.admission.finish(
1875
+ alias,
1876
+ owner,
1877
+ env.request_id,
1878
+ refund_r=0,
1879
+ refund_t=0,
1880
+ outcome="none",
1881
+ latency_ms=-1,
1882
+ )
1883
+ return "retry", ("retryable", None, exc)
1884
+ if info.kind is ErrorKind.THROTTLE:
1885
+ refund = tokens.refund(estimate, "throttle")
1886
+ await self.admission.finish(
1887
+ alias,
1888
+ owner,
1889
+ env.request_id,
1890
+ refund_r=refund.requests,
1891
+ refund_t=refund.tokens,
1892
+ outcome="throttle",
1893
+ latency_ms=-1,
1894
+ )
1895
+ return "throttle", ("throttled", info.retry_after_secs, exc)
1896
+ if info.kind is ErrorKind.RETRYABLE or isinstance(exc, TimeoutError):
1897
+ refund = tokens.refund(estimate, "cut" if cut else "fail")
1898
+ await self.admission.finish(
1899
+ alias,
1900
+ owner,
1901
+ env.request_id,
1902
+ refund_r=refund.requests,
1903
+ refund_t=refund.tokens,
1904
+ outcome="fail",
1905
+ latency_ms=latency,
1906
+ )
1907
+ return "retry", ("retryable", None, exc)
1908
+ refund = tokens.refund(estimate, "throttle") # refused before it ran: all back
1909
+ await self.admission.finish(
1910
+ alias,
1911
+ owner,
1912
+ env.request_id,
1913
+ refund_r=refund.requests,
1914
+ refund_t=refund.tokens,
1915
+ outcome="none",
1916
+ latency_ms=-1,
1917
+ )
1918
+ if is_target_error(exc, info):
1919
+ return await self._target_error(view, alias, held, dep_key, exc)
1920
+ if isinstance(exc, CredentialsUnavailable) or _auth_error(info.code):
1921
+ self.providers.pool.forget(route)
1922
+ where = (deployment.account, deployment.region)
1923
+ _, backoff = self._bad_accounts.get(where, (0.0, 2.5))
1924
+ backoff = min(backoff * 2, 60.0)
1925
+ self._bad_accounts[where] = (time.monotonic() + backoff, backoff)
1926
+ logger.error(
1927
+ "account_credentials_failed",
1928
+ account=deployment.account,
1929
+ region=deployment.region,
1930
+ )
1931
+ return "retry", ("auth", None, exc)
1932
+ return "fatal", ("fatal", None, exc)
1933
+ latency = (time.monotonic() - started) * 1000
1934
+ used = tokens.used_tokens(
1935
+ result.usage,
1936
+ output_weight=float(view.value(alias, "tpm_output_weight", dep_key)),
1937
+ count_cached=bool(view.value(alias, "tpm_count_cached", dep_key)),
1938
+ )
1939
+ refund = tokens.refund(estimate, "ok", used)
1940
+ await self.admission.finish(
1941
+ alias,
1942
+ owner,
1943
+ env.request_id,
1944
+ refund_r=refund.requests,
1945
+ refund_t=refund.tokens,
1946
+ outcome="ok",
1947
+ latency_ms=latency,
1948
+ )
1949
+ meta = Meta(
1950
+ served_model=alias,
1951
+ deployment=dep_key,
1952
+ fallback="model"
1953
+ if alias != env.model
1954
+ else ("region" if deployment.group == "fallback" else None),
1955
+ attempts=attempt,
1956
+ queue_wait_ms=int(max(0.0, held.first_seen - held.position_ms / 1000.0) * 1000),
1957
+ provider_ms=int(latency),
1958
+ tokens_in=result.usage.input_tokens,
1959
+ tokens_out=result.usage.output_tokens,
1960
+ worker=self.worker_id,
1961
+ )
1962
+ reply = Reply(request_id=env.request_id, ok=True, response=result.response, meta=meta)
1963
+ await self.requests.put_result(reply, final_state="done")
1964
+ await self._count_target_errors(held, succeeded=True)
1965
+ self._record(held, alias, deployment.account, deployment.region, "ok", meta)
1966
+ if self.settings.debug_capture_allowed and view.value(alias, "debug_capture"):
1967
+ await self._debug(env.request_id, params, result.response)
1968
+ await self.requests.set_status(
1969
+ env.request_id,
1970
+ "done",
1971
+ deployment=dep_key,
1972
+ served_model=alias,
1973
+ attempts=attempt,
1974
+ tokens_in=result.usage.input_tokens,
1975
+ tokens_out=result.usage.output_tokens,
1976
+ )
1977
+ await self._ack(held)
1978
+ return "ok", None
1979
+
1980
+ async def _target_error(
1981
+ self, view: ResolvedRegistry, alias: str, held: _Held, dep_key: str, exc: BaseException
1982
+ ) -> tuple[str, Any]:
1983
+ """Note a target error; a request failing this way on two deployments is the cause.
1984
+
1985
+ Nothing is counted yet: `_count_target_errors` records them once the call's outcome
1986
+ shows whether the deployment or the request was at fault (plan §3.18 R16).
1987
+ """
1988
+ held.target_failed.append((alias, dep_key))
1989
+ if len({d for _, d in held.target_failed}) >= 2:
1990
+ return "fatal", ("request", None, exc)
1991
+ return "retry", ("target", None, exc)
1992
+
1993
+ async def _count_target_errors(self, held: _Held, *, succeeded: bool) -> None:
1994
+ """Count noted target errors: after a success elsewhere, or a failure on one deployment."""
1995
+ noted = held.target_failed
1996
+ held.target_failed = []
1997
+ if not noted or (not succeeded and len({d for _, d in noted}) >= 2):
1998
+ return
1999
+ view = self.registry.current
2000
+ for alias, dep_key in noted:
2001
+ settings = view.value(alias, "health", dep_key)
2002
+ others = [d.key for d in view.deployments(alias) if d.key != dep_key]
2003
+ outcome = await self.health.record_target_error(
2004
+ alias,
2005
+ dep_key,
2006
+ held.envelope.request_id,
2007
+ others=others,
2008
+ window_secs=float(settings["disable_window_secs"]),
2009
+ needed=int(settings["disable_errors"]),
2010
+ probe_after_secs=float(settings["probe_after_secs"]),
2011
+ )
2012
+ if outcome == "disabled":
2013
+ logger.error("deployment_disabled", model=alias, deployment=dep_key)
2014
+ elif outcome == "last":
2015
+ logger.error("deployment_failing_last_usable", model=alias, deployment=dep_key)
2016
+
2017
+ async def _prober(self) -> None:
2018
+ """One process per model probes disabled deployments when they are due."""
2019
+ while True:
2020
+ await asyncio.sleep(5)
2021
+ try:
2022
+ view = self.registry.current
2023
+ for alias in view.registry.models:
2024
+ lock = await self.state.set(
2025
+ f"{self.keys.prefix}:lock:probe:{alias}", self.worker_id, nx=True, ex=5
2026
+ )
2027
+ if lock:
2028
+ await self.probe_due(alias)
2029
+ except Exception as exc:
2030
+ logger.warning("prober_failed", error=str(exc))
2031
+
2032
+ async def probe_due(self, alias: str) -> dict[str, str]:
2033
+ """Probe every disabled deployment of a model that is due; their outcomes."""
2034
+ view = self.registry.current
2035
+ now = self.clock.now()
2036
+ out: dict[str, str] = {}
2037
+ for dep_key, due in (await self.health.disabled(alias)).items():
2038
+ if due > now + 0.5: # the local view of the store clock may lag slightly
2039
+ continue
2040
+ ok = await self._probe(view, alias, dep_key)
2041
+ settings = view.value(alias, "health", dep_key)
2042
+ out[dep_key] = await self.health.probed(
2043
+ alias, dep_key, ok=ok, probe_max_secs=float(settings["probe_max_secs"])
2044
+ )
2045
+ if out[dep_key] == "recovered":
2046
+ logger.info("deployment_recovered", model=alias, deployment=dep_key)
2047
+ return out
2048
+
2049
+ async def _probe(self, view: ResolvedRegistry, alias: str, dep_key: str) -> bool:
2050
+ """One small real call on a deployment: reachable and allowed (a throttle counts)."""
2051
+ entry = view.registry.models[alias]
2052
+ deployment = next((d for d in view.deployments(alias) if d.key == dep_key), None)
2053
+ if deployment is None:
2054
+ return True
2055
+ op = OPS.get((entry.provider, entry.type), "chat" if entry.type == "chat" else "embeddings")
2056
+ name = view.value(alias, "probe")
2057
+ hook = hooks.get("probe", name)
2058
+ params = (
2059
+ hooks.call_safely("probe", name, hook, alias, op, default=None) if hook else None
2060
+ ) or _probe_params(entry.provider, entry.type, entry.model_id)
2061
+ provider = self.providers.get(entry.provider)
2062
+ try:
2063
+ await provider.call(op, params, route_for(view, deployment), Timing(30, 30))
2064
+ except Exception as exc:
2065
+ return provider.classify(exc).kind is ErrorKind.THROTTLE
2066
+ return True
2067
+
2068
+ # ================================================================ worker: settling
2069
+
2070
+ async def _reply(
2071
+ self,
2072
+ env: Envelope,
2073
+ kind: str,
2074
+ message: str,
2075
+ *,
2076
+ final: str = "failed",
2077
+ retry_after: float | None = None,
2078
+ partial: str | None = None,
2079
+ ) -> None:
2080
+ reply = Reply(
2081
+ request_id=env.request_id,
2082
+ ok=False,
2083
+ error=kind,
2084
+ message=message,
2085
+ retry_after_secs=retry_after,
2086
+ partial_text=partial or None,
2087
+ meta=Meta(served_model=env.model, worker=self.worker_id),
2088
+ )
2089
+ await self.requests.put_result(reply, final_state=final)
2090
+ await self.requests.set_status(env.request_id, final, error=kind)
2091
+
2092
+ async def _finish_call(
2093
+ self,
2094
+ held: _Held,
2095
+ kind: str,
2096
+ message: str,
2097
+ *,
2098
+ final: str = "failed",
2099
+ retry_after: float | None = None,
2100
+ partial: str | None = None,
2101
+ ) -> None:
2102
+ await self._reply(
2103
+ held.envelope, kind, message, final=final, retry_after=retry_after, partial=partial
2104
+ )
2105
+ await self._count_target_errors(held, succeeded=False)
2106
+ self._record(held, held.envelope.model, "", "", kind, None)
2107
+ await self._ack(held)
2108
+
2109
+ def _record(
2110
+ self, held: _Held, alias: str, account: str, region: str, outcome: str, meta: Meta | None
2111
+ ) -> None:
2112
+ """Metrics and the durable per-call record (`llmq_call_finished`, no text; E134)."""
2113
+ env = held.envelope
2114
+ labels = {"model": alias, "account": account or "-", "region": region or "-"}
2115
+ self.metrics.count("llmq_calls_total", 1.0, outcome=outcome, **labels)
2116
+ wait = max(0.0, held.first_seen - held.position_ms / 1000.0)
2117
+ self.metrics.observe(
2118
+ "llmq_wait_seconds", wait, model=env.model, priority="1" if env.priority else "0"
2119
+ )
2120
+ if meta is not None:
2121
+ self.metrics.count("llmq_tokens_input_total", float(meta.tokens_in or 0), **labels)
2122
+ self.metrics.count("llmq_tokens_output_total", float(meta.tokens_out or 0), **labels)
2123
+ if meta.fallback:
2124
+ self.metrics.count("llmq_fallback_total", kind=meta.fallback, model=env.model)
2125
+ user = self.hash_id(env.user_id) if env.user_id else None
2126
+ logger.info(
2127
+ "llmq_call_finished",
2128
+ request_id=env.request_id,
2129
+ model=env.model,
2130
+ served_model=alias,
2131
+ deployment=meta.deployment if meta else None,
2132
+ outcome=outcome,
2133
+ priority=env.priority,
2134
+ attempts=meta.attempts if meta else None,
2135
+ tokens_in=meta.tokens_in if meta else None,
2136
+ tokens_out=meta.tokens_out if meta else None,
2137
+ queue_wait_ms=int(wait * 1000),
2138
+ provider_ms=meta.provider_ms if meta else None,
2139
+ user=user,
2140
+ tenant=env.tenant_id,
2141
+ )
2142
+
2143
+ async def _debug(
2144
+ self, request_id: str, params: Mapping[str, Any], response: Mapping[str, Any]
2145
+ ) -> None:
2146
+ """`debug_capture`: truncated text for at most 60 minutes (plan §3.12)."""
2147
+ import orjson
2148
+
2149
+ body = orjson.dumps(
2150
+ {
2151
+ "request": orjson.dumps(params).decode()[:2000],
2152
+ "response": orjson.dumps(response).decode()[:2000],
2153
+ }
2154
+ )
2155
+ await self.state.set(self.keys.request(request_id, "debug"), body, ex=3600)
2156
+
2157
+ async def status(self, model: str | None = None) -> dict[str, Any]:
2158
+ """Status of one model, or of everything (plan §3.12)."""
2159
+ view = self.registry.current
2160
+ if model is not None:
2161
+ return await self.status_reader.model(view, view.registry.alias_of(model))
2162
+ return await self.status_reader.report(view)
2163
+
2164
+ async def request_status(self, request_id: str) -> dict[str, Any]:
2165
+ """Status of one call (empty when unknown or expired)."""
2166
+ return await self.status_reader.request(request_id)
2167
+
2168
+ async def health_check(self) -> dict[str, Any]:
2169
+ """Liveness and readiness (plan E124): ready = registry loaded and not draining.
2170
+
2171
+ The stores' state is reported but never makes the process unready. It is checked with a
2172
+ short-lived write, not PING: a replica answers PING but refuses writes (`read_only`).
2173
+ """
2174
+ probe = f"{self.keys.prefix}:health:{self.worker_id}"
2175
+
2176
+ async def check(client: Any) -> str:
2177
+ # Bounded: a blackholed store took 41 s (connect timeout x retries), past any probe.
2178
+ try:
2179
+ await asyncio.wait_for(
2180
+ client.set(probe, b"1", px=5000), self.settings.timings.health_probe_secs
2181
+ )
2182
+ except StoreReadOnlyError:
2183
+ return "read_only"
2184
+ except (*STORE_ERRORS, TimeoutError):
2185
+ return "down"
2186
+ return "ok"
2187
+
2188
+ if self._queue_store is None:
2189
+ state = queue = await check(self.state) # one store: probe it once
2190
+ else:
2191
+ state, queue = await asyncio.gather(check(self.state), check(self._queue_store))
2192
+ stores = {"state": state, "queue": queue}
2193
+ loaded = self.registry._current is not None
2194
+ return {"live": True, "ready": bool(self.ready and loaded), "stores": stores}
2195
+
2196
+ async def _alert_loop(self) -> None:
2197
+ """Every 30 s one process evaluates alerts: gauges every time, a log line per change."""
2198
+ while True:
2199
+ await asyncio.sleep(30)
2200
+ try:
2201
+ lock = await self.state.set(
2202
+ f"{self.keys.prefix}:lock:alerts", self.worker_id, nx=True, ex=25
2203
+ )
2204
+ if not lock:
2205
+ continue
2206
+ view = self.registry.current
2207
+ report = await self.status_reader.report(view)
2208
+ alerts = self.alerts.evaluate(view, report)
2209
+ for alert in alerts:
2210
+ self.metrics.gauge(
2211
+ f"llmq_alert_{alert.name}", 1.0 if alert.on else 0.0, **alert.labels
2212
+ )
2213
+ await self.alerts.changed(alerts)
2214
+ except Exception as exc:
2215
+ logger.warning("alerts_failed", error=str(exc))
2216
+
2217
+ async def _ack(self, held: _Held) -> None:
2218
+ if held.message is not None:
2219
+ await self.queues.ack(held.message)
2220
+ self._drop(held)
2221
+
2222
+ def _drop(self, held: _Held) -> None:
2223
+ """Forget a call, but only this one: a newer delivery may hold the same request id."""
2224
+ if self._held.get(held.envelope.request_id) is held:
2225
+ del self._held[held.envelope.request_id]
2226
+
2227
+ async def _give_back(self, held: _Held) -> None:
2228
+ """Settle every grant of a call as a cut attempt (nothing learned, unused budget back)."""
2229
+ for served, owner in list(held.grants):
2230
+ with contextlib.suppress(Exception):
2231
+ await self.admission.finish(
2232
+ served, owner, held.envelope.request_id, refund_r=0, refund_t=0, outcome="none"
2233
+ )
2234
+ held.grants.clear()
2235
+
2236
+ async def _release(self, held: _Held, delay_secs: float = 0.0) -> None:
2237
+ """Let go of a call after an unexpected error: grants back, not held, message returned."""
2238
+ await self._give_back(held)
2239
+ self._drop(held)
2240
+ if held.message is not None:
2241
+ with contextlib.suppress(Exception):
2242
+ await self.queues.return_(held.message, delay_secs=delay_secs)
2243
+
2244
+ # ================================================================ background
2245
+
2246
+ def _free_stuck(self) -> None:
2247
+ """Let go of calls held a lease past their deadline; nothing should still hold them.
2248
+
2249
+ Every path settles a call by its deadline, so one still held a lease later is a bug or
2250
+ a hung store call. The heartbeat renewed its grants for good (one-hour soak: two calls
2251
+ in progress for 50 minutes after a store block). Now it is logged, its task cancelled,
2252
+ its grants given back and its message returned (the next worker finds it expired).
2253
+ """
2254
+ late = self.clock.now() - self.settings.lease_secs - self.settings.caller_margin_secs
2255
+ for held in list(self._held.values()):
2256
+ if self._deadline(held) >= late:
2257
+ continue
2258
+ running = held.run is not None and not held.run.done()
2259
+ logger.error(
2260
+ "held_call_stuck",
2261
+ request_id=held.envelope.request_id,
2262
+ model=held.envelope.model,
2263
+ grants=sorted(held.grants),
2264
+ running=running,
2265
+ past_deadline_secs=round(self.clock.now() - self._deadline(held), 1),
2266
+ )
2267
+ self._drop(held) # never renewed again, even if the release below fails
2268
+ if running:
2269
+ assert held.run is not None
2270
+ held.run.cancel()
2271
+ task = asyncio.create_task(self._release(held))
2272
+ self._running.add(task)
2273
+ task.add_done_callback(self._running.discard)
2274
+
2275
+ async def _heartbeat(self) -> None:
2276
+ """Renew leases and holds; keep the caller liveness key; resync the clock.
2277
+
2278
+ It beats at least three times per lease and per held message's visibility timeout, so
2279
+ a short `visibility_secs` never lets a running call's message reappear. Each step fails
2280
+ on its own: one bad message (e.g. of a model removed by a reload) skips only itself.
2281
+ """
2282
+ while True:
2283
+ beat = self.settings.lease_secs / 3
2284
+ self._free_stuck()
2285
+ try:
2286
+ await self.state.set(
2287
+ self.keys.caller(self.worker_id), "1", ex=self.settings.caller_ttl_secs
2288
+ )
2289
+ items = [
2290
+ (alias, owner, held.envelope.request_id)
2291
+ for held in list(self._held.values())
2292
+ for alias, owner in list(held.grants)
2293
+ ]
2294
+ await self.admission.renew(items)
2295
+ now = time.monotonic()
2296
+ if now - self._beat_ok_at > 2 * beat + 1: # a gap: the store was out of reach
2297
+ self._store_trouble_at = now
2298
+ self._beat_ok_at = now
2299
+ except Exception as exc: # the heartbeat must survive store hiccups
2300
+ logger.warning("heartbeat_failed", step="renew", error=str(exc))
2301
+ self._store_trouble_at = time.monotonic()
2302
+ beat = 1.0 # renew the caller key as soon as the store is back
2303
+ view = self.registry.current
2304
+ for held in list(self._held.values()):
2305
+ if held.message is None:
2306
+ continue
2307
+ try:
2308
+ visibility = float(view.value(held.envelope.model, "queue")["visibility_secs"])
2309
+ beat = min(beat, visibility / 3)
2310
+ await self.queues.extend(held.message, visibility)
2311
+ except Exception as exc:
2312
+ logger.warning(
2313
+ "heartbeat_failed",
2314
+ step="extend",
2315
+ request_id=held.envelope.request_id,
2316
+ error=str(exc),
2317
+ )
2318
+ try:
2319
+ await self.clock.sync(self.state)
2320
+ await self._check_epoch()
2321
+ except Exception as exc:
2322
+ logger.warning("heartbeat_failed", step="clock", error=str(exc))
2323
+ await asyncio.sleep(max(0.2, beat))
2324
+
2325
+ async def _check_epoch(self) -> None:
2326
+ """The store lost its data (restart, failover): re-register running calls (T10)."""
2327
+ if await self.state.exists(self.keys.epoch):
2328
+ return
2329
+ logger.error("state_store_reset_detected")
2330
+ await self.state.set(self.keys.epoch, str(self.clock.now()), nx=True)
2331
+ await self.state.set(
2332
+ f"{self.keys.prefix}:reset_minute", int(self.clock.now() // 60), ex=120
2333
+ )
2334
+ items = [
2335
+ meta
2336
+ for held in list(self._held.values())
2337
+ for owner, meta in held.meta.items()
2338
+ if any(owner == o for _, o in held.grants)
2339
+ ]
2340
+ await self.admission.reregister(items)
2341
+
2342
+ async def _sweeper(self) -> None:
2343
+ """One process at a time removes in-progress entries of dead owners."""
2344
+ while True:
2345
+ await asyncio.sleep(self.settings.sweep_secs)
2346
+ try:
2347
+ lock = await self.state.set(
2348
+ f"{self.keys.prefix}:lock:sweep",
2349
+ self.worker_id,
2350
+ nx=True,
2351
+ ex=int(self.settings.sweep_secs),
2352
+ )
2353
+ if lock:
2354
+ for alias in self.registry.current.registry.models:
2355
+ await self.admission.sweep(alias)
2356
+ except Exception as exc:
2357
+ logger.warning("sweep_failed", error=str(exc))