llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,2357 @@
|
|
|
1
|
+
"""The engine of one process: caller and worker over the registry, queues and admission.
|
|
2
|
+
|
|
3
|
+
`Engine.call` registers a call (duplicate ids attach to the first run), sends it to its model's
|
|
4
|
+
queue and waits for the result. The worker side reads each model's priority queue first, admits
|
|
5
|
+
calls with one store call each, holds a refused call (and stops reading its class) until it is
|
|
6
|
+
granted or expires, runs granted calls with the attempt timers, retries by the refund rules and
|
|
7
|
+
falls back to fallback regions and other models by the registry's rules (plan §3.3 to §3.18).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import contextlib
|
|
14
|
+
import hashlib
|
|
15
|
+
import hmac
|
|
16
|
+
import random
|
|
17
|
+
import time
|
|
18
|
+
from collections.abc import AsyncIterator, Mapping, Sequence
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import TYPE_CHECKING, Any
|
|
21
|
+
|
|
22
|
+
from redis.exceptions import ConnectionError as StoreConnectionError
|
|
23
|
+
from redis.exceptions import OutOfMemoryError, ResponseError
|
|
24
|
+
from redis.exceptions import ReadOnlyError as StoreReadOnlyError
|
|
25
|
+
from redis.exceptions import TimeoutError as StoreTimeoutError
|
|
26
|
+
|
|
27
|
+
from llm_queue._logging import get_logger
|
|
28
|
+
from llm_queue.clients.pool import CredentialsUnavailable, route_for
|
|
29
|
+
from llm_queue.clients.providers import Providers
|
|
30
|
+
from llm_queue.clients.timing import AttemptTimeout, Timing
|
|
31
|
+
from llm_queue.engine import hooks, inference, tokens
|
|
32
|
+
from llm_queue.engine.admission import (
|
|
33
|
+
Admission,
|
|
34
|
+
Ask,
|
|
35
|
+
Candidate,
|
|
36
|
+
Decision,
|
|
37
|
+
owner_token,
|
|
38
|
+
position_of,
|
|
39
|
+
)
|
|
40
|
+
from llm_queue.engine.envelope import Envelope, Meta, Reply, params_hash, parse
|
|
41
|
+
from llm_queue.engine.errors import CallFailed
|
|
42
|
+
from llm_queue.engine.health import Health
|
|
43
|
+
from llm_queue.engine.keys import Keys
|
|
44
|
+
from llm_queue.engine.metrics import Metrics, NullSink
|
|
45
|
+
from llm_queue.engine.offload import ResultOffload
|
|
46
|
+
from llm_queue.engine.queue import (
|
|
47
|
+
Message,
|
|
48
|
+
NotReplicated,
|
|
49
|
+
QueueFull,
|
|
50
|
+
QueueSettings,
|
|
51
|
+
StreamQueues,
|
|
52
|
+
)
|
|
53
|
+
from llm_queue.engine.requests import RequestStore
|
|
54
|
+
from llm_queue.engine.status import AlertEvaluator, StatusReader
|
|
55
|
+
from llm_queue.engine.streaming import ChunkReader, ChunkWriter, StreamTooLarge
|
|
56
|
+
from llm_queue.errors import ErrorKind, UnsupportedEnvelopeError
|
|
57
|
+
from llm_queue.ids import new_id
|
|
58
|
+
from llm_queue.providers.classify import is_target_error
|
|
59
|
+
from llm_queue.tuning import EngineTimings
|
|
60
|
+
|
|
61
|
+
if TYPE_CHECKING:
|
|
62
|
+
import redis.asyncio
|
|
63
|
+
|
|
64
|
+
from llm_queue.registry import Deployment, ResolvedRegistry
|
|
65
|
+
from llm_queue.registry.store import RegistryStore
|
|
66
|
+
|
|
67
|
+
logger = get_logger(__name__)
|
|
68
|
+
OPS = {("bedrock", "chat"): "converse", ("bedrock", "embedding"): "invoke_model"}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _probe_params(provider: str, kind: str, model_id: str) -> dict[str, Any]:
|
|
72
|
+
"""The default probe request: one short word, one output token."""
|
|
73
|
+
if kind == "embedding":
|
|
74
|
+
if provider == "openai":
|
|
75
|
+
return {"input": "ping"}
|
|
76
|
+
if "cohere" in model_id:
|
|
77
|
+
return {"body": {"texts": ["ping"], "input_type": "search_query"}}
|
|
78
|
+
return {"body": {"inputText": "ping"}}
|
|
79
|
+
if provider == "openai":
|
|
80
|
+
return {"messages": [{"role": "user", "content": "ping"}], "max_tokens": 1}
|
|
81
|
+
return {
|
|
82
|
+
"messages": [{"role": "user", "content": [{"text": "ping"}]}],
|
|
83
|
+
"inferenceConfig": {"maxTokens": 1},
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _auth_error(code: str) -> bool:
|
|
88
|
+
"""Credential errors: drop the cached client so the secret is read again."""
|
|
89
|
+
lowered = code.lower()
|
|
90
|
+
return any(
|
|
91
|
+
part in lowered
|
|
92
|
+
for part in ("unrecognizedclient", "expiredtoken", "authentication", "invalid_api_key")
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True, slots=True)
|
|
97
|
+
class EngineSettings:
|
|
98
|
+
"""Process-level settings (plan §3.18 infrastructure table)."""
|
|
99
|
+
|
|
100
|
+
queue_prefix: str = ""
|
|
101
|
+
lease_secs: float = 60.0
|
|
102
|
+
waiter_secs: float = 2.0
|
|
103
|
+
result_ttl_secs: int = 300
|
|
104
|
+
result_dedupe_secs: int = 0
|
|
105
|
+
status_ttl_secs: int = 1800
|
|
106
|
+
caller_ttl_secs: int = 30
|
|
107
|
+
poll_secs: float = 5.0 # safety-net result check (the wake-up is published with the result)
|
|
108
|
+
sweep_secs: float = 10.0
|
|
109
|
+
receive_batch: int = 10
|
|
110
|
+
caller_margin_secs: float = 5.0
|
|
111
|
+
local_limit_cap: int = 10
|
|
112
|
+
drain_secs: float = 45.0
|
|
113
|
+
id_hash_salt: str = "" # LLMQ_ID_HASH_SALT: the key of every user / session id hash
|
|
114
|
+
cfg_reload_secs: float = 30.0 # LLMQ_CFG_RELOAD_SECS: how often the registry is re-read
|
|
115
|
+
queue_failover_secs: float = 5.0 # queue-store errors this long before calls run direct
|
|
116
|
+
queue_wait_replicas: int = 0 # a send waits for this many replicas (0: off)
|
|
117
|
+
queue_wait_ms: int = 100 # how long it waits for them
|
|
118
|
+
debug_capture_allowed: bool = False # LLMQ_DEBUG_CAPTURE_FROM_CONFIG (never in prod)
|
|
119
|
+
timings: EngineTimings = field(default_factory=EngineTimings) # `Tuning.engine`
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ARGV: key prefix, request ids... Reply: the 0-based indexes of the ids with a stored result.
|
|
123
|
+
_RESULTS_EXIST = """
|
|
124
|
+
local out = {}
|
|
125
|
+
for i = 2, #ARGV do
|
|
126
|
+
if redis.call('EXISTS', ARGV[1] .. ':{r:' .. ARGV[i] .. '}:res') == 1 then
|
|
127
|
+
table.insert(out, i - 2)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
return out
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _refusal(exc: BaseException) -> str:
|
|
135
|
+
"""A caller-facing message for a store that answered with a refusal."""
|
|
136
|
+
if isinstance(exc, OutOfMemoryError) or "OOM" in str(exc):
|
|
137
|
+
return "the state store is out of memory" # redis-py strips the "OOM" prefix
|
|
138
|
+
return f"the state store refused the call: {exc}"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# READONLY: a failover left this connection on the old primary (now a replica).
|
|
142
|
+
STORE_ERRORS = (
|
|
143
|
+
StoreConnectionError,
|
|
144
|
+
StoreTimeoutError,
|
|
145
|
+
StoreReadOnlyError,
|
|
146
|
+
ConnectionError,
|
|
147
|
+
OSError,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@dataclass(slots=True)
|
|
152
|
+
class _Held:
|
|
153
|
+
"""A call this process holds (waiting for budget, or running).
|
|
154
|
+
|
|
155
|
+
`message` is None on the direct path (kill switch, `queue_pct`, queue-store failover); its
|
|
156
|
+
position is then the store's time at registration.
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
message: Message | None
|
|
160
|
+
envelope: Envelope
|
|
161
|
+
first_seen: float
|
|
162
|
+
position_ms: int
|
|
163
|
+
tag: str
|
|
164
|
+
grants: set[tuple[str, str]] = field(default_factory=set)
|
|
165
|
+
meta: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
166
|
+
broken: bool = False
|
|
167
|
+
cancelled: bool = False
|
|
168
|
+
target_failed: list[tuple[str, str]] = field(default_factory=list)
|
|
169
|
+
run: asyncio.Task[None] | None = None
|
|
170
|
+
# Model-level request and estimate, built once per registry view (a deep copy of the
|
|
171
|
+
# prompt each time: ~0.7 ms for 200 KB, and it was rebuilt in _ask, _decide and _attempt).
|
|
172
|
+
charged: tuple[Any, str, tokens.Estimate] | None = None
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def position(self) -> float:
|
|
176
|
+
"""Queue position as a number (direct calls: their registration time)."""
|
|
177
|
+
if self.message is not None:
|
|
178
|
+
return position_of(self.message.id)
|
|
179
|
+
return float(self.position_ms * 1000)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class _QueueDown(Exception):
|
|
183
|
+
"""The queue store refused a send while the state store works: run the call direct."""
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class _SendNotReplicated(Exception): # internal signal, never raised to callers
|
|
187
|
+
"""The send reached the primary but no replica in time: the caller runs it direct."""
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class StoreClock:
|
|
191
|
+
"""The state store's clock, from one `TIME` read and the local monotonic clock."""
|
|
192
|
+
|
|
193
|
+
def __init__(self) -> None:
|
|
194
|
+
"""No offset until `sync`."""
|
|
195
|
+
self._offset = 0.0
|
|
196
|
+
|
|
197
|
+
async def sync(self, client: redis.asyncio.Redis) -> None:
|
|
198
|
+
"""Measure the offset between the store's clock and this process."""
|
|
199
|
+
before = time.monotonic()
|
|
200
|
+
secs, micros = await client.time()
|
|
201
|
+
after = time.monotonic()
|
|
202
|
+
self._offset = (secs + micros / 1e6) - (before + after) / 2
|
|
203
|
+
|
|
204
|
+
def now(self) -> float:
|
|
205
|
+
"""Seconds on the store's clock."""
|
|
206
|
+
return time.monotonic() + self._offset
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class Engine:
|
|
210
|
+
"""Caller and worker of one process (plan E72: workers run in the app pods)."""
|
|
211
|
+
|
|
212
|
+
def __init__(
|
|
213
|
+
self,
|
|
214
|
+
state: redis.asyncio.Redis,
|
|
215
|
+
registry: RegistryStore,
|
|
216
|
+
providers: Providers,
|
|
217
|
+
*,
|
|
218
|
+
queue_store: redis.asyncio.Redis | None = None,
|
|
219
|
+
settings: EngineSettings | None = None,
|
|
220
|
+
worker_id: str | None = None,
|
|
221
|
+
metrics: Metrics | None = None,
|
|
222
|
+
offload: ResultOffload | None = None,
|
|
223
|
+
offload_bytes: int | None = None,
|
|
224
|
+
) -> None:
|
|
225
|
+
"""Wire the stores; nothing runs until `start`."""
|
|
226
|
+
self.settings = settings or EngineSettings()
|
|
227
|
+
self.keys = Keys(registry.prefix)
|
|
228
|
+
self.state = state
|
|
229
|
+
self.registry = registry
|
|
230
|
+
self.providers = providers
|
|
231
|
+
self._queue_store = queue_store
|
|
232
|
+
self.queues = StreamQueues(queue_store or state, self.keys)
|
|
233
|
+
self.requests = RequestStore(
|
|
234
|
+
state,
|
|
235
|
+
self.keys,
|
|
236
|
+
result_ttl_secs=self.settings.result_ttl_secs,
|
|
237
|
+
result_dedupe_secs=self.settings.result_dedupe_secs,
|
|
238
|
+
status_ttl_secs=self.settings.status_ttl_secs,
|
|
239
|
+
offload=offload,
|
|
240
|
+
offload_bytes=offload_bytes,
|
|
241
|
+
)
|
|
242
|
+
self.admission = Admission(
|
|
243
|
+
state,
|
|
244
|
+
self.keys,
|
|
245
|
+
lease_secs=self.settings.lease_secs,
|
|
246
|
+
waiter_secs=self.settings.waiter_secs,
|
|
247
|
+
timings=self.settings.timings,
|
|
248
|
+
)
|
|
249
|
+
self.worker_id = worker_id or new_id()[-14:] # the random tail: the head is time
|
|
250
|
+
self.clock = StoreClock()
|
|
251
|
+
self._tasks: list[asyncio.Task[None]] = []
|
|
252
|
+
self._running: set[asyncio.Task[None]] = set()
|
|
253
|
+
self._held: dict[str, _Held] = {}
|
|
254
|
+
self._held_class: set[tuple[str, str]] = set()
|
|
255
|
+
self._slots: dict[str, asyncio.Semaphore] = {}
|
|
256
|
+
self._waiters: dict[str, asyncio.Future[None]] = {}
|
|
257
|
+
self._pubsub: Any = None
|
|
258
|
+
self._cancels: Any = None
|
|
259
|
+
self._stopping = False
|
|
260
|
+
self._paused = False
|
|
261
|
+
self._local_callers: dict[str, int] = {} # request id -> coroutines waiting on it here
|
|
262
|
+
self._wake: dict[str, asyncio.Event] = {} # model -> wakes its receive loop
|
|
263
|
+
self._beat_ok_at = time.monotonic() # last heartbeat that reached the store
|
|
264
|
+
self._store_trouble_at = -1e9 # monotonic time this process last saw the store fail
|
|
265
|
+
self._results_exist = state.register_script(_RESULTS_EXIST)
|
|
266
|
+
self.ready = False
|
|
267
|
+
self._direct_locks: dict[tuple[str, str], asyncio.Lock] = {}
|
|
268
|
+
self.health = Health(state, self.keys)
|
|
269
|
+
self.metrics: Metrics = metrics or NullSink()
|
|
270
|
+
self.status_reader = StatusReader(state, self.keys, self.queues, self.settings.queue_prefix)
|
|
271
|
+
self.alerts = AlertEvaluator(state, self.keys)
|
|
272
|
+
# Credentials that failed in this process, per account and region: (account, region)
|
|
273
|
+
# -> (until, backoff). Per region: a token can be refused in one region only (an
|
|
274
|
+
# opt-in region such as ap-south-2 refuses v1 STS tokens), and keying by account took
|
|
275
|
+
# every region of it down (scenario P5).
|
|
276
|
+
self._bad_accounts: dict[tuple[str, str], tuple[float, float]] = {}
|
|
277
|
+
self._idle = asyncio.Event()
|
|
278
|
+
self._static: dict[str, asyncio.Semaphore] = {}
|
|
279
|
+
self._rng = random.Random()
|
|
280
|
+
self.chunks = ChunkReader(state, block_ms=self.settings.timings.stream_read_block_ms)
|
|
281
|
+
|
|
282
|
+
# ================================================================ lifecycle
|
|
283
|
+
|
|
284
|
+
async def start(self, *, worker: bool = True, caller: bool = True) -> None:
|
|
285
|
+
"""Boot the registry, then start the worker loops and the result reader."""
|
|
286
|
+
await self.registry.boot() # runs on its own config when the store is down
|
|
287
|
+
try:
|
|
288
|
+
await self.clock.sync(self.state)
|
|
289
|
+
await self.state.set(self.keys.epoch, str(self.clock.now()), nx=True)
|
|
290
|
+
except STORE_ERRORS as exc:
|
|
291
|
+
# Start anyway (plan §3.18): calls follow each model's `store_down` rule and
|
|
292
|
+
# the background loops keep retrying until the store answers.
|
|
293
|
+
logger.error("state_store_unreachable_at_start", error=str(exc))
|
|
294
|
+
if caller:
|
|
295
|
+
self._pubsub = self.state.pubsub()
|
|
296
|
+
self._tasks.append(asyncio.create_task(self._result_reader()))
|
|
297
|
+
self._tasks.append(asyncio.create_task(self._result_sweep()))
|
|
298
|
+
if worker:
|
|
299
|
+
for alias in self.registry.current.registry.models:
|
|
300
|
+
slots = int(self.registry.current.value(alias, "worker_slots"))
|
|
301
|
+
self._slots[alias] = asyncio.Semaphore(slots)
|
|
302
|
+
self._tasks.append(asyncio.create_task(self._model_loop(alias)))
|
|
303
|
+
self._tasks.append(asyncio.create_task(self._sweeper()))
|
|
304
|
+
self._tasks.append(asyncio.create_task(self._prober()))
|
|
305
|
+
self._tasks.append(asyncio.create_task(self._alert_loop()))
|
|
306
|
+
self._cancels = self.state.pubsub()
|
|
307
|
+
self._tasks.append(asyncio.create_task(self._cancel_reader()))
|
|
308
|
+
self._tasks.append(asyncio.create_task(self._warm_clients()))
|
|
309
|
+
self._tasks.append(asyncio.create_task(self._heartbeat()))
|
|
310
|
+
self._tasks.append(asyncio.create_task(self.registry.run(self.settings.cfg_reload_secs)))
|
|
311
|
+
self.ready = True
|
|
312
|
+
|
|
313
|
+
def pause_receiving(self) -> None:
|
|
314
|
+
"""Not ready, and take no new calls from the queues; running and held calls go on.
|
|
315
|
+
|
|
316
|
+
The first step of a shutdown, for a preStop hook: the load balancer stops sending
|
|
317
|
+
requests while this pod finishes its work. `stop()` follows.
|
|
318
|
+
"""
|
|
319
|
+
self.ready = False
|
|
320
|
+
self._paused = True
|
|
321
|
+
|
|
322
|
+
async def stop(self, drain_secs: float | None = None) -> None:
|
|
323
|
+
"""Shut down in order (plan §3.18): not ready, stop receiving, finish running calls.
|
|
324
|
+
|
|
325
|
+
Held calls go back to the queue; results keep being read for this pod's own callers
|
|
326
|
+
until they finish or the drain ends; then everything stops.
|
|
327
|
+
"""
|
|
328
|
+
drain = self.settings.drain_secs if drain_secs is None else drain_secs
|
|
329
|
+
loop = asyncio.get_running_loop()
|
|
330
|
+
until = loop.time() + drain
|
|
331
|
+
self.ready = False
|
|
332
|
+
self._stopping = True
|
|
333
|
+
for held in list(self._held.values()):
|
|
334
|
+
if not held.grants and held.run is None and held.message is not None:
|
|
335
|
+
with contextlib.suppress(*STORE_ERRORS):
|
|
336
|
+
await self.queues.return_(held.message)
|
|
337
|
+
self._drop(held) # else the heartbeat hides it again for the whole drain
|
|
338
|
+
if self._running:
|
|
339
|
+
await asyncio.wait(set(self._running), timeout=max(0.0, until - loop.time()))
|
|
340
|
+
if self._waiters:
|
|
341
|
+
self._idle.clear()
|
|
342
|
+
with contextlib.suppress(TimeoutError):
|
|
343
|
+
await asyncio.wait_for(self._idle.wait(), max(0.0, until - loop.time()))
|
|
344
|
+
for task in [*self._tasks, *self._running]:
|
|
345
|
+
task.cancel()
|
|
346
|
+
for task in [*self._tasks, *self._running]:
|
|
347
|
+
with contextlib.suppress(asyncio.CancelledError, Exception):
|
|
348
|
+
await task
|
|
349
|
+
await self.chunks.aclose()
|
|
350
|
+
for pubsub in (self._pubsub, self._cancels):
|
|
351
|
+
if pubsub is not None:
|
|
352
|
+
with contextlib.suppress(Exception):
|
|
353
|
+
await pubsub.aclose()
|
|
354
|
+
|
|
355
|
+
async def aclose(self) -> None:
|
|
356
|
+
"""Close the provider clients and both store clients (after `stop()`)."""
|
|
357
|
+
await self.providers.aclose()
|
|
358
|
+
if self._queue_store is not None:
|
|
359
|
+
await self._queue_store.aclose()
|
|
360
|
+
await self.state.aclose()
|
|
361
|
+
|
|
362
|
+
# ================================================================ caller
|
|
363
|
+
|
|
364
|
+
def _queue_name(self, view: ResolvedRegistry, alias: str, priority: bool) -> str:
|
|
365
|
+
name = self.settings.queue_prefix + view.queue_name(alias)
|
|
366
|
+
return f"{name}-priority" if priority else name
|
|
367
|
+
|
|
368
|
+
async def call(
|
|
369
|
+
self,
|
|
370
|
+
model: str,
|
|
371
|
+
params: Mapping[str, Any],
|
|
372
|
+
*,
|
|
373
|
+
priority: bool = False,
|
|
374
|
+
timeout_secs: float | None = None,
|
|
375
|
+
max_wait_secs: float | None = None,
|
|
376
|
+
request_id: str | None = None,
|
|
377
|
+
session_id: str | None = None,
|
|
378
|
+
user_id: str | None = None,
|
|
379
|
+
tenant_id: str | None = None,
|
|
380
|
+
allow_fallback: str = "all",
|
|
381
|
+
allowed_regions: Sequence[str] | None = None,
|
|
382
|
+
allowed_accounts: Sequence[str] | None = None,
|
|
383
|
+
force_direct: bool = False,
|
|
384
|
+
stream: bool = False,
|
|
385
|
+
stream_id: str | None = None,
|
|
386
|
+
) -> Reply:
|
|
387
|
+
"""Run one call through the queue and return its reply; `CallFailed` on failure.
|
|
388
|
+
|
|
389
|
+
`force_direct` runs it in this process (sync calls), with the same admission.
|
|
390
|
+
`stream` has the worker write events to the chunk stream of `stream_id` (`stream`).
|
|
391
|
+
"""
|
|
392
|
+
view = self.registry.current
|
|
393
|
+
alias = view.registry.alias_of(model)
|
|
394
|
+
if view.effective(alias, "enabled").level == "model" and not view.value(alias, "enabled"):
|
|
395
|
+
rule = view.value(alias, "model_fallback")
|
|
396
|
+
if rule and allow_fallback == "all" and rule["models"]:
|
|
397
|
+
alias = rule["models"][0]["model"]
|
|
398
|
+
else:
|
|
399
|
+
raise CallFailed("model_disabled", f"{alias} is disabled")
|
|
400
|
+
entry = view.registry.models[alias]
|
|
401
|
+
op = OPS.get((entry.provider, entry.type), "chat" if entry.type == "chat" else "embeddings")
|
|
402
|
+
timeout = float(timeout_secs or view.value(alias, "timeout_secs"))
|
|
403
|
+
timeout = min(timeout, float(view.value(alias, "timeout_max_secs")))
|
|
404
|
+
rid = request_id or stream_id or new_id()
|
|
405
|
+
envelope = Envelope(
|
|
406
|
+
request_id=rid,
|
|
407
|
+
op=op,
|
|
408
|
+
model=alias,
|
|
409
|
+
params=dict(params),
|
|
410
|
+
priority=priority,
|
|
411
|
+
caller=self.worker_id,
|
|
412
|
+
timeout_secs=timeout,
|
|
413
|
+
max_wait_secs=max_wait_secs or view.value(alias, "max_wait_secs"),
|
|
414
|
+
allow_fallback=allow_fallback,
|
|
415
|
+
allowed_regions=tuple(allowed_regions) if allowed_regions else None,
|
|
416
|
+
allowed_accounts=tuple(allowed_accounts) if allowed_accounts else None,
|
|
417
|
+
session_id=session_id,
|
|
418
|
+
user_id=user_id,
|
|
419
|
+
tenant_id=tenant_id,
|
|
420
|
+
app_request_id=request_id is not None,
|
|
421
|
+
stream=stream,
|
|
422
|
+
)
|
|
423
|
+
hook_name = view.value(alias, "admission")
|
|
424
|
+
hook = hooks.get("admission", hook_name)
|
|
425
|
+
if hook is not None:
|
|
426
|
+
ctx = hooks.CallContext(alias, rid, priority, session_id, user_id, tenant_id)
|
|
427
|
+
verdict = hooks.call_safely("admission", hook_name, hook, ctx, default=None)
|
|
428
|
+
if isinstance(verdict, hooks.Reject):
|
|
429
|
+
raise CallFailed("rejected", verdict.reason, request_id=rid)
|
|
430
|
+
direct = force_direct or self._direct_route(view, alias, rid, user_id)
|
|
431
|
+
self._local_callers[rid] = self._local_callers.get(rid, 0) + 1
|
|
432
|
+
# One deadline for the whole call, whatever path it takes: a store outage sent calls
|
|
433
|
+
# down the store_down path with a fresh timeout, and the soak saw 135 answers come
|
|
434
|
+
# after their deadline (20 s store block, 30 s calls).
|
|
435
|
+
deadline = asyncio.get_running_loop().time() + timeout + self.settings.caller_margin_secs
|
|
436
|
+
try:
|
|
437
|
+
async with asyncio.timeout_at(deadline):
|
|
438
|
+
try:
|
|
439
|
+
return await self._call_once(
|
|
440
|
+
view, alias, envelope, params, priority, timeout, request_id, direct
|
|
441
|
+
)
|
|
442
|
+
except _QueueDown:
|
|
443
|
+
return await self._call_once(
|
|
444
|
+
view, alias, envelope, params, priority, timeout, request_id, True
|
|
445
|
+
)
|
|
446
|
+
except STORE_ERRORS as exc:
|
|
447
|
+
if stream: # chunks travel through the state store
|
|
448
|
+
message = "the state store is unreachable"
|
|
449
|
+
raise CallFailed("queue_unavailable", message) from exc
|
|
450
|
+
return await self._store_down(view, alias, envelope, exc, deadline)
|
|
451
|
+
except ResponseError as exc:
|
|
452
|
+
# The store answered but refused (out of memory, a script error): still one
|
|
453
|
+
# error family for callers. A refused registration raised redis-py's
|
|
454
|
+
# OutOfMemoryError straight to the app (scenario S9).
|
|
455
|
+
raise CallFailed("queue_unavailable", _refusal(exc), request_id=rid) from exc
|
|
456
|
+
except TimeoutError as exc:
|
|
457
|
+
raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid) from exc
|
|
458
|
+
except asyncio.CancelledError:
|
|
459
|
+
# The awaiting task was cancelled: cancel the call unless others wait on its id,
|
|
460
|
+
# in other processes or in this one (they share this process's caller entry).
|
|
461
|
+
with contextlib.suppress(Exception):
|
|
462
|
+
callers = await asyncio.shield(self.requests.callers(rid))
|
|
463
|
+
alone_here = self._local_callers.get(rid, 0) <= 1
|
|
464
|
+
if alone_here and set(callers) <= {self.worker_id}:
|
|
465
|
+
await asyncio.shield(self.cancel(rid))
|
|
466
|
+
raise
|
|
467
|
+
finally:
|
|
468
|
+
left = self._local_callers.get(rid, 1) - 1
|
|
469
|
+
if left:
|
|
470
|
+
self._local_callers[rid] = left
|
|
471
|
+
else:
|
|
472
|
+
self._local_callers.pop(rid, None)
|
|
473
|
+
|
|
474
|
+
async def stream(
|
|
475
|
+
self, model: str, params: Mapping[str, Any], **call: Any
|
|
476
|
+
) -> AsyncIterator[dict[str, Any]]:
|
|
477
|
+
"""Run a chat call and yield the provider's events as they arrive (plan §3.13, F6).
|
|
478
|
+
|
|
479
|
+
Takes `call`'s options. The events are ConverseStream events (Bedrock) or chat
|
|
480
|
+
completion chunks (OpenAI-compatible). A failure after events were yielded raises
|
|
481
|
+
`CallFailed` with `partial_text`; no switch to another deployment happens then.
|
|
482
|
+
"""
|
|
483
|
+
view = self.registry.current
|
|
484
|
+
alias = view.registry.alias_of(model)
|
|
485
|
+
idle = float(view.value(alias, "stream_idle_secs"))
|
|
486
|
+
rid = call.pop("request_id", None) or new_id()
|
|
487
|
+
key = self.keys.request(rid, "chunks")
|
|
488
|
+
queue = self.chunks.open(key)
|
|
489
|
+
# The call's own deadline, as `call` computes it: the idle wait between chunks
|
|
490
|
+
# (stream_idle_secs, 30) once let a stream whose worker died answer after it (scenario
|
|
491
|
+
# L2 at 300 %: 38.8 s for a 15 s deadline).
|
|
492
|
+
timeout = float(call.get("timeout_secs") or view.value(alias, "timeout_secs"))
|
|
493
|
+
timeout = min(timeout, float(view.value(alias, "timeout_max_secs")))
|
|
494
|
+
loop = asyncio.get_running_loop()
|
|
495
|
+
deadline = loop.time() + timeout + self.settings.caller_margin_secs
|
|
496
|
+
task = asyncio.create_task(self.call(model, params, stream=True, stream_id=rid, **call))
|
|
497
|
+
expected, started = 1, False
|
|
498
|
+
try:
|
|
499
|
+
while True:
|
|
500
|
+
left = deadline - loop.time()
|
|
501
|
+
if left <= 0:
|
|
502
|
+
raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid)
|
|
503
|
+
getter = asyncio.ensure_future(queue.get())
|
|
504
|
+
finished = task.done()
|
|
505
|
+
done, _ = await asyncio.wait(
|
|
506
|
+
{getter} if finished else {getter, task},
|
|
507
|
+
timeout=min(idle, left) if started or finished else left,
|
|
508
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
509
|
+
)
|
|
510
|
+
if getter not in done:
|
|
511
|
+
getter.cancel()
|
|
512
|
+
if not done and deadline - loop.time() <= 0: # the deadline, not idleness
|
|
513
|
+
raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid)
|
|
514
|
+
if task in done:
|
|
515
|
+
if not started:
|
|
516
|
+
# The store may be down too: then nothing can be replayed, and
|
|
517
|
+
# the call's own failure is the answer (scenario L2: this read
|
|
518
|
+
# raised redis-py's ConnectionError to the app).
|
|
519
|
+
try:
|
|
520
|
+
pending = await self.state.xlen(key)
|
|
521
|
+
except STORE_ERRORS:
|
|
522
|
+
task.result() # the call's failure, if it failed
|
|
523
|
+
raise CallFailed(
|
|
524
|
+
"queue_unavailable",
|
|
525
|
+
"the store is unreachable; the stream cannot be read",
|
|
526
|
+
request_id=rid,
|
|
527
|
+
) from None
|
|
528
|
+
if not pending:
|
|
529
|
+
task.result() # raises the call's failure (nothing streamed)
|
|
530
|
+
return # a finished call whose chunks expired
|
|
531
|
+
continue # its last entries are still on the way
|
|
532
|
+
if finished:
|
|
533
|
+
task.result()
|
|
534
|
+
raise CallFailed(
|
|
535
|
+
"worker_lost", "the stream ended without its end entry", request_id=rid
|
|
536
|
+
)
|
|
537
|
+
raise CallFailed(
|
|
538
|
+
"deadline_exceeded",
|
|
539
|
+
f"no chunk for {idle:g} s (stream_idle_secs)",
|
|
540
|
+
request_id=rid,
|
|
541
|
+
)
|
|
542
|
+
entry = getter.result()
|
|
543
|
+
if entry["seq"] != expected:
|
|
544
|
+
raise CallFailed(
|
|
545
|
+
"worker_lost", f"gap in the stream at {expected}", request_id=rid
|
|
546
|
+
)
|
|
547
|
+
expected += 1
|
|
548
|
+
started = True
|
|
549
|
+
if entry["t"] == "chunk":
|
|
550
|
+
for event in entry["d"]:
|
|
551
|
+
yield event
|
|
552
|
+
elif entry["t"] == "end":
|
|
553
|
+
await task
|
|
554
|
+
return
|
|
555
|
+
else:
|
|
556
|
+
data = entry["d"]
|
|
557
|
+
raise CallFailed(
|
|
558
|
+
data.get("kind", "provider_error"),
|
|
559
|
+
data.get("message", ""),
|
|
560
|
+
partial_text=data.get("partial") or None,
|
|
561
|
+
request_id=rid,
|
|
562
|
+
)
|
|
563
|
+
finally:
|
|
564
|
+
self.chunks.close(key)
|
|
565
|
+
if not task.done():
|
|
566
|
+
task.cancel() # the call's own handler cancels the request
|
|
567
|
+
with contextlib.suppress(BaseException):
|
|
568
|
+
await task
|
|
569
|
+
|
|
570
|
+
def hash_id(self, value: str) -> str:
|
|
571
|
+
"""Keyed hash (HMAC-SHA256, `LLMQ_ID_HASH_SALT`) of a user or session id, 16 hex chars.
|
|
572
|
+
|
|
573
|
+
Every id outside the message itself is stored and logged only as this (plan §3.17):
|
|
574
|
+
status records, sticky keys, logs and `routing.allow_users`. Unkeyed, a low-entropy id
|
|
575
|
+
(a client code, a phone number) is brute-forced back from the hash.
|
|
576
|
+
"""
|
|
577
|
+
key = self.settings.id_hash_salt.encode()
|
|
578
|
+
return hmac.new(key, value.encode(), hashlib.sha256).hexdigest()[:16]
|
|
579
|
+
|
|
580
|
+
def _direct_route(
|
|
581
|
+
self, view: ResolvedRegistry, alias: str, rid: str, user_id: str | None
|
|
582
|
+
) -> bool:
|
|
583
|
+
"""True when this call runs direct: kill switch, or outside `queue_pct` (canary)."""
|
|
584
|
+
routing = view.registry.routing
|
|
585
|
+
if user_id and self.hash_id(user_id) in routing.allow_users:
|
|
586
|
+
return False
|
|
587
|
+
if routing.kill_switch:
|
|
588
|
+
return True
|
|
589
|
+
pct = int(view.value(alias, "queue_pct"))
|
|
590
|
+
return pct < 100 and int(hashlib.sha256(rid.encode()).hexdigest()[:8], 16) % 100 >= pct
|
|
591
|
+
|
|
592
|
+
async def _call_once(
|
|
593
|
+
self,
|
|
594
|
+
view: ResolvedRegistry,
|
|
595
|
+
alias: str,
|
|
596
|
+
envelope: Envelope,
|
|
597
|
+
params: Mapping[str, Any],
|
|
598
|
+
priority: bool,
|
|
599
|
+
call_secs: float,
|
|
600
|
+
request_id: str | None,
|
|
601
|
+
direct: bool,
|
|
602
|
+
) -> Reply:
|
|
603
|
+
rid = envelope.request_id
|
|
604
|
+
user_id, session_id = envelope.user_id, envelope.session_id
|
|
605
|
+
user_hash = self.hash_id(user_id or session_id or "")
|
|
606
|
+
found = await self.requests.register(
|
|
607
|
+
rid,
|
|
608
|
+
caller=self.worker_id,
|
|
609
|
+
user_hash=user_hash,
|
|
610
|
+
params_hash=params_hash({"m": alias, "p": params}),
|
|
611
|
+
model=alias,
|
|
612
|
+
priority=priority,
|
|
613
|
+
app_id=request_id is not None,
|
|
614
|
+
)
|
|
615
|
+
if found.outcome == "conflict":
|
|
616
|
+
raise CallFailed(
|
|
617
|
+
"request_id_conflict", "the id was used with other params", request_id=rid
|
|
618
|
+
)
|
|
619
|
+
if found.outcome == "expired":
|
|
620
|
+
raise CallFailed("result_expired", "the id finished earlier", request_id=rid)
|
|
621
|
+
waiter = self._waiters.setdefault(rid, asyncio.get_running_loop().create_future())
|
|
622
|
+
loop = asyncio.get_running_loop()
|
|
623
|
+
until = loop.time() + call_secs + self.settings.caller_margin_secs
|
|
624
|
+
outcome = found.outcome
|
|
625
|
+
try:
|
|
626
|
+
while True:
|
|
627
|
+
if outcome in ("new", "takeover"):
|
|
628
|
+
if direct:
|
|
629
|
+
await self.requests.confirm(rid, keep_secs=call_secs)
|
|
630
|
+
task = asyncio.create_task(self._direct(envelope))
|
|
631
|
+
self._running.add(task)
|
|
632
|
+
task.add_done_callback(self._running.discard)
|
|
633
|
+
else:
|
|
634
|
+
try:
|
|
635
|
+
await self._send(rid, envelope, view, alias, priority, call_secs)
|
|
636
|
+
except _SendNotReplicated:
|
|
637
|
+
await self.requests.confirm(rid, keep_secs=call_secs)
|
|
638
|
+
task = asyncio.create_task(self._direct(envelope))
|
|
639
|
+
self._running.add(task)
|
|
640
|
+
task.add_done_callback(self._running.discard)
|
|
641
|
+
reply = await self._wait_result(
|
|
642
|
+
rid, waiter, until - loop.time(), recheck=outcome == "attached"
|
|
643
|
+
)
|
|
644
|
+
if reply is not None:
|
|
645
|
+
return reply
|
|
646
|
+
# The send this caller attached to never happened: register again.
|
|
647
|
+
again = await self.requests.register(
|
|
648
|
+
rid,
|
|
649
|
+
caller=self.worker_id,
|
|
650
|
+
user_hash=user_hash,
|
|
651
|
+
params_hash=params_hash({"m": alias, "p": params}),
|
|
652
|
+
model=alias,
|
|
653
|
+
priority=priority,
|
|
654
|
+
app_id=request_id is not None,
|
|
655
|
+
)
|
|
656
|
+
outcome = again.outcome
|
|
657
|
+
if outcome == "conflict":
|
|
658
|
+
raise CallFailed("request_id_conflict", "the id was used with other params")
|
|
659
|
+
finally:
|
|
660
|
+
# Other coroutines here may wait on the same id and share its wake-up: removing
|
|
661
|
+
# it left them asleep until their deadline's last read (one waiter cancelled).
|
|
662
|
+
if self._local_callers.get(rid, 1) <= 1:
|
|
663
|
+
self._waiters.pop(rid, None)
|
|
664
|
+
if not self._waiters:
|
|
665
|
+
self._idle.set()
|
|
666
|
+
|
|
667
|
+
async def embed(
|
|
668
|
+
self,
|
|
669
|
+
model: str,
|
|
670
|
+
texts: Sequence[str],
|
|
671
|
+
*,
|
|
672
|
+
priority: bool = False,
|
|
673
|
+
timeout_secs: float | None = None,
|
|
674
|
+
input_type: str | None = None,
|
|
675
|
+
**call: Any,
|
|
676
|
+
) -> list[list[float]]:
|
|
677
|
+
"""One vector per text, in order (batched by the model's `embed_batch_size`)."""
|
|
678
|
+
from llm_queue.engine.embed import embed
|
|
679
|
+
|
|
680
|
+
return await embed(
|
|
681
|
+
self,
|
|
682
|
+
model,
|
|
683
|
+
texts,
|
|
684
|
+
priority=priority,
|
|
685
|
+
timeout_secs=timeout_secs,
|
|
686
|
+
input_type=input_type,
|
|
687
|
+
**call,
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
async def _send(
|
|
691
|
+
self,
|
|
692
|
+
rid: str,
|
|
693
|
+
envelope: Envelope,
|
|
694
|
+
view: ResolvedRegistry,
|
|
695
|
+
alias: str,
|
|
696
|
+
priority: bool,
|
|
697
|
+
keep_secs: float,
|
|
698
|
+
) -> None:
|
|
699
|
+
"""Put the message on its queue and confirm the record; undo the record on failure.
|
|
700
|
+
|
|
701
|
+
A separate queue store that keeps failing for `queue_failover_secs` sends the call
|
|
702
|
+
down the direct path; shorter blips are retried. With `queue_wait_replicas`, a send
|
|
703
|
+
that no replica acknowledged in time is a failed send (plan B23) and the call runs
|
|
704
|
+
direct.
|
|
705
|
+
"""
|
|
706
|
+
settings = QueueSettings(max_waiting=int(view.value(alias, "max_waiting")))
|
|
707
|
+
loop = asyncio.get_running_loop()
|
|
708
|
+
give_up = loop.time() + self.settings.queue_failover_secs
|
|
709
|
+
body = envelope.dumps()
|
|
710
|
+
while True:
|
|
711
|
+
try:
|
|
712
|
+
await self.queues.send(
|
|
713
|
+
self._queue_name(view, alias, priority),
|
|
714
|
+
body,
|
|
715
|
+
settings,
|
|
716
|
+
wait_replicas=self.settings.queue_wait_replicas,
|
|
717
|
+
wait_ms=self.settings.queue_wait_ms,
|
|
718
|
+
)
|
|
719
|
+
break
|
|
720
|
+
except QueueFull as exc:
|
|
721
|
+
await self.requests.abandon(rid, caller=self.worker_id)
|
|
722
|
+
raise CallFailed("queue_full", str(exc), retry_after_secs=5) from exc
|
|
723
|
+
except NotReplicated as exc:
|
|
724
|
+
# A failed send (plan B23): the call runs direct now, under this same record
|
|
725
|
+
# (re-registering raced the unreplicated copy's claim into a conflict). Not
|
|
726
|
+
# re-sent: each try would add a copy; the copy meets a claimed job and stops.
|
|
727
|
+
logger.warning("send_not_replicated", request_id=rid, error=str(exc))
|
|
728
|
+
raise _SendNotReplicated from exc
|
|
729
|
+
except ResponseError as exc:
|
|
730
|
+
await self.requests.abandon(rid, caller=self.worker_id)
|
|
731
|
+
if isinstance(exc, OutOfMemoryError) or "OOM" in str(exc):
|
|
732
|
+
raise CallFailed("queue_unavailable", _refusal(exc)) from exc
|
|
733
|
+
raise
|
|
734
|
+
except STORE_ERRORS as exc:
|
|
735
|
+
if not self.queues_shared and loop.time() < give_up:
|
|
736
|
+
await asyncio.sleep(0.25)
|
|
737
|
+
continue
|
|
738
|
+
await self.requests.abandon(rid, caller=self.worker_id)
|
|
739
|
+
if self.queues_shared:
|
|
740
|
+
raise
|
|
741
|
+
raise _QueueDown from exc
|
|
742
|
+
except BaseException:
|
|
743
|
+
await self.requests.abandon(rid, caller=self.worker_id)
|
|
744
|
+
raise
|
|
745
|
+
try:
|
|
746
|
+
await self.requests.confirm(rid, keep_secs=keep_secs)
|
|
747
|
+
except STORE_ERRORS as exc: # the message is on the queue: never run it a second time
|
|
748
|
+
logger.warning("confirm_failed", request_id=rid, error=str(exc))
|
|
749
|
+
|
|
750
|
+
@property
|
|
751
|
+
def queues_shared(self) -> bool:
|
|
752
|
+
"""True when queues live in the state store (one store for both)."""
|
|
753
|
+
return self._queue_store is None
|
|
754
|
+
|
|
755
|
+
async def cancel(self, request_id: str) -> None:
|
|
756
|
+
"""Cancel a call: a waiting one never starts, a running one is stopped."""
|
|
757
|
+
await self.state.set(
|
|
758
|
+
self.keys.request(request_id, "cancel"), "1", ex=self.settings.status_ttl_secs
|
|
759
|
+
)
|
|
760
|
+
await self.state.publish(self.keys.cancel_channel, request_id)
|
|
761
|
+
|
|
762
|
+
async def _cancel_reader(self) -> None:
|
|
763
|
+
"""Stop running attempts whose call was cancelled (by any process)."""
|
|
764
|
+
subscribed = False
|
|
765
|
+
while True:
|
|
766
|
+
try:
|
|
767
|
+
if not subscribed:
|
|
768
|
+
await self._cancels.subscribe(self.keys.cancel_channel)
|
|
769
|
+
subscribed = True
|
|
770
|
+
message = await self._cancels.get_message(
|
|
771
|
+
ignore_subscribe_messages=True, timeout=1.0
|
|
772
|
+
)
|
|
773
|
+
except Exception as exc:
|
|
774
|
+
logger.warning("cancel_reader_error", error=str(exc))
|
|
775
|
+
subscribed = False
|
|
776
|
+
with contextlib.suppress(Exception):
|
|
777
|
+
await self._cancels.reset()
|
|
778
|
+
await asyncio.sleep(1)
|
|
779
|
+
continue
|
|
780
|
+
if not message:
|
|
781
|
+
continue
|
|
782
|
+
data = message["data"]
|
|
783
|
+
rid = data.decode() if isinstance(data, bytes) else str(data)
|
|
784
|
+
held = self._held.get(rid)
|
|
785
|
+
if held is not None and held.run is not None and not held.run.done():
|
|
786
|
+
held.cancelled = True
|
|
787
|
+
held.run.cancel()
|
|
788
|
+
|
|
789
|
+
async def _store_down(
|
|
790
|
+
self,
|
|
791
|
+
view: ResolvedRegistry,
|
|
792
|
+
alias: str,
|
|
793
|
+
envelope: Envelope,
|
|
794
|
+
exc: BaseException,
|
|
795
|
+
deadline: float,
|
|
796
|
+
) -> Reply:
|
|
797
|
+
"""The state store is unreachable: the model's `store_down` rule (plan §3.11).
|
|
798
|
+
|
|
799
|
+
`deadline` is the call's own (loop time, margin included): `pause` waits only until
|
|
800
|
+
then, never a fresh timeout from the moment the store failed.
|
|
801
|
+
"""
|
|
802
|
+
mode = view.value(alias, "store_down")
|
|
803
|
+
logger.warning("store_down", model=alias, mode=mode, error=str(exc))
|
|
804
|
+
if mode == "static_cap":
|
|
805
|
+
return await self._static_cap(view, alias, envelope)
|
|
806
|
+
if mode == "pause":
|
|
807
|
+
loop = asyncio.get_running_loop()
|
|
808
|
+
until = deadline - self.settings.caller_margin_secs
|
|
809
|
+
while loop.time() < until:
|
|
810
|
+
await asyncio.sleep(1.0)
|
|
811
|
+
try:
|
|
812
|
+
await self.state.ping()
|
|
813
|
+
except STORE_ERRORS:
|
|
814
|
+
continue
|
|
815
|
+
return await self.call(
|
|
816
|
+
alias,
|
|
817
|
+
envelope.params or {},
|
|
818
|
+
priority=envelope.priority,
|
|
819
|
+
timeout_secs=max(1.0, until - loop.time()),
|
|
820
|
+
request_id=envelope.request_id if envelope.app_request_id else None,
|
|
821
|
+
session_id=envelope.session_id,
|
|
822
|
+
user_id=envelope.user_id,
|
|
823
|
+
tenant_id=envelope.tenant_id,
|
|
824
|
+
)
|
|
825
|
+
raise CallFailed("queue_unavailable", "the state store is unreachable") from exc
|
|
826
|
+
|
|
827
|
+
async def _static_cap(self, view: ResolvedRegistry, alias: str, envelope: Envelope) -> Reply:
|
|
828
|
+
"""Run without the store: at most `local_limit_cap` calls per process and model."""
|
|
829
|
+
entry = view.registry.models[alias]
|
|
830
|
+
cap = self._static.setdefault(alias, asyncio.Semaphore(self.settings.local_limit_cap))
|
|
831
|
+
provider = self.providers.get(entry.provider)
|
|
832
|
+
error: BaseException | None = None
|
|
833
|
+
async with cap:
|
|
834
|
+
for dep in view.deployments(alias, "confirmed"):
|
|
835
|
+
if not view.value(alias, "enabled", dep.key) or view.outside_residency(dep):
|
|
836
|
+
continue
|
|
837
|
+
route = route_for(view, dep)
|
|
838
|
+
params = self._params(view, alias, envelope, dep.key)
|
|
839
|
+
timing = Timing(
|
|
840
|
+
float(view.value(alias, "attempt_timeout_secs", dep.key)),
|
|
841
|
+
float(view.value(alias, "stream_idle_secs")),
|
|
842
|
+
)
|
|
843
|
+
try:
|
|
844
|
+
result = await provider.call(envelope.op, params, route, timing)
|
|
845
|
+
except Exception as exc:
|
|
846
|
+
error = exc
|
|
847
|
+
continue
|
|
848
|
+
meta = Meta(served_model=alias, deployment=dep.key, attempts=1)
|
|
849
|
+
return Reply(
|
|
850
|
+
request_id=envelope.request_id, ok=True, response=result.response, meta=meta
|
|
851
|
+
)
|
|
852
|
+
raise CallFailed("provider_error", str(error or "no usable deployment"))
|
|
853
|
+
|
|
854
|
+
async def _direct(self, env: Envelope) -> None:
|
|
855
|
+
"""Run a call in this process (no queue), with the same admission and waiter order.
|
|
856
|
+
|
|
857
|
+
One waiter per process, model and class registers in the store; the others wait here.
|
|
858
|
+
"""
|
|
859
|
+
held = _Held(
|
|
860
|
+
message=None,
|
|
861
|
+
envelope=env,
|
|
862
|
+
first_seen=self.clock.now(),
|
|
863
|
+
position_ms=int(self.clock.now() * 1000),
|
|
864
|
+
tag=f"d{new_id()[-10:]}", # random tail (a v7 id starts with the time)
|
|
865
|
+
)
|
|
866
|
+
self._held[env.request_id] = held
|
|
867
|
+
alias = env.model
|
|
868
|
+
cls = "p" if env.priority else "n"
|
|
869
|
+
lock = self._direct_locks.setdefault((alias, cls), asyncio.Lock())
|
|
870
|
+
try:
|
|
871
|
+
async with lock:
|
|
872
|
+
decision = await self._direct_wait(held)
|
|
873
|
+
if decision is not None:
|
|
874
|
+
await self._run(held, decision, 1, alias)
|
|
875
|
+
finally:
|
|
876
|
+
if not held.grants:
|
|
877
|
+
self._drop(held)
|
|
878
|
+
|
|
879
|
+
async def _direct_wait(self, held: _Held) -> Decision | None:
|
|
880
|
+
"""Admission for a direct call until granted (the decision) or settled (None)."""
|
|
881
|
+
env = held.envelope
|
|
882
|
+
alias = env.model
|
|
883
|
+
while True:
|
|
884
|
+
view = self.registry.current
|
|
885
|
+
now = self.clock.now()
|
|
886
|
+
if now >= self._deadline(held):
|
|
887
|
+
await self._finish_call(
|
|
888
|
+
held, "rate_limited", "no budget before the deadline", final="expired"
|
|
889
|
+
)
|
|
890
|
+
return None
|
|
891
|
+
waited = now - held.position_ms / 1000.0
|
|
892
|
+
if env.max_wait_secs is not None and waited >= env.max_wait_secs:
|
|
893
|
+
await self._finish_call(
|
|
894
|
+
held, "rate_limited", "max_wait_secs passed", final="expired"
|
|
895
|
+
)
|
|
896
|
+
return None
|
|
897
|
+
decision = await self._decide(view, alias, held, 1, frozenset())
|
|
898
|
+
if decision is None:
|
|
899
|
+
return None
|
|
900
|
+
if decision.kind == "granted":
|
|
901
|
+
return decision
|
|
902
|
+
if await self._model_fallback(view, alias, held, 1, waited):
|
|
903
|
+
return None
|
|
904
|
+
await asyncio.sleep(min(max(decision.wait_secs, 0.01), 1.0))
|
|
905
|
+
|
|
906
|
+
def _rearm(self, rid: str, done: asyncio.Future[None]) -> asyncio.Future[None]:
|
|
907
|
+
"""The id's next wake-up after `done` fired: made by the first waiter to see it, shared.
|
|
908
|
+
|
|
909
|
+
Each waiter making its own replaced the others', so all but the last slept until a
|
|
910
|
+
recheck (two coroutines waiting on one id in one process).
|
|
911
|
+
"""
|
|
912
|
+
current = self._waiters.get(rid)
|
|
913
|
+
if current is None or current is done:
|
|
914
|
+
current = asyncio.get_running_loop().create_future()
|
|
915
|
+
self._waiters[rid] = current
|
|
916
|
+
return current
|
|
917
|
+
|
|
918
|
+
async def _wait_result(
|
|
919
|
+
self, rid: str, waiter: asyncio.Future[None], secs: float, *, recheck: bool = False
|
|
920
|
+
) -> Reply | None:
|
|
921
|
+
"""The reply; None (only with `recheck`) when the call's record disappeared."""
|
|
922
|
+
loop = asyncio.get_running_loop()
|
|
923
|
+
until = loop.time() + secs
|
|
924
|
+
if not recheck:
|
|
925
|
+
# Just sent: the reply cannot be there yet. Wait for the wake-up (or one poll)
|
|
926
|
+
# before the first read; reading at once cost a round trip on every call.
|
|
927
|
+
with contextlib.suppress(TimeoutError):
|
|
928
|
+
await asyncio.wait_for(asyncio.shield(waiter), min(secs, self.settings.poll_secs))
|
|
929
|
+
if waiter.done():
|
|
930
|
+
waiter = self._rearm(rid, waiter)
|
|
931
|
+
while True:
|
|
932
|
+
try:
|
|
933
|
+
reply = await self.requests.take_result(rid)
|
|
934
|
+
gone = reply is None and recheck and not await self.requests.job(rid)
|
|
935
|
+
except STORE_ERRORS as exc:
|
|
936
|
+
# The call is already sent or running: wait for the store, never re-run it
|
|
937
|
+
# on the store_down path (that ran it twice, once outside the budget).
|
|
938
|
+
if until - loop.time() <= 0:
|
|
939
|
+
raise CallFailed(
|
|
940
|
+
"deadline_exceeded", "the store did not answer in time", request_id=rid
|
|
941
|
+
) from exc
|
|
942
|
+
await asyncio.sleep(min(0.5, max(0.0, until - loop.time())))
|
|
943
|
+
continue
|
|
944
|
+
if gone:
|
|
945
|
+
return None
|
|
946
|
+
if reply is not None:
|
|
947
|
+
if reply.ok:
|
|
948
|
+
return reply
|
|
949
|
+
raise CallFailed(
|
|
950
|
+
reply.error or "provider_error",
|
|
951
|
+
reply.message,
|
|
952
|
+
retry_after_secs=reply.retry_after_secs,
|
|
953
|
+
partial_text=reply.partial_text,
|
|
954
|
+
request_id=rid,
|
|
955
|
+
)
|
|
956
|
+
left = until - loop.time()
|
|
957
|
+
if left <= 0:
|
|
958
|
+
raise CallFailed("deadline_exceeded", "no reply in time", request_id=rid)
|
|
959
|
+
# The wake-up comes by pub/sub (published with the result), or from this process's
|
|
960
|
+
# one result sweep (`_result_sweep`); a read per waiting caller made store load
|
|
961
|
+
# grow with the backlog (B56). The timeout is only a last-resort recheck.
|
|
962
|
+
with contextlib.suppress(TimeoutError):
|
|
963
|
+
await asyncio.wait_for(
|
|
964
|
+
asyncio.shield(waiter), min(left, self.settings.timings.result_recheck_secs)
|
|
965
|
+
)
|
|
966
|
+
if waiter.done():
|
|
967
|
+
waiter = self._rearm(rid, waiter)
|
|
968
|
+
|
|
969
|
+
async def _warm_clients(self) -> None:
|
|
970
|
+
"""Create every Bedrock deployment's client now, not on its first call.
|
|
971
|
+
|
|
972
|
+
A client takes about a second to build (measured: the first call of a fresh worker took
|
|
973
|
+
1,032 ms against ~140 ms after; B92 asks for <= 500 ms of first-call overhead).
|
|
974
|
+
Failures are ignored here: a call meets them again, and handles them.
|
|
975
|
+
"""
|
|
976
|
+
view = self.registry.current
|
|
977
|
+
for alias, entry in view.registry.models.items():
|
|
978
|
+
if entry.provider != "bedrock":
|
|
979
|
+
continue
|
|
980
|
+
for deployment in view.deployments(alias):
|
|
981
|
+
with contextlib.suppress(Exception):
|
|
982
|
+
await self.providers.pool.bedrock(route_for(view, deployment))
|
|
983
|
+
|
|
984
|
+
async def _result_sweep(self) -> None:
|
|
985
|
+
"""Every `poll_secs`, one store call finds results of this process's waiting callers.
|
|
986
|
+
|
|
987
|
+
The safety net behind the pub/sub wake-up: one script per process per period, however
|
|
988
|
+
many calls wait (a read per waiting caller grew with the backlog, plan B56).
|
|
989
|
+
"""
|
|
990
|
+
while True:
|
|
991
|
+
await asyncio.sleep(self.settings.poll_secs)
|
|
992
|
+
waiting = [rid for rid, w in self._waiters.items() if not w.done()]
|
|
993
|
+
for start in range(0, len(waiting), 500):
|
|
994
|
+
chunk = waiting[start : start + 500]
|
|
995
|
+
try:
|
|
996
|
+
found = await self._results_exist([], [self.keys.prefix, *chunk])
|
|
997
|
+
except Exception as exc: # the sweep must survive store hiccups
|
|
998
|
+
logger.warning("result_sweep_failed", error=str(exc))
|
|
999
|
+
break
|
|
1000
|
+
for index in found or ():
|
|
1001
|
+
waiter = self._waiters.get(chunk[int(index)])
|
|
1002
|
+
if waiter is not None and not waiter.done():
|
|
1003
|
+
waiter.set_result(None)
|
|
1004
|
+
|
|
1005
|
+
async def _result_reader(self) -> None:
|
|
1006
|
+
"""Wake waiting callers when their result is published (re-subscribes after errors)."""
|
|
1007
|
+
subscribed = False
|
|
1008
|
+
while True:
|
|
1009
|
+
try:
|
|
1010
|
+
if not subscribed:
|
|
1011
|
+
await self._pubsub.subscribe(self.keys.done_channel(self.worker_id))
|
|
1012
|
+
subscribed = True
|
|
1013
|
+
# Results published while unsubscribed were missed: every waiter re-reads.
|
|
1014
|
+
for pending in list(self._waiters.values()):
|
|
1015
|
+
if not pending.done():
|
|
1016
|
+
pending.set_result(None)
|
|
1017
|
+
message = await self._pubsub.get_message(
|
|
1018
|
+
ignore_subscribe_messages=True, timeout=1.0
|
|
1019
|
+
)
|
|
1020
|
+
except Exception as exc: # the reader must survive store hiccups
|
|
1021
|
+
logger.warning("result_reader_error", error=str(exc))
|
|
1022
|
+
subscribed = False
|
|
1023
|
+
with contextlib.suppress(Exception):
|
|
1024
|
+
await self._pubsub.reset()
|
|
1025
|
+
await asyncio.sleep(1)
|
|
1026
|
+
continue
|
|
1027
|
+
if not message:
|
|
1028
|
+
continue
|
|
1029
|
+
rid = (
|
|
1030
|
+
message["data"].decode()
|
|
1031
|
+
if isinstance(message["data"], bytes)
|
|
1032
|
+
else str(message["data"])
|
|
1033
|
+
)
|
|
1034
|
+
waiter = self._waiters.get(rid)
|
|
1035
|
+
if waiter is not None and not waiter.done():
|
|
1036
|
+
waiter.set_result(None)
|
|
1037
|
+
|
|
1038
|
+
# ================================================================ worker: receiving
|
|
1039
|
+
|
|
1040
|
+
async def _model_loop(self, alias: str) -> None:
|
|
1041
|
+
"""Receive this model's calls (priority first) and admit them in order.
|
|
1042
|
+
|
|
1043
|
+
Idle, it blocks on its queues' doorbell (a send wakes one worker at once) for 50 ms at a
|
|
1044
|
+
time, then 1 s after a second with nothing: polling idle loops made ~96 receive
|
|
1045
|
+
scripts/s per pod with 4 models (measured); a busy loop never waits.
|
|
1046
|
+
"""
|
|
1047
|
+
empty = 0
|
|
1048
|
+
wake = self._wake.setdefault(alias, asyncio.Event())
|
|
1049
|
+
while not self._stopping:
|
|
1050
|
+
if self._paused:
|
|
1051
|
+
await asyncio.sleep(0.2)
|
|
1052
|
+
continue
|
|
1053
|
+
try:
|
|
1054
|
+
got = await self._receive_once(alias)
|
|
1055
|
+
except Exception as exc: # a store blip must not end the loop
|
|
1056
|
+
logger.warning("receive_failed", model=alias, error=str(exc))
|
|
1057
|
+
if isinstance(exc, STORE_ERRORS):
|
|
1058
|
+
# Not only the heartbeat (every lease/3): a block between two beats went
|
|
1059
|
+
# unseen, and callers whose keys lapsed in it were dropped as gone (soak).
|
|
1060
|
+
self._store_trouble_at = time.monotonic()
|
|
1061
|
+
await asyncio.sleep(1)
|
|
1062
|
+
continue
|
|
1063
|
+
held = (alias, "p") in self._held_class or (alias, "n") in self._held_class
|
|
1064
|
+
# Idle only when the queues were empty, not when a class waits on its held call.
|
|
1065
|
+
empty = 0 if got or held else empty + 1
|
|
1066
|
+
if not got:
|
|
1067
|
+
pace = self.settings.timings
|
|
1068
|
+
pause = (
|
|
1069
|
+
pace.idle_poll_secs if empty < pace.idle_after_empty else pace.idle_slow_secs
|
|
1070
|
+
)
|
|
1071
|
+
if held: # a held call ending (granted or settled) wakes it at once
|
|
1072
|
+
with contextlib.suppress(TimeoutError):
|
|
1073
|
+
await asyncio.wait_for(wake.wait(), pause)
|
|
1074
|
+
else: # a send rings the doorbell: one idle worker wakes per call
|
|
1075
|
+
view = self.registry.current
|
|
1076
|
+
names = [self._queue_name(view, alias, p) for p in (True, False)]
|
|
1077
|
+
try:
|
|
1078
|
+
await self.queues.wait_bell(names, pause)
|
|
1079
|
+
except STORE_ERRORS:
|
|
1080
|
+
self._store_trouble_at = time.monotonic() # (was swallowed unseen)
|
|
1081
|
+
wake.clear()
|
|
1082
|
+
|
|
1083
|
+
async def _receive_once(self, alias: str) -> bool:
|
|
1084
|
+
view = self.registry.current
|
|
1085
|
+
if alias not in view.registry.models:
|
|
1086
|
+
await asyncio.sleep(1)
|
|
1087
|
+
return False
|
|
1088
|
+
slots = self._slots[alias]
|
|
1089
|
+
free = slots._value # free worker slots of this model, read only
|
|
1090
|
+
if free <= 0:
|
|
1091
|
+
await asyncio.sleep(0.02)
|
|
1092
|
+
return False
|
|
1093
|
+
settings = self._queue_settings(view, alias)
|
|
1094
|
+
got_any = False
|
|
1095
|
+
for priority in (True, False):
|
|
1096
|
+
cls = "p" if priority else "n"
|
|
1097
|
+
if (alias, cls) in self._held_class:
|
|
1098
|
+
continue
|
|
1099
|
+
name = self._queue_name(view, alias, priority)
|
|
1100
|
+
messages, dead = await self.queues.receive(
|
|
1101
|
+
name, min(free, self.settings.receive_batch), settings
|
|
1102
|
+
)
|
|
1103
|
+
for letter in dead:
|
|
1104
|
+
await self._dead_lettered(letter.body)
|
|
1105
|
+
if not messages:
|
|
1106
|
+
continue
|
|
1107
|
+
got_any = True
|
|
1108
|
+
for index, message in enumerate(messages):
|
|
1109
|
+
try:
|
|
1110
|
+
held = await self._admit_new(alias, message)
|
|
1111
|
+
except Exception:
|
|
1112
|
+
for rest in messages[index + 1 :]: # not seen: back to the queue now
|
|
1113
|
+
with contextlib.suppress(Exception):
|
|
1114
|
+
await self.queues.return_(rest)
|
|
1115
|
+
raise
|
|
1116
|
+
if held:
|
|
1117
|
+
for rest in messages[index + 1 :]:
|
|
1118
|
+
await self.queues.return_(rest)
|
|
1119
|
+
break
|
|
1120
|
+
if got_any:
|
|
1121
|
+
break
|
|
1122
|
+
return got_any
|
|
1123
|
+
|
|
1124
|
+
def _queue_settings(self, view: ResolvedRegistry, alias: str) -> QueueSettings:
|
|
1125
|
+
queue = view.value(alias, "queue")
|
|
1126
|
+
return QueueSettings(
|
|
1127
|
+
visibility_secs=float(queue["visibility_secs"]),
|
|
1128
|
+
max_receives=int(queue["max_receives"]),
|
|
1129
|
+
retention_secs=int(queue["retention_secs"]),
|
|
1130
|
+
dlq_retention_secs=int(queue["dlq_retention_secs"]),
|
|
1131
|
+
max_waiting=int(view.value(alias, "max_waiting")),
|
|
1132
|
+
)
|
|
1133
|
+
|
|
1134
|
+
async def _dead_lettered(self, body: bytes) -> None:
|
|
1135
|
+
try:
|
|
1136
|
+
env = parse(body)
|
|
1137
|
+
except Exception:
|
|
1138
|
+
return
|
|
1139
|
+
await self._reply(env, "dead_lettered", "the call failed too many times", final="failed")
|
|
1140
|
+
|
|
1141
|
+
async def _admit_new(self, alias: str, message: Message) -> bool:
|
|
1142
|
+
"""Admit a newly received message; True when it is now held (its class stops)."""
|
|
1143
|
+
try:
|
|
1144
|
+
env = parse(message.body)
|
|
1145
|
+
except UnsupportedEnvelopeError:
|
|
1146
|
+
await self.queues.return_(message, delay_secs=5)
|
|
1147
|
+
return False
|
|
1148
|
+
except Exception as exc:
|
|
1149
|
+
logger.warning("bad_message", queue=message.queue, error=str(exc))
|
|
1150
|
+
await self.queues.fail(message, QueueSettings())
|
|
1151
|
+
return False
|
|
1152
|
+
held = _Held(
|
|
1153
|
+
message,
|
|
1154
|
+
env,
|
|
1155
|
+
first_seen=self.clock.now(),
|
|
1156
|
+
position_ms=message.position_ms,
|
|
1157
|
+
tag=message.id,
|
|
1158
|
+
)
|
|
1159
|
+
self._held[env.request_id] = held
|
|
1160
|
+
try:
|
|
1161
|
+
if not await self._caller_alive(env):
|
|
1162
|
+
# A warning: a dropped call is worth seeing (the soak and scenario S5 found
|
|
1163
|
+
# calls dropped wrongly, and INFO logs alone did not show it).
|
|
1164
|
+
logger.warning("caller_gone", request_id=env.request_id, model=env.model)
|
|
1165
|
+
await self._finish_call(held, "cancelled", "caller_gone", final="cancelled")
|
|
1166
|
+
return False
|
|
1167
|
+
return await self._try(held, attempt=1)
|
|
1168
|
+
except Exception:
|
|
1169
|
+
await self._release(held)
|
|
1170
|
+
raise
|
|
1171
|
+
|
|
1172
|
+
async def _caller_alive(self, env: Envelope) -> bool:
|
|
1173
|
+
"""False when every caller of this call is gone (plan §3.18 R24).
|
|
1174
|
+
|
|
1175
|
+
After a store outage the callers' liveness keys may have expired only because nobody
|
|
1176
|
+
could renew them: for a caller TTL and a heartbeat after this process last saw the
|
|
1177
|
+
store fail, callers count as alive (B95: a 45 s block dropped 429 of 500 calls).
|
|
1178
|
+
|
|
1179
|
+
Judged from this process's own heartbeat too: a half-open store hung every command for
|
|
1180
|
+
the read timeout, the doorbell wait swallowed its error, and calls were dropped the
|
|
1181
|
+
moment the store came back (scenario S5). A heartbeat that is late or just recovered
|
|
1182
|
+
from a gap means the store was out of reach here as well, so a lapsed key proves
|
|
1183
|
+
nothing.
|
|
1184
|
+
"""
|
|
1185
|
+
beat = self.settings.lease_secs / 3
|
|
1186
|
+
grace = self.settings.caller_ttl_secs + beat + 5
|
|
1187
|
+
now = time.monotonic()
|
|
1188
|
+
if now - self._store_trouble_at < grace or now - self._beat_ok_at > 2 * beat + 1:
|
|
1189
|
+
return True
|
|
1190
|
+
try:
|
|
1191
|
+
return await self.requests.callers_alive(
|
|
1192
|
+
env.request_id, prefix=self.keys.prefix, caller_ttl=self.settings.caller_ttl_secs
|
|
1193
|
+
)
|
|
1194
|
+
except STORE_ERRORS:
|
|
1195
|
+
self._store_trouble_at = time.monotonic()
|
|
1196
|
+
return True
|
|
1197
|
+
|
|
1198
|
+
# ================================================================ worker: admission
|
|
1199
|
+
|
|
1200
|
+
def _deadline(self, held: _Held) -> float:
|
|
1201
|
+
return held.position_ms / 1000.0 + held.envelope.timeout_secs
|
|
1202
|
+
|
|
1203
|
+
def _candidates(
|
|
1204
|
+
self,
|
|
1205
|
+
view: ResolvedRegistry,
|
|
1206
|
+
alias: str,
|
|
1207
|
+
env: Envelope,
|
|
1208
|
+
age: float,
|
|
1209
|
+
failed: frozenset[str],
|
|
1210
|
+
cause: str,
|
|
1211
|
+
) -> list[Candidate]:
|
|
1212
|
+
deps: list[Deployment] = list(view.deployments(alias))
|
|
1213
|
+
if env.allowed_regions:
|
|
1214
|
+
deps = [d for d in deps if d.region in env.allowed_regions]
|
|
1215
|
+
if env.allowed_accounts:
|
|
1216
|
+
deps = [d for d in deps if d.account in env.allowed_accounts]
|
|
1217
|
+
fallback_ok = self._may_spill(view, alias, env, age, cause, "fallback")
|
|
1218
|
+
now = time.monotonic()
|
|
1219
|
+
deps = [
|
|
1220
|
+
d for d in deps if self._bad_accounts.get((d.account, d.region), (0.0, 0.0))[0] <= now
|
|
1221
|
+
]
|
|
1222
|
+
out = []
|
|
1223
|
+
for dep in deps:
|
|
1224
|
+
if dep.group == "fallback" and not fallback_ok:
|
|
1225
|
+
continue
|
|
1226
|
+
out.append(Candidate(dep.key, dep.group, last=dep.key in failed)) # type: ignore[arg-type]
|
|
1227
|
+
return out
|
|
1228
|
+
|
|
1229
|
+
def _ctx(self, alias: str, held: _Held, waited: float) -> hooks.CallContext:
|
|
1230
|
+
env = held.envelope
|
|
1231
|
+
return hooks.CallContext(
|
|
1232
|
+
alias, env.request_id, env.priority, env.session_id, env.user_id, env.tenant_id, waited
|
|
1233
|
+
)
|
|
1234
|
+
|
|
1235
|
+
def _may_spill(
|
|
1236
|
+
self,
|
|
1237
|
+
view: ResolvedRegistry,
|
|
1238
|
+
alias: str,
|
|
1239
|
+
env: Envelope,
|
|
1240
|
+
age: float,
|
|
1241
|
+
cause: str,
|
|
1242
|
+
rule_name: str,
|
|
1243
|
+
) -> bool:
|
|
1244
|
+
"""Whether a call may use fallback regions / models now: rule or `fallback_policy`."""
|
|
1245
|
+
allowed = ("all", "regions") if rule_name == "fallback" else ("all",)
|
|
1246
|
+
rule = view.value(alias, rule_name)
|
|
1247
|
+
if env.allow_fallback not in allowed or not rule:
|
|
1248
|
+
return False
|
|
1249
|
+
policy_name = view.value(alias, "fallback_policy")
|
|
1250
|
+
policy = hooks.get("fallback_policy", policy_name)
|
|
1251
|
+
if policy is not None:
|
|
1252
|
+
ctx = hooks.CallContext(
|
|
1253
|
+
alias, env.request_id, env.priority, env.session_id, env.user_id, env.tenant_id, age
|
|
1254
|
+
)
|
|
1255
|
+
default = cause in rule["on"] and age >= float(rule["after_secs"])
|
|
1256
|
+
return bool(
|
|
1257
|
+
hooks.call_safely(
|
|
1258
|
+
"fallback_policy", policy_name, policy, cause, age, rule, ctx, default=default
|
|
1259
|
+
)
|
|
1260
|
+
)
|
|
1261
|
+
return cause in rule["on"] and age >= float(rule["after_secs"])
|
|
1262
|
+
|
|
1263
|
+
def _ask(
|
|
1264
|
+
self,
|
|
1265
|
+
view: ResolvedRegistry,
|
|
1266
|
+
alias: str,
|
|
1267
|
+
held: _Held,
|
|
1268
|
+
attempt: int,
|
|
1269
|
+
failed: frozenset[str],
|
|
1270
|
+
*,
|
|
1271
|
+
foreign_from: str | None = None,
|
|
1272
|
+
require_pct: int = 0,
|
|
1273
|
+
) -> Ask | None:
|
|
1274
|
+
env = held.envelope
|
|
1275
|
+
age = self.clock.now() - held.position_ms / 1000.0
|
|
1276
|
+
cause = "error" if failed else ("broken" if held.broken else "exhausted")
|
|
1277
|
+
candidates = self._candidates(view, alias, env, age, failed, cause)
|
|
1278
|
+
if not candidates:
|
|
1279
|
+
return None
|
|
1280
|
+
entry = view.registry.models[alias]
|
|
1281
|
+
estimate = self._charged(view, alias, held)
|
|
1282
|
+
strategy = view.value(alias, "balance")
|
|
1283
|
+
if env.priority and view.value(alias, "balance_priority"):
|
|
1284
|
+
strategy = view.value(alias, "balance_priority")
|
|
1285
|
+
if str(strategy).startswith("custom:"):
|
|
1286
|
+
candidates = self._custom_order(view, alias, held, candidates, str(strategy)[7:], age)
|
|
1287
|
+
strategy = "ordered"
|
|
1288
|
+
sticky = view.value(alias, "sticky")
|
|
1289
|
+
sticky_hash = None
|
|
1290
|
+
if sticky and entry.type == "chat":
|
|
1291
|
+
key: str | None = env.session_id
|
|
1292
|
+
if str(sticky["key"]).startswith("custom:"):
|
|
1293
|
+
name = str(sticky["key"])[7:]
|
|
1294
|
+
fn = hooks.get("sticky_key", name)
|
|
1295
|
+
ctx = self._ctx(alias, held, age)
|
|
1296
|
+
key = hooks.call_safely("sticky_key", name, fn, ctx, default=None) if fn else None
|
|
1297
|
+
if key:
|
|
1298
|
+
sticky_hash = self.hash_id(str(key))
|
|
1299
|
+
return Ask(
|
|
1300
|
+
alias=alias,
|
|
1301
|
+
request_id=env.request_id,
|
|
1302
|
+
worker=self.worker_id,
|
|
1303
|
+
attempt=attempt,
|
|
1304
|
+
cls="p" if env.priority else "n",
|
|
1305
|
+
position=held.position,
|
|
1306
|
+
candidates=candidates,
|
|
1307
|
+
strategy=str(strategy),
|
|
1308
|
+
pick=str((view.value(alias, "balance_opts") or {}).get("pick", "percent")),
|
|
1309
|
+
cost_r=1,
|
|
1310
|
+
cost_t=estimate.total,
|
|
1311
|
+
claim=foreign_from is None,
|
|
1312
|
+
sticky_hash=sticky_hash,
|
|
1313
|
+
sticky_ttl_secs=float(sticky["idle_ttl_secs"]) if sticky else 1800.0,
|
|
1314
|
+
sticky_move_secs=float(view.value(alias, "sticky_move_after_secs")),
|
|
1315
|
+
foreign_from=foreign_from,
|
|
1316
|
+
require_pct=require_pct,
|
|
1317
|
+
tag=held.tag,
|
|
1318
|
+
)
|
|
1319
|
+
|
|
1320
|
+
def _custom_order(
|
|
1321
|
+
self,
|
|
1322
|
+
view: ResolvedRegistry,
|
|
1323
|
+
alias: str,
|
|
1324
|
+
held: _Held,
|
|
1325
|
+
candidates: list[Candidate],
|
|
1326
|
+
name: str,
|
|
1327
|
+
age: float,
|
|
1328
|
+
) -> list[Candidate]:
|
|
1329
|
+
"""Candidates in the order a custom balancer prefers (unknown keys ignored)."""
|
|
1330
|
+
fn = hooks.get("balancer", name)
|
|
1331
|
+
if fn is None:
|
|
1332
|
+
return candidates
|
|
1333
|
+
deps = {d.key: d for d in view.deployments(alias)}
|
|
1334
|
+
views = [
|
|
1335
|
+
hooks.DeploymentView(
|
|
1336
|
+
key=c.key,
|
|
1337
|
+
account=deps[c.key].account,
|
|
1338
|
+
region=deps[c.key].region,
|
|
1339
|
+
group=c.tier,
|
|
1340
|
+
rpm=view.value(alias, "rpm", c.key),
|
|
1341
|
+
tpm=view.value(alias, "tpm", c.key),
|
|
1342
|
+
weight=float(view.value(alias, "weight", c.key)),
|
|
1343
|
+
tags=view.value(alias, "tags", c.key) or {},
|
|
1344
|
+
)
|
|
1345
|
+
for c in candidates
|
|
1346
|
+
]
|
|
1347
|
+
ctx = self._ctx(alias, held, age)
|
|
1348
|
+
order = hooks.call_safely("balancer", name, fn, views, ctx, default=None)
|
|
1349
|
+
if not order:
|
|
1350
|
+
return candidates
|
|
1351
|
+
by_key = {c.key: c for c in candidates}
|
|
1352
|
+
ranked = [by_key[k] for k in order if k in by_key]
|
|
1353
|
+
return ranked + [c for c in candidates if c.key not in set(order)]
|
|
1354
|
+
|
|
1355
|
+
def _owner(self, held: _Held, attempt: int) -> str:
|
|
1356
|
+
"""Owner token of an attempt; the message id tells two deliveries of one id apart."""
|
|
1357
|
+
worker = f"{self.worker_id}@{held.tag}"
|
|
1358
|
+
return owner_token(held.envelope.request_id, worker, attempt)
|
|
1359
|
+
|
|
1360
|
+
def _params(
|
|
1361
|
+
self, view: ResolvedRegistry, alias: str, env: Envelope, dep: str | None
|
|
1362
|
+
) -> dict[str, Any]:
|
|
1363
|
+
entry = view.registry.models[alias]
|
|
1364
|
+
request = inference.apply(
|
|
1365
|
+
entry.provider,
|
|
1366
|
+
env.op,
|
|
1367
|
+
env.params or {},
|
|
1368
|
+
view.value(alias, "inference", dep) or {},
|
|
1369
|
+
view.value(alias, "inference_limits", dep) or {},
|
|
1370
|
+
)
|
|
1371
|
+
extra = view.value(alias, "params", dep) if dep else None
|
|
1372
|
+
return inference.merge_params(request, extra or {})
|
|
1373
|
+
|
|
1374
|
+
def _charged(self, view: ResolvedRegistry, alias: str, held: _Held) -> tokens.Estimate:
|
|
1375
|
+
"""The token estimate admission charges: from model-level params, cached per view."""
|
|
1376
|
+
cached = held.charged
|
|
1377
|
+
if cached is not None and cached[0] is view and cached[1] == alias:
|
|
1378
|
+
return cached[2]
|
|
1379
|
+
params = self._params(view, alias, held.envelope, None)
|
|
1380
|
+
estimate = self._estimate(view, alias, held.envelope, params)
|
|
1381
|
+
held.charged = (view, alias, estimate)
|
|
1382
|
+
return estimate
|
|
1383
|
+
|
|
1384
|
+
def _estimate(
|
|
1385
|
+
self, view: ResolvedRegistry, alias: str, env: Envelope, params: Mapping[str, Any]
|
|
1386
|
+
) -> tokens.Estimate:
|
|
1387
|
+
entry = view.registry.models[alias]
|
|
1388
|
+
return tokens.estimate(
|
|
1389
|
+
env.op,
|
|
1390
|
+
params,
|
|
1391
|
+
embedding=entry.type == "embedding",
|
|
1392
|
+
default_output_tokens=int(view.value(alias, "default_output_tokens")),
|
|
1393
|
+
output_weight=float(view.value(alias, "tpm_output_weight")),
|
|
1394
|
+
estimator=view.value(alias, "token_estimator"),
|
|
1395
|
+
)
|
|
1396
|
+
|
|
1397
|
+
async def _try(self, held: _Held, attempt: int, failed: frozenset[str] = frozenset()) -> bool:
|
|
1398
|
+
"""Admit (or re-admit) a held call; start it when granted. True when it stays held."""
|
|
1399
|
+
view = self.registry.current
|
|
1400
|
+
env = held.envelope
|
|
1401
|
+
alias = env.model
|
|
1402
|
+
now = self.clock.now()
|
|
1403
|
+
deadline = self._deadline(held)
|
|
1404
|
+
if now >= deadline:
|
|
1405
|
+
await self._finish_call(
|
|
1406
|
+
held, "rate_limited", "no budget before the deadline", final="expired"
|
|
1407
|
+
)
|
|
1408
|
+
return False
|
|
1409
|
+
max_wait = env.max_wait_secs
|
|
1410
|
+
if max_wait is not None and attempt == 1 and now - held.position_ms / 1000.0 >= max_wait:
|
|
1411
|
+
# It already waited its limit in the queue, unreceived: refuse it now rather than
|
|
1412
|
+
# start it with no time left (it ran past its deadline and the caller gave up).
|
|
1413
|
+
await self._finish_call(
|
|
1414
|
+
held, "rate_limited", "max_wait_secs passed", final="expired", retry_after=1.0
|
|
1415
|
+
)
|
|
1416
|
+
return False
|
|
1417
|
+
decision = await self._decide(view, alias, held, attempt, failed)
|
|
1418
|
+
if decision is None:
|
|
1419
|
+
return False
|
|
1420
|
+
if decision.kind == "granted":
|
|
1421
|
+
self._start_run(held, decision, attempt)
|
|
1422
|
+
return False
|
|
1423
|
+
if decision.kind == "wait":
|
|
1424
|
+
max_wait = env.max_wait_secs
|
|
1425
|
+
waited = now - held.position_ms / 1000.0
|
|
1426
|
+
if max_wait is not None and attempt == 1 and waited >= max_wait:
|
|
1427
|
+
await self._finish_call(
|
|
1428
|
+
held,
|
|
1429
|
+
"rate_limited",
|
|
1430
|
+
"max_wait_secs passed",
|
|
1431
|
+
final="expired",
|
|
1432
|
+
retry_after=decision.wait_secs,
|
|
1433
|
+
)
|
|
1434
|
+
return False
|
|
1435
|
+
fallen = await self._model_fallback(view, alias, held, attempt, waited)
|
|
1436
|
+
if fallen:
|
|
1437
|
+
return False
|
|
1438
|
+
if attempt == 1:
|
|
1439
|
+
cls = "p" if env.priority else "n"
|
|
1440
|
+
self._held_class.add((alias, cls))
|
|
1441
|
+
task = asyncio.create_task(self._hold(held, decision.wait_secs))
|
|
1442
|
+
self._running.add(task)
|
|
1443
|
+
task.add_done_callback(self._running.discard)
|
|
1444
|
+
return True
|
|
1445
|
+
await asyncio.sleep(min(decision.wait_secs, 1.0))
|
|
1446
|
+
return await self._try(held, attempt, failed)
|
|
1447
|
+
return False
|
|
1448
|
+
|
|
1449
|
+
async def _decide(
|
|
1450
|
+
self,
|
|
1451
|
+
view: ResolvedRegistry,
|
|
1452
|
+
alias: str,
|
|
1453
|
+
held: _Held,
|
|
1454
|
+
attempt: int,
|
|
1455
|
+
failed: frozenset[str],
|
|
1456
|
+
) -> Decision | None:
|
|
1457
|
+
"""One admission call and the terminal outcomes; None when the call was settled."""
|
|
1458
|
+
ask = self._ask(view, alias, held, attempt, failed)
|
|
1459
|
+
if ask is None:
|
|
1460
|
+
if self._bad_accounts and any(
|
|
1461
|
+
(d.account, d.region) in self._bad_accounts for d in view.deployments(alias)
|
|
1462
|
+
):
|
|
1463
|
+
await self._finish_call(
|
|
1464
|
+
held,
|
|
1465
|
+
"credentials_unavailable",
|
|
1466
|
+
"no account with working credentials",
|
|
1467
|
+
retry_after=60,
|
|
1468
|
+
)
|
|
1469
|
+
else:
|
|
1470
|
+
await self._finish_call(
|
|
1471
|
+
held, "no_deployment", "the call's filter leaves no deployment"
|
|
1472
|
+
)
|
|
1473
|
+
return None
|
|
1474
|
+
estimate = self._charged(view, alias, held)
|
|
1475
|
+
tpm = [view.value(alias, "tpm", c.key) for c in ask.candidates]
|
|
1476
|
+
if all(isinstance(v, int) for v in tpm) and tpm and estimate.input > max(tpm):
|
|
1477
|
+
await self._finish_call(held, "request_too_large", "input above the token budget")
|
|
1478
|
+
return None
|
|
1479
|
+
decision = await self.admission.pick_take(ask, self.registry.changes)
|
|
1480
|
+
if decision.kind == "wait":
|
|
1481
|
+
self.metrics.count("llmq_admission_refusals_total", model=alias, reason=decision.reason)
|
|
1482
|
+
if decision.kind == "stale_config":
|
|
1483
|
+
await self.registry.load()
|
|
1484
|
+
decision = await self.admission.pick_take(ask, self.registry.changes)
|
|
1485
|
+
if decision.kind == "no_config": # the store lost the numbers: restore, then once more
|
|
1486
|
+
await self.registry.refresh()
|
|
1487
|
+
decision = await self.admission.pick_take(ask, self.registry.changes)
|
|
1488
|
+
if decision.kind in ("no_config", "stale_config"):
|
|
1489
|
+
# Still no usable numbers. Before, the call stayed held and hidden for good.
|
|
1490
|
+
logger.error("admission_config_missing", model=alias, kind=decision.kind)
|
|
1491
|
+
if held.message is None:
|
|
1492
|
+
await self._release(held)
|
|
1493
|
+
raise CallFailed("queue_unavailable", f"{alias}: admission numbers are missing")
|
|
1494
|
+
await self._release(held, delay_secs=5.0)
|
|
1495
|
+
return None
|
|
1496
|
+
if decision.kind in ("done", "cancelled"):
|
|
1497
|
+
if decision.kind == "cancelled":
|
|
1498
|
+
await self._finish_call(held, "cancelled", "cancelled", final="cancelled")
|
|
1499
|
+
else:
|
|
1500
|
+
await self._ack(held)
|
|
1501
|
+
return None
|
|
1502
|
+
if decision.kind == "duplicate":
|
|
1503
|
+
self._drop(held)
|
|
1504
|
+
if held.message is not None:
|
|
1505
|
+
await self.queues.return_(held.message, delay_secs=self.settings.lease_secs / 2)
|
|
1506
|
+
return None
|
|
1507
|
+
if decision.kind == "none":
|
|
1508
|
+
has_fallback = any(d.group == "fallback" for d in view.deployments(alias))
|
|
1509
|
+
rule = view.value(alias, "fallback")
|
|
1510
|
+
if not held.broken and has_fallback and rule and "broken" in rule["on"]:
|
|
1511
|
+
held.broken = True
|
|
1512
|
+
age = self.clock.now() - held.position_ms / 1000.0
|
|
1513
|
+
if age < float(rule["after_secs"]):
|
|
1514
|
+
return Decision(
|
|
1515
|
+
"wait", wait_secs=float(rule["after_secs"]) - age, reason="broken"
|
|
1516
|
+
)
|
|
1517
|
+
return await self._decide(view, alias, held, attempt, failed)
|
|
1518
|
+
fell = await self._model_fallback(view, alias, held, attempt, 1e9, broken=True)
|
|
1519
|
+
if not fell:
|
|
1520
|
+
due = await self.health.disabled(alias)
|
|
1521
|
+
wait = max(1.0, min(due.values()) - self.clock.now()) if due else 60.0
|
|
1522
|
+
# Why no deployment was usable, per deployment: an operator needs to tell an
|
|
1523
|
+
# open breaker from a disabled deployment (the bare line said neither).
|
|
1524
|
+
why: dict[str, str] = {}
|
|
1525
|
+
for dep in view.deployments(alias):
|
|
1526
|
+
with contextlib.suppress(*STORE_ERRORS):
|
|
1527
|
+
cb = await self.state.hgetall(self.keys.model(alias, f"cb:{dep.key}"))
|
|
1528
|
+
raw = (cb or {}).get(b"state", b"closed")
|
|
1529
|
+
state = raw.decode() if isinstance(raw, bytes) else str(raw)
|
|
1530
|
+
disabled = " (disabled)" if dep.key in due else ""
|
|
1531
|
+
why[dep.key] = f"breaker {state}{disabled}"
|
|
1532
|
+
logger.error("model_unavailable", model=alias, deployments=why)
|
|
1533
|
+
await self._finish_call(
|
|
1534
|
+
held, "model_unavailable", "no usable deployment", retry_after=wait
|
|
1535
|
+
)
|
|
1536
|
+
return None
|
|
1537
|
+
return decision
|
|
1538
|
+
|
|
1539
|
+
async def _hold(self, held: _Held, wait: float) -> None:
|
|
1540
|
+
"""Keep a refused call: retry admission until granted, settled or expired."""
|
|
1541
|
+
alias = held.envelope.model
|
|
1542
|
+
cls = "p" if held.envelope.priority else "n"
|
|
1543
|
+
try:
|
|
1544
|
+
while not self._stopping:
|
|
1545
|
+
await asyncio.sleep(min(max(wait, 0.01), 1.0))
|
|
1546
|
+
view = self.registry.current
|
|
1547
|
+
if held.message is not None:
|
|
1548
|
+
await self.queues.extend(
|
|
1549
|
+
held.message, float(view.value(alias, "queue")["visibility_secs"])
|
|
1550
|
+
)
|
|
1551
|
+
now = self.clock.now()
|
|
1552
|
+
if now >= self._deadline(held):
|
|
1553
|
+
await self._finish_call(
|
|
1554
|
+
held, "rate_limited", "no budget before the deadline", final="expired"
|
|
1555
|
+
)
|
|
1556
|
+
return
|
|
1557
|
+
waited = now - held.position_ms / 1000.0
|
|
1558
|
+
if (
|
|
1559
|
+
held.envelope.max_wait_secs is not None
|
|
1560
|
+
and waited >= held.envelope.max_wait_secs
|
|
1561
|
+
):
|
|
1562
|
+
await self._finish_call(
|
|
1563
|
+
held,
|
|
1564
|
+
"rate_limited",
|
|
1565
|
+
"max_wait_secs passed",
|
|
1566
|
+
final="expired",
|
|
1567
|
+
retry_after=wait,
|
|
1568
|
+
)
|
|
1569
|
+
return
|
|
1570
|
+
if await self._model_fallback(view, alias, held, 1, waited):
|
|
1571
|
+
return
|
|
1572
|
+
decision = await self._decide(view, alias, held, 1, frozenset())
|
|
1573
|
+
if decision is None:
|
|
1574
|
+
return
|
|
1575
|
+
if decision.kind == "granted":
|
|
1576
|
+
self._start_run(held, decision, 1)
|
|
1577
|
+
return
|
|
1578
|
+
wait = decision.wait_secs
|
|
1579
|
+
except Exception as exc: # never leave the call held with nobody working on it
|
|
1580
|
+
if isinstance(exc, STORE_ERRORS):
|
|
1581
|
+
self._store_trouble_at = time.monotonic()
|
|
1582
|
+
logger.warning("hold_failed", request_id=held.envelope.request_id, error=str(exc))
|
|
1583
|
+
await self._release(held)
|
|
1584
|
+
finally:
|
|
1585
|
+
self._held_class.discard((alias, cls))
|
|
1586
|
+
if (event := self._wake.get(alias)) is not None:
|
|
1587
|
+
event.set() # the class is free: receive its next call now
|
|
1588
|
+
|
|
1589
|
+
async def _model_fallback(
|
|
1590
|
+
self,
|
|
1591
|
+
view: ResolvedRegistry,
|
|
1592
|
+
alias: str,
|
|
1593
|
+
held: _Held,
|
|
1594
|
+
attempt: int,
|
|
1595
|
+
waited: float,
|
|
1596
|
+
*,
|
|
1597
|
+
broken: bool = False,
|
|
1598
|
+
) -> bool:
|
|
1599
|
+
"""Try the model's fallback models when its rule allows; True when one granted."""
|
|
1600
|
+
rule = view.value(alias, "model_fallback")
|
|
1601
|
+
env = held.envelope
|
|
1602
|
+
cause = "broken" if broken else "exhausted"
|
|
1603
|
+
if not self._may_spill(view, alias, env, waited, cause, "model_fallback"):
|
|
1604
|
+
return False
|
|
1605
|
+
for item in rule["models"]:
|
|
1606
|
+
other = item["model"]
|
|
1607
|
+
if other not in view.registry.models:
|
|
1608
|
+
continue
|
|
1609
|
+
pct = int(view.value(other, "require_headroom_pct")) if item["require_headroom"] else 0
|
|
1610
|
+
ask = self._ask(
|
|
1611
|
+
view, other, held, attempt, frozenset(), foreign_from=alias, require_pct=pct
|
|
1612
|
+
)
|
|
1613
|
+
if ask is None:
|
|
1614
|
+
continue
|
|
1615
|
+
decision = await self.admission.pick_take(ask, self.registry.changes)
|
|
1616
|
+
if decision.kind == "granted":
|
|
1617
|
+
await self.requests.set_status(env.request_id, "in_progress", served_model=other)
|
|
1618
|
+
self._start_run(held, decision, attempt, served=other)
|
|
1619
|
+
return True
|
|
1620
|
+
return False
|
|
1621
|
+
|
|
1622
|
+
# ================================================================ worker: running
|
|
1623
|
+
|
|
1624
|
+
def _start_run(
|
|
1625
|
+
self, held: _Held, decision: Decision, attempt: int, served: str | None = None
|
|
1626
|
+
) -> None:
|
|
1627
|
+
alias = served or held.envelope.model
|
|
1628
|
+
held.grants.add((alias, self._owner(held, attempt)))
|
|
1629
|
+
task = asyncio.create_task(self._run(held, decision, attempt, alias))
|
|
1630
|
+
self._running.add(task)
|
|
1631
|
+
task.add_done_callback(self._running.discard)
|
|
1632
|
+
|
|
1633
|
+
async def _run(self, held: _Held, decision: Decision, attempt: int, alias: str) -> None:
|
|
1634
|
+
slots = self._slots.get(held.envelope.model)
|
|
1635
|
+
held.run = asyncio.current_task()
|
|
1636
|
+
try:
|
|
1637
|
+
if slots is None:
|
|
1638
|
+
await self._attempts(held, decision, attempt, alias)
|
|
1639
|
+
else:
|
|
1640
|
+
async with slots:
|
|
1641
|
+
await self._attempts(held, decision, attempt, alias)
|
|
1642
|
+
except asyncio.CancelledError:
|
|
1643
|
+
if not held.cancelled:
|
|
1644
|
+
raise
|
|
1645
|
+
# Cancelled by a caller: give the grant back as a cut attempt, reply `cancelled`.
|
|
1646
|
+
try:
|
|
1647
|
+
await self._give_back(held)
|
|
1648
|
+
await self._finish_call(held, "cancelled", "cancelled", final="cancelled")
|
|
1649
|
+
except Exception as exc:
|
|
1650
|
+
logger.warning("cancel_failed", request_id=held.envelope.request_id, error=str(exc))
|
|
1651
|
+
await self._release(held)
|
|
1652
|
+
except Exception as exc:
|
|
1653
|
+
# A store outage, a store refusal (OOM) or a bug mid-call. Never leave the call held
|
|
1654
|
+
# with nobody working on it (the heartbeat would hide its message for good): give
|
|
1655
|
+
# the grants back and return the message, so it runs again (at least once; a stored
|
|
1656
|
+
# result is reused). A direct call has no message: its caller gets the error.
|
|
1657
|
+
if isinstance(exc, STORE_ERRORS):
|
|
1658
|
+
self._store_trouble_at = time.monotonic()
|
|
1659
|
+
logger.warning(
|
|
1660
|
+
"run_failed",
|
|
1661
|
+
request_id=held.envelope.request_id,
|
|
1662
|
+
error=str(exc),
|
|
1663
|
+
error_type=type(exc).__name__,
|
|
1664
|
+
)
|
|
1665
|
+
await self._release(held)
|
|
1666
|
+
if held.message is None:
|
|
1667
|
+
raise
|
|
1668
|
+
|
|
1669
|
+
async def _attempts(self, held: _Held, decision: Decision, attempt: int, alias: str) -> None:
|
|
1670
|
+
throttles = 0
|
|
1671
|
+
tries = 0
|
|
1672
|
+
failed: set[str] = set()
|
|
1673
|
+
retry_secs = 0.0
|
|
1674
|
+
while True:
|
|
1675
|
+
view = self.registry.current
|
|
1676
|
+
dep_key = decision.deployment or ""
|
|
1677
|
+
owner = self._owner(held, attempt)
|
|
1678
|
+
held.grants.add((alias, owner))
|
|
1679
|
+
outcome, reply_or_error = await self._attempt(
|
|
1680
|
+
view, alias, held, dep_key, owner, attempt
|
|
1681
|
+
)
|
|
1682
|
+
held.grants.discard((alias, owner))
|
|
1683
|
+
if outcome == "ok":
|
|
1684
|
+
return
|
|
1685
|
+
kind, info, exc = reply_or_error
|
|
1686
|
+
if outcome == "stream_failed": # never switched silently once chunks were sent
|
|
1687
|
+
await self._finish_call(held, kind, str(exc), partial=info)
|
|
1688
|
+
return
|
|
1689
|
+
entry = view.registry.models[alias]
|
|
1690
|
+
if outcome == "throttle":
|
|
1691
|
+
retry_secs = 0.0 # a throttle moves on at once (a 5xx's backoff is not reused)
|
|
1692
|
+
throttles += 1
|
|
1693
|
+
limit = int(view.value(alias, "throttle_retry_attempts", dep_key))
|
|
1694
|
+
if throttles > limit:
|
|
1695
|
+
await self._finish_call(
|
|
1696
|
+
held, "rate_limited", "the provider kept throttling", retry_after=info
|
|
1697
|
+
)
|
|
1698
|
+
return
|
|
1699
|
+
else:
|
|
1700
|
+
tries += 1
|
|
1701
|
+
if outcome == "fatal" or tries >= int(view.value(alias, "max_attempts")):
|
|
1702
|
+
await self._finish_call(held, "provider_error", str(exc))
|
|
1703
|
+
return
|
|
1704
|
+
# A move to a deployment this call has not tried is free: it is failover, not
|
|
1705
|
+
# a retry storm (bounded by the deployments and max_attempts). Counting it
|
|
1706
|
+
# failed calls as retry_budget_exhausted while another region was healthy
|
|
1707
|
+
# (scenario P3: 20 of 60 calls with one region hanging).
|
|
1708
|
+
tried = frozenset(failed | {dep_key})
|
|
1709
|
+
age = self.clock.now() - held.position_ms / 1000.0
|
|
1710
|
+
untried = any(
|
|
1711
|
+
not c.last
|
|
1712
|
+
for c in self._candidates(view, alias, held.envelope, age, tried, "error")
|
|
1713
|
+
)
|
|
1714
|
+
if not untried and not await self._retry_allowed(alias, view):
|
|
1715
|
+
await self._finish_call(held, "retry_budget_exhausted", str(exc))
|
|
1716
|
+
return
|
|
1717
|
+
retry_secs = self._backoff(view, alias, dep_key, tries)
|
|
1718
|
+
failed.add(dep_key)
|
|
1719
|
+
attempt += 1
|
|
1720
|
+
if retry_secs: # never past the deadline: the caller is told in time
|
|
1721
|
+
left = self._deadline(held) - self.clock.now()
|
|
1722
|
+
await asyncio.sleep(max(0.0, min(retry_secs, left)))
|
|
1723
|
+
while True:
|
|
1724
|
+
if self.clock.now() >= self._deadline(held):
|
|
1725
|
+
await self._finish_call(held, "deadline_exceeded", f"last error: {kind}")
|
|
1726
|
+
return
|
|
1727
|
+
next_decision = await self._decide(view, alias, held, attempt, frozenset(failed))
|
|
1728
|
+
if next_decision is None:
|
|
1729
|
+
return
|
|
1730
|
+
if next_decision.kind == "granted":
|
|
1731
|
+
decision = next_decision
|
|
1732
|
+
break
|
|
1733
|
+
if next_decision.kind == "busy":
|
|
1734
|
+
await self._finish_call(held, "rate_limited", "fallback model is busy")
|
|
1735
|
+
return
|
|
1736
|
+
await asyncio.sleep(min(next_decision.wait_secs, 1.0))
|
|
1737
|
+
del entry
|
|
1738
|
+
|
|
1739
|
+
def _backoff(self, view: ResolvedRegistry, alias: str, dep: str, tries: int) -> float:
|
|
1740
|
+
base = float(view.value(alias, "retry_base_secs", dep))
|
|
1741
|
+
factor = float(view.value(alias, "retry_multiplier", dep))
|
|
1742
|
+
spread = float(view.value(alias, "retry_spread", dep))
|
|
1743
|
+
low = base * factor ** (tries - 1)
|
|
1744
|
+
return min(
|
|
1745
|
+
low * (1 + (spread - 1) * self._rng.random()),
|
|
1746
|
+
float(view.value(alias, "retry_max_secs", dep)),
|
|
1747
|
+
)
|
|
1748
|
+
|
|
1749
|
+
async def _retry_allowed(self, alias: str, view: ResolvedRegistry) -> bool:
|
|
1750
|
+
minute = int(self.clock.now() // 60)
|
|
1751
|
+
key = self.keys.model(alias, f"retries:{minute}")
|
|
1752
|
+
async with self.state.pipeline(transaction=False) as pipe:
|
|
1753
|
+
pipe.hincrby(key, "retries", 1)
|
|
1754
|
+
pipe.hget(key, "first")
|
|
1755
|
+
pipe.expire(key, 120)
|
|
1756
|
+
retries, first, _ = await pipe.execute()
|
|
1757
|
+
first_n = int(first or 0)
|
|
1758
|
+
allowed = max(
|
|
1759
|
+
int(view.value(alias, "retry_budget_min")),
|
|
1760
|
+
first_n * int(view.value(alias, "retry_budget_pct")) // 100,
|
|
1761
|
+
)
|
|
1762
|
+
return int(retries) <= allowed
|
|
1763
|
+
|
|
1764
|
+
async def _attempt(
|
|
1765
|
+
self,
|
|
1766
|
+
view: ResolvedRegistry,
|
|
1767
|
+
alias: str,
|
|
1768
|
+
held: _Held,
|
|
1769
|
+
dep_key: str,
|
|
1770
|
+
owner: str,
|
|
1771
|
+
attempt: int,
|
|
1772
|
+
) -> tuple[str, Any]:
|
|
1773
|
+
"""One provider attempt and its settlement: ('ok', None) or (outcome, (kind, info, exc))."""
|
|
1774
|
+
env = held.envelope
|
|
1775
|
+
entry = view.registry.models[alias]
|
|
1776
|
+
deployment = next(d for d in view.deployments(alias) if d.key == dep_key)
|
|
1777
|
+
route = route_for(view, deployment)
|
|
1778
|
+
params = self._params(view, alias, env, dep_key)
|
|
1779
|
+
# Refunds settle what admission charged: `_ask`'s estimate from model-level params. One
|
|
1780
|
+
# from this deployment's params (e.g. a larger account max_tokens) refunded more than
|
|
1781
|
+
# was taken, and TPM stopped holding (found in the 0.3.0 review).
|
|
1782
|
+
estimate = self._charged(view, alias, held)
|
|
1783
|
+
remaining = max(0.1, self._deadline(held) - self.clock.now())
|
|
1784
|
+
attempt_budget = float(view.value(alias, "attempt_timeout_secs", dep_key))
|
|
1785
|
+
timing = Timing(
|
|
1786
|
+
first_response_secs=min(
|
|
1787
|
+
attempt_budget,
|
|
1788
|
+
remaining / 2 if attempt == 1 else remaining,
|
|
1789
|
+
),
|
|
1790
|
+
idle_secs=float(view.value(alias, "stream_idle_secs")),
|
|
1791
|
+
)
|
|
1792
|
+
provider = self.providers.get(entry.provider)
|
|
1793
|
+
held.meta[owner] = {
|
|
1794
|
+
"alias": alias,
|
|
1795
|
+
"owner": owner,
|
|
1796
|
+
"request_id": env.request_id,
|
|
1797
|
+
"dep": dep_key,
|
|
1798
|
+
"cls": "p" if env.priority else "n",
|
|
1799
|
+
"cost_r": 1,
|
|
1800
|
+
"cost_t": estimate.total,
|
|
1801
|
+
}
|
|
1802
|
+
if attempt == 1:
|
|
1803
|
+
minute = int(self.clock.now() // 60)
|
|
1804
|
+
await self.state.hincrby(self.keys.model(env.model, f"retries:{minute}"), "first", 1)
|
|
1805
|
+
writer = None
|
|
1806
|
+
if env.stream and env.op in ("converse", "chat"):
|
|
1807
|
+
writer = ChunkWriter(
|
|
1808
|
+
self.state,
|
|
1809
|
+
self.keys.request(env.request_id, "chunks"),
|
|
1810
|
+
owner,
|
|
1811
|
+
flush_ms=int(view.value(alias, "stream_flush_ms", dep_key)),
|
|
1812
|
+
max_bytes=int(view.value(alias, "stream_max_bytes")),
|
|
1813
|
+
ttl_secs=self.settings.result_ttl_secs + int(env.timeout_secs),
|
|
1814
|
+
)
|
|
1815
|
+
if await self.state.xlen(writer.key):
|
|
1816
|
+
# Another worker already streamed part of this answer (plan R19).
|
|
1817
|
+
await self.admission.finish(
|
|
1818
|
+
alias,
|
|
1819
|
+
owner,
|
|
1820
|
+
env.request_id,
|
|
1821
|
+
refund_r=1,
|
|
1822
|
+
refund_t=estimate.total,
|
|
1823
|
+
outcome="none",
|
|
1824
|
+
latency_ms=-1,
|
|
1825
|
+
)
|
|
1826
|
+
lost = "the worker serving this stream was lost"
|
|
1827
|
+
await writer.error("worker_lost", lost)
|
|
1828
|
+
return "stream_failed", ("worker_lost", None, RuntimeError(lost))
|
|
1829
|
+
started = time.monotonic()
|
|
1830
|
+
try:
|
|
1831
|
+
result = await asyncio.wait_for(
|
|
1832
|
+
provider.call(env.op, params, route, timing, sink=writer.add if writer else None),
|
|
1833
|
+
remaining,
|
|
1834
|
+
)
|
|
1835
|
+
if writer is not None:
|
|
1836
|
+
await writer.end(
|
|
1837
|
+
{
|
|
1838
|
+
"inputTokens": result.usage.input_tokens,
|
|
1839
|
+
"outputTokens": result.usage.output_tokens,
|
|
1840
|
+
}
|
|
1841
|
+
)
|
|
1842
|
+
except Exception as exc:
|
|
1843
|
+
latency = (time.monotonic() - started) * 1000
|
|
1844
|
+
info = provider.classify(exc)
|
|
1845
|
+
if writer is not None and writer.sent:
|
|
1846
|
+
# Chunks reached the caller: end the stream with the partial text (plan §3.13).
|
|
1847
|
+
refund = tokens.refund(estimate, "cut")
|
|
1848
|
+
await self.admission.finish(
|
|
1849
|
+
alias,
|
|
1850
|
+
owner,
|
|
1851
|
+
env.request_id,
|
|
1852
|
+
refund_r=refund.requests,
|
|
1853
|
+
refund_t=refund.tokens,
|
|
1854
|
+
outcome="fail",
|
|
1855
|
+
latency_ms=latency,
|
|
1856
|
+
)
|
|
1857
|
+
kind = "rate_limited" if info.kind is ErrorKind.THROTTLE else "provider_error"
|
|
1858
|
+
message = (
|
|
1859
|
+
"stream above stream_max_bytes" if isinstance(exc, StreamTooLarge) else str(exc)
|
|
1860
|
+
)
|
|
1861
|
+
with contextlib.suppress(Exception):
|
|
1862
|
+
await writer.error(kind, message)
|
|
1863
|
+
return "stream_failed", (kind, writer.partial_text, exc)
|
|
1864
|
+
cut = isinstance(exc, AttemptTimeout) and "chunk" in str(exc)
|
|
1865
|
+
# The caller's own deadline cut this attempt (the outer timer, or a first-response
|
|
1866
|
+
# timer shortened below `attempt_timeout_secs`): not the provider's fault. Counted
|
|
1867
|
+
# as a failure, late grants under a rate limit opened the breaker and turned the
|
|
1868
|
+
# rest into model_unavailable (found by scripts/load.py).
|
|
1869
|
+
deadline_cut = isinstance(exc, TimeoutError) and (
|
|
1870
|
+
not isinstance(exc, AttemptTimeout)
|
|
1871
|
+
or ("first response" in str(exc) and timing.first_response_secs < attempt_budget)
|
|
1872
|
+
)
|
|
1873
|
+
if deadline_cut:
|
|
1874
|
+
await self.admission.finish(
|
|
1875
|
+
alias,
|
|
1876
|
+
owner,
|
|
1877
|
+
env.request_id,
|
|
1878
|
+
refund_r=0,
|
|
1879
|
+
refund_t=0,
|
|
1880
|
+
outcome="none",
|
|
1881
|
+
latency_ms=-1,
|
|
1882
|
+
)
|
|
1883
|
+
return "retry", ("retryable", None, exc)
|
|
1884
|
+
if info.kind is ErrorKind.THROTTLE:
|
|
1885
|
+
refund = tokens.refund(estimate, "throttle")
|
|
1886
|
+
await self.admission.finish(
|
|
1887
|
+
alias,
|
|
1888
|
+
owner,
|
|
1889
|
+
env.request_id,
|
|
1890
|
+
refund_r=refund.requests,
|
|
1891
|
+
refund_t=refund.tokens,
|
|
1892
|
+
outcome="throttle",
|
|
1893
|
+
latency_ms=-1,
|
|
1894
|
+
)
|
|
1895
|
+
return "throttle", ("throttled", info.retry_after_secs, exc)
|
|
1896
|
+
if info.kind is ErrorKind.RETRYABLE or isinstance(exc, TimeoutError):
|
|
1897
|
+
refund = tokens.refund(estimate, "cut" if cut else "fail")
|
|
1898
|
+
await self.admission.finish(
|
|
1899
|
+
alias,
|
|
1900
|
+
owner,
|
|
1901
|
+
env.request_id,
|
|
1902
|
+
refund_r=refund.requests,
|
|
1903
|
+
refund_t=refund.tokens,
|
|
1904
|
+
outcome="fail",
|
|
1905
|
+
latency_ms=latency,
|
|
1906
|
+
)
|
|
1907
|
+
return "retry", ("retryable", None, exc)
|
|
1908
|
+
refund = tokens.refund(estimate, "throttle") # refused before it ran: all back
|
|
1909
|
+
await self.admission.finish(
|
|
1910
|
+
alias,
|
|
1911
|
+
owner,
|
|
1912
|
+
env.request_id,
|
|
1913
|
+
refund_r=refund.requests,
|
|
1914
|
+
refund_t=refund.tokens,
|
|
1915
|
+
outcome="none",
|
|
1916
|
+
latency_ms=-1,
|
|
1917
|
+
)
|
|
1918
|
+
if is_target_error(exc, info):
|
|
1919
|
+
return await self._target_error(view, alias, held, dep_key, exc)
|
|
1920
|
+
if isinstance(exc, CredentialsUnavailable) or _auth_error(info.code):
|
|
1921
|
+
self.providers.pool.forget(route)
|
|
1922
|
+
where = (deployment.account, deployment.region)
|
|
1923
|
+
_, backoff = self._bad_accounts.get(where, (0.0, 2.5))
|
|
1924
|
+
backoff = min(backoff * 2, 60.0)
|
|
1925
|
+
self._bad_accounts[where] = (time.monotonic() + backoff, backoff)
|
|
1926
|
+
logger.error(
|
|
1927
|
+
"account_credentials_failed",
|
|
1928
|
+
account=deployment.account,
|
|
1929
|
+
region=deployment.region,
|
|
1930
|
+
)
|
|
1931
|
+
return "retry", ("auth", None, exc)
|
|
1932
|
+
return "fatal", ("fatal", None, exc)
|
|
1933
|
+
latency = (time.monotonic() - started) * 1000
|
|
1934
|
+
used = tokens.used_tokens(
|
|
1935
|
+
result.usage,
|
|
1936
|
+
output_weight=float(view.value(alias, "tpm_output_weight", dep_key)),
|
|
1937
|
+
count_cached=bool(view.value(alias, "tpm_count_cached", dep_key)),
|
|
1938
|
+
)
|
|
1939
|
+
refund = tokens.refund(estimate, "ok", used)
|
|
1940
|
+
await self.admission.finish(
|
|
1941
|
+
alias,
|
|
1942
|
+
owner,
|
|
1943
|
+
env.request_id,
|
|
1944
|
+
refund_r=refund.requests,
|
|
1945
|
+
refund_t=refund.tokens,
|
|
1946
|
+
outcome="ok",
|
|
1947
|
+
latency_ms=latency,
|
|
1948
|
+
)
|
|
1949
|
+
meta = Meta(
|
|
1950
|
+
served_model=alias,
|
|
1951
|
+
deployment=dep_key,
|
|
1952
|
+
fallback="model"
|
|
1953
|
+
if alias != env.model
|
|
1954
|
+
else ("region" if deployment.group == "fallback" else None),
|
|
1955
|
+
attempts=attempt,
|
|
1956
|
+
queue_wait_ms=int(max(0.0, held.first_seen - held.position_ms / 1000.0) * 1000),
|
|
1957
|
+
provider_ms=int(latency),
|
|
1958
|
+
tokens_in=result.usage.input_tokens,
|
|
1959
|
+
tokens_out=result.usage.output_tokens,
|
|
1960
|
+
worker=self.worker_id,
|
|
1961
|
+
)
|
|
1962
|
+
reply = Reply(request_id=env.request_id, ok=True, response=result.response, meta=meta)
|
|
1963
|
+
await self.requests.put_result(reply, final_state="done")
|
|
1964
|
+
await self._count_target_errors(held, succeeded=True)
|
|
1965
|
+
self._record(held, alias, deployment.account, deployment.region, "ok", meta)
|
|
1966
|
+
if self.settings.debug_capture_allowed and view.value(alias, "debug_capture"):
|
|
1967
|
+
await self._debug(env.request_id, params, result.response)
|
|
1968
|
+
await self.requests.set_status(
|
|
1969
|
+
env.request_id,
|
|
1970
|
+
"done",
|
|
1971
|
+
deployment=dep_key,
|
|
1972
|
+
served_model=alias,
|
|
1973
|
+
attempts=attempt,
|
|
1974
|
+
tokens_in=result.usage.input_tokens,
|
|
1975
|
+
tokens_out=result.usage.output_tokens,
|
|
1976
|
+
)
|
|
1977
|
+
await self._ack(held)
|
|
1978
|
+
return "ok", None
|
|
1979
|
+
|
|
1980
|
+
async def _target_error(
|
|
1981
|
+
self, view: ResolvedRegistry, alias: str, held: _Held, dep_key: str, exc: BaseException
|
|
1982
|
+
) -> tuple[str, Any]:
|
|
1983
|
+
"""Note a target error; a request failing this way on two deployments is the cause.
|
|
1984
|
+
|
|
1985
|
+
Nothing is counted yet: `_count_target_errors` records them once the call's outcome
|
|
1986
|
+
shows whether the deployment or the request was at fault (plan §3.18 R16).
|
|
1987
|
+
"""
|
|
1988
|
+
held.target_failed.append((alias, dep_key))
|
|
1989
|
+
if len({d for _, d in held.target_failed}) >= 2:
|
|
1990
|
+
return "fatal", ("request", None, exc)
|
|
1991
|
+
return "retry", ("target", None, exc)
|
|
1992
|
+
|
|
1993
|
+
async def _count_target_errors(self, held: _Held, *, succeeded: bool) -> None:
|
|
1994
|
+
"""Count noted target errors: after a success elsewhere, or a failure on one deployment."""
|
|
1995
|
+
noted = held.target_failed
|
|
1996
|
+
held.target_failed = []
|
|
1997
|
+
if not noted or (not succeeded and len({d for _, d in noted}) >= 2):
|
|
1998
|
+
return
|
|
1999
|
+
view = self.registry.current
|
|
2000
|
+
for alias, dep_key in noted:
|
|
2001
|
+
settings = view.value(alias, "health", dep_key)
|
|
2002
|
+
others = [d.key for d in view.deployments(alias) if d.key != dep_key]
|
|
2003
|
+
outcome = await self.health.record_target_error(
|
|
2004
|
+
alias,
|
|
2005
|
+
dep_key,
|
|
2006
|
+
held.envelope.request_id,
|
|
2007
|
+
others=others,
|
|
2008
|
+
window_secs=float(settings["disable_window_secs"]),
|
|
2009
|
+
needed=int(settings["disable_errors"]),
|
|
2010
|
+
probe_after_secs=float(settings["probe_after_secs"]),
|
|
2011
|
+
)
|
|
2012
|
+
if outcome == "disabled":
|
|
2013
|
+
logger.error("deployment_disabled", model=alias, deployment=dep_key)
|
|
2014
|
+
elif outcome == "last":
|
|
2015
|
+
logger.error("deployment_failing_last_usable", model=alias, deployment=dep_key)
|
|
2016
|
+
|
|
2017
|
+
async def _prober(self) -> None:
|
|
2018
|
+
"""One process per model probes disabled deployments when they are due."""
|
|
2019
|
+
while True:
|
|
2020
|
+
await asyncio.sleep(5)
|
|
2021
|
+
try:
|
|
2022
|
+
view = self.registry.current
|
|
2023
|
+
for alias in view.registry.models:
|
|
2024
|
+
lock = await self.state.set(
|
|
2025
|
+
f"{self.keys.prefix}:lock:probe:{alias}", self.worker_id, nx=True, ex=5
|
|
2026
|
+
)
|
|
2027
|
+
if lock:
|
|
2028
|
+
await self.probe_due(alias)
|
|
2029
|
+
except Exception as exc:
|
|
2030
|
+
logger.warning("prober_failed", error=str(exc))
|
|
2031
|
+
|
|
2032
|
+
async def probe_due(self, alias: str) -> dict[str, str]:
|
|
2033
|
+
"""Probe every disabled deployment of a model that is due; their outcomes."""
|
|
2034
|
+
view = self.registry.current
|
|
2035
|
+
now = self.clock.now()
|
|
2036
|
+
out: dict[str, str] = {}
|
|
2037
|
+
for dep_key, due in (await self.health.disabled(alias)).items():
|
|
2038
|
+
if due > now + 0.5: # the local view of the store clock may lag slightly
|
|
2039
|
+
continue
|
|
2040
|
+
ok = await self._probe(view, alias, dep_key)
|
|
2041
|
+
settings = view.value(alias, "health", dep_key)
|
|
2042
|
+
out[dep_key] = await self.health.probed(
|
|
2043
|
+
alias, dep_key, ok=ok, probe_max_secs=float(settings["probe_max_secs"])
|
|
2044
|
+
)
|
|
2045
|
+
if out[dep_key] == "recovered":
|
|
2046
|
+
logger.info("deployment_recovered", model=alias, deployment=dep_key)
|
|
2047
|
+
return out
|
|
2048
|
+
|
|
2049
|
+
async def _probe(self, view: ResolvedRegistry, alias: str, dep_key: str) -> bool:
|
|
2050
|
+
"""One small real call on a deployment: reachable and allowed (a throttle counts)."""
|
|
2051
|
+
entry = view.registry.models[alias]
|
|
2052
|
+
deployment = next((d for d in view.deployments(alias) if d.key == dep_key), None)
|
|
2053
|
+
if deployment is None:
|
|
2054
|
+
return True
|
|
2055
|
+
op = OPS.get((entry.provider, entry.type), "chat" if entry.type == "chat" else "embeddings")
|
|
2056
|
+
name = view.value(alias, "probe")
|
|
2057
|
+
hook = hooks.get("probe", name)
|
|
2058
|
+
params = (
|
|
2059
|
+
hooks.call_safely("probe", name, hook, alias, op, default=None) if hook else None
|
|
2060
|
+
) or _probe_params(entry.provider, entry.type, entry.model_id)
|
|
2061
|
+
provider = self.providers.get(entry.provider)
|
|
2062
|
+
try:
|
|
2063
|
+
await provider.call(op, params, route_for(view, deployment), Timing(30, 30))
|
|
2064
|
+
except Exception as exc:
|
|
2065
|
+
return provider.classify(exc).kind is ErrorKind.THROTTLE
|
|
2066
|
+
return True
|
|
2067
|
+
|
|
2068
|
+
# ================================================================ worker: settling
|
|
2069
|
+
|
|
2070
|
+
async def _reply(
|
|
2071
|
+
self,
|
|
2072
|
+
env: Envelope,
|
|
2073
|
+
kind: str,
|
|
2074
|
+
message: str,
|
|
2075
|
+
*,
|
|
2076
|
+
final: str = "failed",
|
|
2077
|
+
retry_after: float | None = None,
|
|
2078
|
+
partial: str | None = None,
|
|
2079
|
+
) -> None:
|
|
2080
|
+
reply = Reply(
|
|
2081
|
+
request_id=env.request_id,
|
|
2082
|
+
ok=False,
|
|
2083
|
+
error=kind,
|
|
2084
|
+
message=message,
|
|
2085
|
+
retry_after_secs=retry_after,
|
|
2086
|
+
partial_text=partial or None,
|
|
2087
|
+
meta=Meta(served_model=env.model, worker=self.worker_id),
|
|
2088
|
+
)
|
|
2089
|
+
await self.requests.put_result(reply, final_state=final)
|
|
2090
|
+
await self.requests.set_status(env.request_id, final, error=kind)
|
|
2091
|
+
|
|
2092
|
+
async def _finish_call(
|
|
2093
|
+
self,
|
|
2094
|
+
held: _Held,
|
|
2095
|
+
kind: str,
|
|
2096
|
+
message: str,
|
|
2097
|
+
*,
|
|
2098
|
+
final: str = "failed",
|
|
2099
|
+
retry_after: float | None = None,
|
|
2100
|
+
partial: str | None = None,
|
|
2101
|
+
) -> None:
|
|
2102
|
+
await self._reply(
|
|
2103
|
+
held.envelope, kind, message, final=final, retry_after=retry_after, partial=partial
|
|
2104
|
+
)
|
|
2105
|
+
await self._count_target_errors(held, succeeded=False)
|
|
2106
|
+
self._record(held, held.envelope.model, "", "", kind, None)
|
|
2107
|
+
await self._ack(held)
|
|
2108
|
+
|
|
2109
|
+
def _record(
|
|
2110
|
+
self, held: _Held, alias: str, account: str, region: str, outcome: str, meta: Meta | None
|
|
2111
|
+
) -> None:
|
|
2112
|
+
"""Metrics and the durable per-call record (`llmq_call_finished`, no text; E134)."""
|
|
2113
|
+
env = held.envelope
|
|
2114
|
+
labels = {"model": alias, "account": account or "-", "region": region or "-"}
|
|
2115
|
+
self.metrics.count("llmq_calls_total", 1.0, outcome=outcome, **labels)
|
|
2116
|
+
wait = max(0.0, held.first_seen - held.position_ms / 1000.0)
|
|
2117
|
+
self.metrics.observe(
|
|
2118
|
+
"llmq_wait_seconds", wait, model=env.model, priority="1" if env.priority else "0"
|
|
2119
|
+
)
|
|
2120
|
+
if meta is not None:
|
|
2121
|
+
self.metrics.count("llmq_tokens_input_total", float(meta.tokens_in or 0), **labels)
|
|
2122
|
+
self.metrics.count("llmq_tokens_output_total", float(meta.tokens_out or 0), **labels)
|
|
2123
|
+
if meta.fallback:
|
|
2124
|
+
self.metrics.count("llmq_fallback_total", kind=meta.fallback, model=env.model)
|
|
2125
|
+
user = self.hash_id(env.user_id) if env.user_id else None
|
|
2126
|
+
logger.info(
|
|
2127
|
+
"llmq_call_finished",
|
|
2128
|
+
request_id=env.request_id,
|
|
2129
|
+
model=env.model,
|
|
2130
|
+
served_model=alias,
|
|
2131
|
+
deployment=meta.deployment if meta else None,
|
|
2132
|
+
outcome=outcome,
|
|
2133
|
+
priority=env.priority,
|
|
2134
|
+
attempts=meta.attempts if meta else None,
|
|
2135
|
+
tokens_in=meta.tokens_in if meta else None,
|
|
2136
|
+
tokens_out=meta.tokens_out if meta else None,
|
|
2137
|
+
queue_wait_ms=int(wait * 1000),
|
|
2138
|
+
provider_ms=meta.provider_ms if meta else None,
|
|
2139
|
+
user=user,
|
|
2140
|
+
tenant=env.tenant_id,
|
|
2141
|
+
)
|
|
2142
|
+
|
|
2143
|
+
async def _debug(
|
|
2144
|
+
self, request_id: str, params: Mapping[str, Any], response: Mapping[str, Any]
|
|
2145
|
+
) -> None:
|
|
2146
|
+
"""`debug_capture`: truncated text for at most 60 minutes (plan §3.12)."""
|
|
2147
|
+
import orjson
|
|
2148
|
+
|
|
2149
|
+
body = orjson.dumps(
|
|
2150
|
+
{
|
|
2151
|
+
"request": orjson.dumps(params).decode()[:2000],
|
|
2152
|
+
"response": orjson.dumps(response).decode()[:2000],
|
|
2153
|
+
}
|
|
2154
|
+
)
|
|
2155
|
+
await self.state.set(self.keys.request(request_id, "debug"), body, ex=3600)
|
|
2156
|
+
|
|
2157
|
+
async def status(self, model: str | None = None) -> dict[str, Any]:
|
|
2158
|
+
"""Status of one model, or of everything (plan §3.12)."""
|
|
2159
|
+
view = self.registry.current
|
|
2160
|
+
if model is not None:
|
|
2161
|
+
return await self.status_reader.model(view, view.registry.alias_of(model))
|
|
2162
|
+
return await self.status_reader.report(view)
|
|
2163
|
+
|
|
2164
|
+
async def request_status(self, request_id: str) -> dict[str, Any]:
|
|
2165
|
+
"""Status of one call (empty when unknown or expired)."""
|
|
2166
|
+
return await self.status_reader.request(request_id)
|
|
2167
|
+
|
|
2168
|
+
async def health_check(self) -> dict[str, Any]:
|
|
2169
|
+
"""Liveness and readiness (plan E124): ready = registry loaded and not draining.
|
|
2170
|
+
|
|
2171
|
+
The stores' state is reported but never makes the process unready. It is checked with a
|
|
2172
|
+
short-lived write, not PING: a replica answers PING but refuses writes (`read_only`).
|
|
2173
|
+
"""
|
|
2174
|
+
probe = f"{self.keys.prefix}:health:{self.worker_id}"
|
|
2175
|
+
|
|
2176
|
+
async def check(client: Any) -> str:
|
|
2177
|
+
# Bounded: a blackholed store took 41 s (connect timeout x retries), past any probe.
|
|
2178
|
+
try:
|
|
2179
|
+
await asyncio.wait_for(
|
|
2180
|
+
client.set(probe, b"1", px=5000), self.settings.timings.health_probe_secs
|
|
2181
|
+
)
|
|
2182
|
+
except StoreReadOnlyError:
|
|
2183
|
+
return "read_only"
|
|
2184
|
+
except (*STORE_ERRORS, TimeoutError):
|
|
2185
|
+
return "down"
|
|
2186
|
+
return "ok"
|
|
2187
|
+
|
|
2188
|
+
if self._queue_store is None:
|
|
2189
|
+
state = queue = await check(self.state) # one store: probe it once
|
|
2190
|
+
else:
|
|
2191
|
+
state, queue = await asyncio.gather(check(self.state), check(self._queue_store))
|
|
2192
|
+
stores = {"state": state, "queue": queue}
|
|
2193
|
+
loaded = self.registry._current is not None
|
|
2194
|
+
return {"live": True, "ready": bool(self.ready and loaded), "stores": stores}
|
|
2195
|
+
|
|
2196
|
+
async def _alert_loop(self) -> None:
|
|
2197
|
+
"""Every 30 s one process evaluates alerts: gauges every time, a log line per change."""
|
|
2198
|
+
while True:
|
|
2199
|
+
await asyncio.sleep(30)
|
|
2200
|
+
try:
|
|
2201
|
+
lock = await self.state.set(
|
|
2202
|
+
f"{self.keys.prefix}:lock:alerts", self.worker_id, nx=True, ex=25
|
|
2203
|
+
)
|
|
2204
|
+
if not lock:
|
|
2205
|
+
continue
|
|
2206
|
+
view = self.registry.current
|
|
2207
|
+
report = await self.status_reader.report(view)
|
|
2208
|
+
alerts = self.alerts.evaluate(view, report)
|
|
2209
|
+
for alert in alerts:
|
|
2210
|
+
self.metrics.gauge(
|
|
2211
|
+
f"llmq_alert_{alert.name}", 1.0 if alert.on else 0.0, **alert.labels
|
|
2212
|
+
)
|
|
2213
|
+
await self.alerts.changed(alerts)
|
|
2214
|
+
except Exception as exc:
|
|
2215
|
+
logger.warning("alerts_failed", error=str(exc))
|
|
2216
|
+
|
|
2217
|
+
async def _ack(self, held: _Held) -> None:
|
|
2218
|
+
if held.message is not None:
|
|
2219
|
+
await self.queues.ack(held.message)
|
|
2220
|
+
self._drop(held)
|
|
2221
|
+
|
|
2222
|
+
def _drop(self, held: _Held) -> None:
|
|
2223
|
+
"""Forget a call, but only this one: a newer delivery may hold the same request id."""
|
|
2224
|
+
if self._held.get(held.envelope.request_id) is held:
|
|
2225
|
+
del self._held[held.envelope.request_id]
|
|
2226
|
+
|
|
2227
|
+
async def _give_back(self, held: _Held) -> None:
|
|
2228
|
+
"""Settle every grant of a call as a cut attempt (nothing learned, unused budget back)."""
|
|
2229
|
+
for served, owner in list(held.grants):
|
|
2230
|
+
with contextlib.suppress(Exception):
|
|
2231
|
+
await self.admission.finish(
|
|
2232
|
+
served, owner, held.envelope.request_id, refund_r=0, refund_t=0, outcome="none"
|
|
2233
|
+
)
|
|
2234
|
+
held.grants.clear()
|
|
2235
|
+
|
|
2236
|
+
async def _release(self, held: _Held, delay_secs: float = 0.0) -> None:
|
|
2237
|
+
"""Let go of a call after an unexpected error: grants back, not held, message returned."""
|
|
2238
|
+
await self._give_back(held)
|
|
2239
|
+
self._drop(held)
|
|
2240
|
+
if held.message is not None:
|
|
2241
|
+
with contextlib.suppress(Exception):
|
|
2242
|
+
await self.queues.return_(held.message, delay_secs=delay_secs)
|
|
2243
|
+
|
|
2244
|
+
# ================================================================ background
|
|
2245
|
+
|
|
2246
|
+
def _free_stuck(self) -> None:
|
|
2247
|
+
"""Let go of calls held a lease past their deadline; nothing should still hold them.
|
|
2248
|
+
|
|
2249
|
+
Every path settles a call by its deadline, so one still held a lease later is a bug or
|
|
2250
|
+
a hung store call. The heartbeat renewed its grants for good (one-hour soak: two calls
|
|
2251
|
+
in progress for 50 minutes after a store block). Now it is logged, its task cancelled,
|
|
2252
|
+
its grants given back and its message returned (the next worker finds it expired).
|
|
2253
|
+
"""
|
|
2254
|
+
late = self.clock.now() - self.settings.lease_secs - self.settings.caller_margin_secs
|
|
2255
|
+
for held in list(self._held.values()):
|
|
2256
|
+
if self._deadline(held) >= late:
|
|
2257
|
+
continue
|
|
2258
|
+
running = held.run is not None and not held.run.done()
|
|
2259
|
+
logger.error(
|
|
2260
|
+
"held_call_stuck",
|
|
2261
|
+
request_id=held.envelope.request_id,
|
|
2262
|
+
model=held.envelope.model,
|
|
2263
|
+
grants=sorted(held.grants),
|
|
2264
|
+
running=running,
|
|
2265
|
+
past_deadline_secs=round(self.clock.now() - self._deadline(held), 1),
|
|
2266
|
+
)
|
|
2267
|
+
self._drop(held) # never renewed again, even if the release below fails
|
|
2268
|
+
if running:
|
|
2269
|
+
assert held.run is not None
|
|
2270
|
+
held.run.cancel()
|
|
2271
|
+
task = asyncio.create_task(self._release(held))
|
|
2272
|
+
self._running.add(task)
|
|
2273
|
+
task.add_done_callback(self._running.discard)
|
|
2274
|
+
|
|
2275
|
+
async def _heartbeat(self) -> None:
|
|
2276
|
+
"""Renew leases and holds; keep the caller liveness key; resync the clock.
|
|
2277
|
+
|
|
2278
|
+
It beats at least three times per lease and per held message's visibility timeout, so
|
|
2279
|
+
a short `visibility_secs` never lets a running call's message reappear. Each step fails
|
|
2280
|
+
on its own: one bad message (e.g. of a model removed by a reload) skips only itself.
|
|
2281
|
+
"""
|
|
2282
|
+
while True:
|
|
2283
|
+
beat = self.settings.lease_secs / 3
|
|
2284
|
+
self._free_stuck()
|
|
2285
|
+
try:
|
|
2286
|
+
await self.state.set(
|
|
2287
|
+
self.keys.caller(self.worker_id), "1", ex=self.settings.caller_ttl_secs
|
|
2288
|
+
)
|
|
2289
|
+
items = [
|
|
2290
|
+
(alias, owner, held.envelope.request_id)
|
|
2291
|
+
for held in list(self._held.values())
|
|
2292
|
+
for alias, owner in list(held.grants)
|
|
2293
|
+
]
|
|
2294
|
+
await self.admission.renew(items)
|
|
2295
|
+
now = time.monotonic()
|
|
2296
|
+
if now - self._beat_ok_at > 2 * beat + 1: # a gap: the store was out of reach
|
|
2297
|
+
self._store_trouble_at = now
|
|
2298
|
+
self._beat_ok_at = now
|
|
2299
|
+
except Exception as exc: # the heartbeat must survive store hiccups
|
|
2300
|
+
logger.warning("heartbeat_failed", step="renew", error=str(exc))
|
|
2301
|
+
self._store_trouble_at = time.monotonic()
|
|
2302
|
+
beat = 1.0 # renew the caller key as soon as the store is back
|
|
2303
|
+
view = self.registry.current
|
|
2304
|
+
for held in list(self._held.values()):
|
|
2305
|
+
if held.message is None:
|
|
2306
|
+
continue
|
|
2307
|
+
try:
|
|
2308
|
+
visibility = float(view.value(held.envelope.model, "queue")["visibility_secs"])
|
|
2309
|
+
beat = min(beat, visibility / 3)
|
|
2310
|
+
await self.queues.extend(held.message, visibility)
|
|
2311
|
+
except Exception as exc:
|
|
2312
|
+
logger.warning(
|
|
2313
|
+
"heartbeat_failed",
|
|
2314
|
+
step="extend",
|
|
2315
|
+
request_id=held.envelope.request_id,
|
|
2316
|
+
error=str(exc),
|
|
2317
|
+
)
|
|
2318
|
+
try:
|
|
2319
|
+
await self.clock.sync(self.state)
|
|
2320
|
+
await self._check_epoch()
|
|
2321
|
+
except Exception as exc:
|
|
2322
|
+
logger.warning("heartbeat_failed", step="clock", error=str(exc))
|
|
2323
|
+
await asyncio.sleep(max(0.2, beat))
|
|
2324
|
+
|
|
2325
|
+
async def _check_epoch(self) -> None:
|
|
2326
|
+
"""The store lost its data (restart, failover): re-register running calls (T10)."""
|
|
2327
|
+
if await self.state.exists(self.keys.epoch):
|
|
2328
|
+
return
|
|
2329
|
+
logger.error("state_store_reset_detected")
|
|
2330
|
+
await self.state.set(self.keys.epoch, str(self.clock.now()), nx=True)
|
|
2331
|
+
await self.state.set(
|
|
2332
|
+
f"{self.keys.prefix}:reset_minute", int(self.clock.now() // 60), ex=120
|
|
2333
|
+
)
|
|
2334
|
+
items = [
|
|
2335
|
+
meta
|
|
2336
|
+
for held in list(self._held.values())
|
|
2337
|
+
for owner, meta in held.meta.items()
|
|
2338
|
+
if any(owner == o for _, o in held.grants)
|
|
2339
|
+
]
|
|
2340
|
+
await self.admission.reregister(items)
|
|
2341
|
+
|
|
2342
|
+
async def _sweeper(self) -> None:
|
|
2343
|
+
"""One process at a time removes in-progress entries of dead owners."""
|
|
2344
|
+
while True:
|
|
2345
|
+
await asyncio.sleep(self.settings.sweep_secs)
|
|
2346
|
+
try:
|
|
2347
|
+
lock = await self.state.set(
|
|
2348
|
+
f"{self.keys.prefix}:lock:sweep",
|
|
2349
|
+
self.worker_id,
|
|
2350
|
+
nx=True,
|
|
2351
|
+
ex=int(self.settings.sweep_secs),
|
|
2352
|
+
)
|
|
2353
|
+
if lock:
|
|
2354
|
+
for alias in self.registry.current.registry.models:
|
|
2355
|
+
await self.admission.sweep(alias)
|
|
2356
|
+
except Exception as exc:
|
|
2357
|
+
logger.warning("sweep_failed", error=str(exc))
|