llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Process-level settings (plan §3.18 infrastructure table), read from the environment.
|
|
2
|
+
|
|
3
|
+
Everything about models lives in the registry; these are the stores, prefixes, the registry's
|
|
4
|
+
source and a few process limits. Every problem is reported at once.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import os
|
|
11
|
+
from collections.abc import Callable, Mapping
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
from urllib.parse import parse_qs, urlsplit
|
|
16
|
+
|
|
17
|
+
from llm_queue.errors import ConfigError
|
|
18
|
+
|
|
19
|
+
PREFIX = "LLMQ_"
|
|
20
|
+
LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1")
|
|
21
|
+
STORE_SCHEMES = ("redis", "rediss", "valkey", "valkeys")
|
|
22
|
+
TLS_SCHEMES = ("rediss", "valkeys")
|
|
23
|
+
METRICS_SINKS = ("emf", "prometheus", "none")
|
|
24
|
+
# Environments where the store must use verified TLS and ids a keyed hash (fail closed: any
|
|
25
|
+
# name that is not clearly a developer's own is treated as holding real data).
|
|
26
|
+
DEV_ENVIRONMENTS = ("dev", "local", "test", "ci")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _store_url_problems(name: str, url: str, strict: bool) -> list[str]:
|
|
30
|
+
"""What is wrong with one store URL; `strict` (not a dev environment) requires verified TLS."""
|
|
31
|
+
parts = urlsplit(url)
|
|
32
|
+
if parts.scheme not in STORE_SCHEMES:
|
|
33
|
+
return [f"{name} must be a redis:// or rediss:// URL"]
|
|
34
|
+
if not strict:
|
|
35
|
+
return []
|
|
36
|
+
query = {k.lower(): v for k, v in parse_qs(parts.query).items()}
|
|
37
|
+
if parts.scheme not in TLS_SCHEMES and (parts.hostname or "") not in LOCAL_HOSTS:
|
|
38
|
+
return [f"{name} must use TLS (rediss://) outside dev"]
|
|
39
|
+
if query.get("ssl_cert_reqs", ["required"])[-1].lower() in ("none", "optional"):
|
|
40
|
+
return [f"{name} must verify the store's certificate outside dev (ssl_cert_reqs)"]
|
|
41
|
+
if query.get("ssl_check_hostname", ["true"])[-1].lower() in ("false", "0", "no"):
|
|
42
|
+
return [f"{name} must check the store's host name outside dev (ssl_check_hostname)"]
|
|
43
|
+
return []
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True, slots=True)
|
|
47
|
+
class InfraSettings:
|
|
48
|
+
"""The process settings of one environment."""
|
|
49
|
+
|
|
50
|
+
store_url: str = field(repr=False) # holds the password
|
|
51
|
+
queue_store_url: str | None = field(default=None, repr=False)
|
|
52
|
+
store_max_connections: int = 200
|
|
53
|
+
key_prefix: str = "llmq"
|
|
54
|
+
queue_prefix: str = ""
|
|
55
|
+
registry: str | None = None
|
|
56
|
+
registry_file: str | None = None
|
|
57
|
+
registry_live_edit: str = "safe"
|
|
58
|
+
base_url_allowed_hosts: tuple[str, ...] = ()
|
|
59
|
+
cfg_reload_secs: float = 30.0
|
|
60
|
+
result_ttl_secs: int = 300
|
|
61
|
+
result_dedupe_secs: int = 0
|
|
62
|
+
status_ttl_secs: int = 1800
|
|
63
|
+
audit_retention_days: int = 90
|
|
64
|
+
lease_secs: float = 60.0
|
|
65
|
+
caller_ttl_secs: int = 30
|
|
66
|
+
drain_secs: float = 45.0
|
|
67
|
+
local_limit_cap: int = 10
|
|
68
|
+
metrics_sink: str = "emf"
|
|
69
|
+
mode: str = "both"
|
|
70
|
+
bedrock_endpoint_url: str | None = None
|
|
71
|
+
sts_endpoint_url: str | None = None
|
|
72
|
+
id_hash_salt: str | None = None
|
|
73
|
+
environment: str = "dev"
|
|
74
|
+
queue_failover_secs: float = 5.0
|
|
75
|
+
queue_wait_replicas: int = 0
|
|
76
|
+
queue_wait_ms: int = 100
|
|
77
|
+
debug_capture_from_config: bool = False
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def from_env(
|
|
81
|
+
cls,
|
|
82
|
+
environ: Mapping[str, str] | None = None,
|
|
83
|
+
*,
|
|
84
|
+
secrets: Callable[[str], str | None] | None = None,
|
|
85
|
+
) -> InfraSettings:
|
|
86
|
+
"""Read `LLMQ_*` from `environ` (default `os.environ`), secrets first when given."""
|
|
87
|
+
env = dict(os.environ if environ is None else environ)
|
|
88
|
+
|
|
89
|
+
def get(name: str, default: Any = None) -> Any:
|
|
90
|
+
if secrets is not None:
|
|
91
|
+
value = secrets(PREFIX + name)
|
|
92
|
+
if value:
|
|
93
|
+
return value
|
|
94
|
+
return env.get(PREFIX + name, default)
|
|
95
|
+
|
|
96
|
+
problems: list[str] = []
|
|
97
|
+
store = get("STORE_URL")
|
|
98
|
+
environment = str(get("ENVIRONMENT", "dev")).strip().lower()
|
|
99
|
+
if environment == "production":
|
|
100
|
+
environment = "prod"
|
|
101
|
+
strict = environment not in DEV_ENVIRONMENTS
|
|
102
|
+
if not store:
|
|
103
|
+
problems.append("LLMQ_STORE_URL is required (redis:// or rediss://)")
|
|
104
|
+
urls = (("LLMQ_STORE_URL", store), ("LLMQ_QUEUE_STORE_URL", get("QUEUE_STORE_URL")))
|
|
105
|
+
for name, url in urls:
|
|
106
|
+
if url:
|
|
107
|
+
problems += _store_url_problems(name, str(url), strict)
|
|
108
|
+
registry, registry_file = get("REGISTRY"), get("REGISTRY_FILE")
|
|
109
|
+
if registry and registry_file:
|
|
110
|
+
problems.append("set one of LLMQ_REGISTRY and LLMQ_REGISTRY_FILE, not both")
|
|
111
|
+
salt = get("ID_HASH_SALT")
|
|
112
|
+
if strict and not salt:
|
|
113
|
+
problems.append(f"LLMQ_ID_HASH_SALT is required outside dev ({environment})")
|
|
114
|
+
debug_capture = str(get("DEBUG_CAPTURE_FROM_CONFIG", "false")).strip().lower()
|
|
115
|
+
if debug_capture not in ("true", "false", "1", "0", "yes", "no", "on", "off"):
|
|
116
|
+
problems.append("LLMQ_DEBUG_CAPTURE_FROM_CONFIG must be true or false")
|
|
117
|
+
capture = debug_capture in ("true", "1", "yes", "on")
|
|
118
|
+
if capture and environment == "prod":
|
|
119
|
+
problems.append("LLMQ_DEBUG_CAPTURE_FROM_CONFIG is refused in prod (prompt text)")
|
|
120
|
+
sink = str(get("METRICS_SINK", "emf"))
|
|
121
|
+
if sink not in METRICS_SINKS:
|
|
122
|
+
problems.append("LLMQ_METRICS_SINK must be emf, prometheus or none")
|
|
123
|
+
live_edit = str(get("REGISTRY_LIVE_EDIT", "safe"))
|
|
124
|
+
if live_edit not in ("safe", "all", "off"):
|
|
125
|
+
problems.append("LLMQ_REGISTRY_LIVE_EDIT must be safe, all or off")
|
|
126
|
+
mode = str(get("MODE", "both"))
|
|
127
|
+
if mode not in ("both", "caller", "worker"):
|
|
128
|
+
problems.append("LLMQ_MODE must be both, caller or worker")
|
|
129
|
+
|
|
130
|
+
def number(name: str, default: float, kind: type = float, *, zero: bool = False) -> Any:
|
|
131
|
+
raw = get(name, default)
|
|
132
|
+
try:
|
|
133
|
+
value = kind(raw)
|
|
134
|
+
except (TypeError, ValueError):
|
|
135
|
+
problems.append(f"LLMQ_{name} must be a number")
|
|
136
|
+
return default
|
|
137
|
+
if not math.isfinite(value):
|
|
138
|
+
problems.append(f"LLMQ_{name} must be a finite number")
|
|
139
|
+
return default
|
|
140
|
+
if value < 0 or (value == 0 and not zero):
|
|
141
|
+
problems.append(f"LLMQ_{name} must be {'0 or more' if zero else 'above 0'}")
|
|
142
|
+
return value
|
|
143
|
+
|
|
144
|
+
settings = cls(
|
|
145
|
+
store_url=str(store or ""),
|
|
146
|
+
queue_store_url=get("QUEUE_STORE_URL"),
|
|
147
|
+
store_max_connections=number("STORE_MAX_CONNECTIONS", 200, int),
|
|
148
|
+
key_prefix=str(get("KEY_PREFIX", "llmq")),
|
|
149
|
+
queue_prefix=str(get("QUEUE_PREFIX", "")),
|
|
150
|
+
registry=registry,
|
|
151
|
+
registry_file=registry_file,
|
|
152
|
+
registry_live_edit=live_edit,
|
|
153
|
+
base_url_allowed_hosts=tuple(
|
|
154
|
+
h.strip() for h in str(get("BASE_URL_ALLOWED_HOSTS", "")).split(",") if h.strip()
|
|
155
|
+
),
|
|
156
|
+
cfg_reload_secs=number("CFG_RELOAD_SECS", 30.0),
|
|
157
|
+
result_ttl_secs=number("RESULT_TTL_SECS", 300, int),
|
|
158
|
+
result_dedupe_secs=number("RESULT_DEDUPE_SECS", 0, int, zero=True),
|
|
159
|
+
status_ttl_secs=number("STATUS_TTL_SECS", 1800, int),
|
|
160
|
+
audit_retention_days=number("AUDIT_RETENTION_DAYS", 90, int),
|
|
161
|
+
lease_secs=number("LEASE_SECS", 60.0),
|
|
162
|
+
caller_ttl_secs=number("CALLER_TTL_SECS", 30, int),
|
|
163
|
+
drain_secs=number("DRAIN_SECS", 45.0),
|
|
164
|
+
local_limit_cap=number("LOCAL_LIMIT_CAP", 10, int),
|
|
165
|
+
metrics_sink=sink,
|
|
166
|
+
mode=mode,
|
|
167
|
+
bedrock_endpoint_url=get("BEDROCK_ENDPOINT_URL"),
|
|
168
|
+
sts_endpoint_url=get("STS_ENDPOINT_URL"),
|
|
169
|
+
id_hash_salt=salt,
|
|
170
|
+
environment=environment,
|
|
171
|
+
queue_failover_secs=number("QUEUE_FAILOVER_SECS", 5.0, zero=True),
|
|
172
|
+
queue_wait_replicas=number("QUEUE_WAIT_REPLICAS", 0, int, zero=True),
|
|
173
|
+
queue_wait_ms=number("QUEUE_WAIT_MS", 100, int),
|
|
174
|
+
debug_capture_from_config=capture,
|
|
175
|
+
)
|
|
176
|
+
if problems:
|
|
177
|
+
raise ConfigError("invalid LLMQ settings:\n " + "\n ".join(problems))
|
|
178
|
+
return settings
|
|
179
|
+
|
|
180
|
+
def registry_document(self) -> str:
|
|
181
|
+
"""The registry JSON from `LLMQ_REGISTRY` or the file `LLMQ_REGISTRY_FILE`."""
|
|
182
|
+
if self.registry:
|
|
183
|
+
return self.registry
|
|
184
|
+
if self.registry_file:
|
|
185
|
+
return Path(self.registry_file).read_text(encoding="utf-8")
|
|
186
|
+
raise ConfigError("no registry: set LLMQ_REGISTRY or LLMQ_REGISTRY_FILE")
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Status of models, deployments and requests, read from the stores (plan §3.12).
|
|
2
|
+
|
|
3
|
+
`StatusReader.model()` reports, per model: waiting calls per class, calls in progress, and per
|
|
4
|
+
deployment the RPM / TPM used and left this minute, concurrency limit and use, breaker, health,
|
|
5
|
+
latency and throttles. `request()` reports one call. `AlertEvaluator` turns a report into state
|
|
6
|
+
alerts, sent as gauges every sample, with a log line once per change (plan §3.18 R32).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
from llm_queue._logging import get_logger
|
|
17
|
+
from llm_queue.engine.keys import Keys
|
|
18
|
+
from llm_queue.engine.queue import StreamQueues
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
import redis.asyncio
|
|
22
|
+
|
|
23
|
+
from llm_queue.registry import ResolvedRegistry
|
|
24
|
+
|
|
25
|
+
logger = get_logger(__name__)
|
|
26
|
+
|
|
27
|
+
# One model's numbers in one command: TIME, calls in progress, health, and per deployment its
|
|
28
|
+
# rate hash, limiter, slots in use, breaker state and throttles this minute. As ~180 separate
|
|
29
|
+
# commands per model, client-side packing and parsing made a 600-deployment report 140 ms+.
|
|
30
|
+
# KEYS: none. ARGV: model key prefix (`<p>:{m:<alias>}:`), deployment names...
|
|
31
|
+
_MODEL_STATUS = """
|
|
32
|
+
local base = ARGV[1]
|
|
33
|
+
local t = redis.call('TIME')
|
|
34
|
+
local now_ms = tonumber(t[1]) * 1000
|
|
35
|
+
local minute = math.floor(tonumber(t[1]) / 60)
|
|
36
|
+
local out = {t[1], redis.call('ZCOUNT', base .. 'inprog', now_ms, '+inf'),
|
|
37
|
+
redis.call('HGETALL', base .. 'health')}
|
|
38
|
+
for i = 2, #ARGV do
|
|
39
|
+
local dep = ARGV[i]
|
|
40
|
+
table.insert(out, {
|
|
41
|
+
redis.call('HGETALL', base .. 'rate:' .. dep),
|
|
42
|
+
redis.call('HGET', base .. 'ac:' .. dep, 'limit') or '',
|
|
43
|
+
redis.call('ZCOUNT', base .. 'lease:' .. dep .. ':p', now_ms, '+inf')
|
|
44
|
+
+ redis.call('ZCOUNT', base .. 'lease:' .. dep .. ':n', now_ms, '+inf'),
|
|
45
|
+
redis.call('HGET', base .. 'cb:' .. dep, 'state') or 'closed',
|
|
46
|
+
redis.call('GET', base .. 'thr:' .. dep .. ':' .. minute) or '0',
|
|
47
|
+
})
|
|
48
|
+
end
|
|
49
|
+
return out
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _text(value: Any) -> str:
|
|
54
|
+
return value.decode() if isinstance(value, bytes) else str(value)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _num(value: Any, default: float = 0.0) -> float:
|
|
58
|
+
try:
|
|
59
|
+
return float(_text(value))
|
|
60
|
+
except (TypeError, ValueError):
|
|
61
|
+
return default
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class StatusReader:
|
|
65
|
+
"""Read-only view of one environment's stores."""
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self, client: redis.asyncio.Redis, keys: Keys, queues: StreamQueues, queue_prefix: str = ""
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Keep the clients; every report reads the store fresh."""
|
|
71
|
+
self._client = client
|
|
72
|
+
self.keys = keys
|
|
73
|
+
self.queues = queues
|
|
74
|
+
self.queue_prefix = queue_prefix
|
|
75
|
+
self._model_status = client.register_script(_MODEL_STATUS)
|
|
76
|
+
|
|
77
|
+
async def model(self, view: ResolvedRegistry, alias: str) -> dict[str, Any]:
|
|
78
|
+
"""One model: queues, calls in progress, and every deployment's numbers.
|
|
79
|
+
|
|
80
|
+
All deployment reads are one script (one command per model): read one by one, 20
|
|
81
|
+
models x 30 deployments took 6.5 s (plan B91 asks for 200 ms; now 47 ms).
|
|
82
|
+
"""
|
|
83
|
+
name = self.queue_prefix + view.queue_name(alias)
|
|
84
|
+
deps = view.deployments(alias)
|
|
85
|
+
base = self.keys.model(alias, "")
|
|
86
|
+
depths, raw = await asyncio.gather(
|
|
87
|
+
asyncio.gather(self.queues.depth(f"{name}-priority"), self.queues.depth(name)),
|
|
88
|
+
self._model_status([], [base, *(d.key for d in deps)]),
|
|
89
|
+
)
|
|
90
|
+
waiting = {
|
|
91
|
+
cls: max(0, total - held)
|
|
92
|
+
for cls, (total, held, _) in zip(("priority", "normal"), depths, strict=True)
|
|
93
|
+
}
|
|
94
|
+
dead = sum(letters for _, _, letters in depths)
|
|
95
|
+
minute = int(raw[0]) // 60
|
|
96
|
+
inprog, flat = raw[1], raw[2]
|
|
97
|
+
health = {_text(flat[i]): _text(flat[i + 1]) for i in range(0, len(flat), 2)}
|
|
98
|
+
deployments = [
|
|
99
|
+
self._deployment(view, alias, dep, raw[3 + i], health, minute)
|
|
100
|
+
for i, dep in enumerate(deps)
|
|
101
|
+
]
|
|
102
|
+
return {
|
|
103
|
+
"model": alias,
|
|
104
|
+
"enabled": bool(view.effective(alias, "enabled").value),
|
|
105
|
+
"waiting": waiting,
|
|
106
|
+
"in_progress": int(inprog),
|
|
107
|
+
"dead_letters": dead,
|
|
108
|
+
"deployments": deployments,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
def _deployment(
|
|
112
|
+
self,
|
|
113
|
+
view: ResolvedRegistry,
|
|
114
|
+
alias: str,
|
|
115
|
+
deployment: Any,
|
|
116
|
+
replies: list[Any],
|
|
117
|
+
health: Mapping[str, str],
|
|
118
|
+
minute: int,
|
|
119
|
+
) -> dict[str, Any]:
|
|
120
|
+
flat_rate, raw_limit, in_use, breaker, throttles = replies
|
|
121
|
+
dep = deployment.key
|
|
122
|
+
rate = {_text(flat_rate[i]): _text(flat_rate[i + 1]) for i in range(0, len(flat_rate), 2)}
|
|
123
|
+
fresh = rate.get("seen_win") == str(minute)
|
|
124
|
+
used_calls = int(_num(rate.get("seen_calls"))) if fresh else 0
|
|
125
|
+
used_tokens = int(_num(rate.get("seen_tokens"))) if fresh else 0
|
|
126
|
+
rpm = view.value(alias, "rpm", dep)
|
|
127
|
+
tpm = view.value(alias, "tpm", dep)
|
|
128
|
+
cap = view.value(alias, "max_concurrency", dep) or 200
|
|
129
|
+
limit = _num(raw_limit, cap) if _text(raw_limit) else cap
|
|
130
|
+
disabled_until = health.get(f"{dep}|until")
|
|
131
|
+
return {
|
|
132
|
+
"name": dep,
|
|
133
|
+
"group": deployment.group,
|
|
134
|
+
"enabled": bool(view.value(alias, "enabled", dep)),
|
|
135
|
+
"healthy": disabled_until is None,
|
|
136
|
+
"next_probe": _num(disabled_until) / 1000 if disabled_until else None,
|
|
137
|
+
"outside_residency": view.outside_residency(deployment),
|
|
138
|
+
"rpm": {
|
|
139
|
+
"limit": rpm,
|
|
140
|
+
"used": used_calls,
|
|
141
|
+
"left": None if rpm is None else max(0, rpm - used_calls),
|
|
142
|
+
},
|
|
143
|
+
"tpm": {
|
|
144
|
+
"limit": tpm,
|
|
145
|
+
"used": used_tokens,
|
|
146
|
+
"left": None if tpm is None else max(0, tpm - used_tokens),
|
|
147
|
+
},
|
|
148
|
+
"concurrency": {"limit": round(limit, 2), "in_use": int(in_use)},
|
|
149
|
+
"breaker": _text(breaker),
|
|
150
|
+
"latency_ms": round(_num(health.get(f"{dep}|lat")), 1)
|
|
151
|
+
if f"{dep}|lat" in health
|
|
152
|
+
else None,
|
|
153
|
+
"samples": int(_num(health.get(f"{dep}|n"))),
|
|
154
|
+
"throttles_1m": int(_num(throttles)),
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async def report(self, view: ResolvedRegistry) -> dict[str, Any]:
|
|
158
|
+
"""Every model, the registry version and the routing switches."""
|
|
159
|
+
models = list(await asyncio.gather(*(self.model(view, a) for a in view.registry.models)))
|
|
160
|
+
routing = view.registry.routing
|
|
161
|
+
return {
|
|
162
|
+
"registry_version": view.registry.version,
|
|
163
|
+
"kill_switch": routing.kill_switch,
|
|
164
|
+
"blocked_regions": list(routing.blocked_regions),
|
|
165
|
+
"blocked_accounts": list(routing.blocked_accounts),
|
|
166
|
+
"models": models,
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async def request(self, request_id: str) -> dict[str, Any]:
|
|
170
|
+
"""One call: its status, and its in-progress entry while it runs."""
|
|
171
|
+
status = {
|
|
172
|
+
_text(k): _text(v)
|
|
173
|
+
for k, v in (await self._client.hgetall(self.keys.status(request_id))).items()
|
|
174
|
+
}
|
|
175
|
+
if not status:
|
|
176
|
+
return {}
|
|
177
|
+
model = status.get("served_model") or status.get("model")
|
|
178
|
+
if model and status.get("state") == "waiting":
|
|
179
|
+
entry = await self._client.hgetall(self.keys.model(model, f"inprog:{request_id}"))
|
|
180
|
+
if entry:
|
|
181
|
+
fields = {_text(k): _text(v) for k, v in entry.items()}
|
|
182
|
+
status["state"] = "in_progress"
|
|
183
|
+
status["deployment"] = fields.get("dep", "")
|
|
184
|
+
return {"request_id": request_id, **status}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(slots=True)
|
|
188
|
+
class Alert:
|
|
189
|
+
"""One alert state: on or off, with its details."""
|
|
190
|
+
|
|
191
|
+
name: str
|
|
192
|
+
on: bool
|
|
193
|
+
labels: dict[str, str] = field(default_factory=dict)
|
|
194
|
+
detail: str = ""
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class AlertEvaluator:
|
|
198
|
+
"""State alerts from a report (thresholds from the model's `alerts` setting).
|
|
199
|
+
|
|
200
|
+
Gauges are returned every time; `changed()` logs a line only when a state flips, with the
|
|
201
|
+
store marker deciding which process logs it.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
def __init__(self, client: redis.asyncio.Redis, keys: Keys) -> None:
|
|
205
|
+
"""Keep the client for the change markers."""
|
|
206
|
+
self._client = client
|
|
207
|
+
self.keys = keys
|
|
208
|
+
|
|
209
|
+
def evaluate(self, view: ResolvedRegistry, report: Mapping[str, Any]) -> list[Alert]:
|
|
210
|
+
"""Every alert state of a report."""
|
|
211
|
+
alerts: list[Alert] = []
|
|
212
|
+
for model in report["models"]:
|
|
213
|
+
alias = model["model"]
|
|
214
|
+
thresholds = view.value(alias, "alerts")
|
|
215
|
+
deps = model["deployments"]
|
|
216
|
+
usable = [
|
|
217
|
+
d for d in deps if d["enabled"] and d["healthy"] and not d["outside_residency"]
|
|
218
|
+
]
|
|
219
|
+
alerts.append(Alert("model_unavailable", not usable, {"model": alias}))
|
|
220
|
+
for dep in deps:
|
|
221
|
+
labels = {"model": alias, "deployment": dep["name"]}
|
|
222
|
+
alerts.append(Alert("deployment_disabled", not dep["healthy"], labels))
|
|
223
|
+
limit = dep["rpm"]["limit"]
|
|
224
|
+
used_pct = 100 * dep["rpm"]["used"] / limit if limit else 0
|
|
225
|
+
alerts.append(
|
|
226
|
+
Alert(
|
|
227
|
+
"quota_near_limit",
|
|
228
|
+
used_pct >= thresholds["quota_used_pct"],
|
|
229
|
+
labels,
|
|
230
|
+
f"{used_pct:.0f}%",
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
quota_left = limit is None or dep["rpm"]["left"] > 0
|
|
234
|
+
alerts.append(
|
|
235
|
+
Alert(
|
|
236
|
+
"quota_above_observed",
|
|
237
|
+
dep["throttles_1m"] > 0 and quota_left,
|
|
238
|
+
labels,
|
|
239
|
+
f"{dep['throttles_1m']} throttles with budget left",
|
|
240
|
+
)
|
|
241
|
+
)
|
|
242
|
+
alerts.append(
|
|
243
|
+
Alert(
|
|
244
|
+
"dead_letters",
|
|
245
|
+
model["dead_letters"] >= thresholds["dlq_messages"],
|
|
246
|
+
{"model": alias},
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
return alerts
|
|
250
|
+
|
|
251
|
+
async def changed(self, alerts: list[Alert]) -> list[Alert]:
|
|
252
|
+
"""Alerts whose state flipped since the last evaluation (logged once, by one process)."""
|
|
253
|
+
flipped = []
|
|
254
|
+
for alert in alerts:
|
|
255
|
+
marker = f"{self.keys.prefix}:alert:{alert.name}:" + ",".join(
|
|
256
|
+
f"{k}={v}" for k, v in sorted(alert.labels.items())
|
|
257
|
+
)
|
|
258
|
+
state = "1" if alert.on else "0"
|
|
259
|
+
previous = await self._client.getset(marker, state)
|
|
260
|
+
await self._client.expire(marker, 86_400)
|
|
261
|
+
if (_text(previous) if previous is not None else "0") != state:
|
|
262
|
+
flipped.append(alert)
|
|
263
|
+
log = logger.error if alert.on else logger.info
|
|
264
|
+
log(
|
|
265
|
+
"llmq_alert", alert=alert.name, on=alert.on, detail=alert.detail, **alert.labels
|
|
266
|
+
)
|
|
267
|
+
return flipped
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""The store client (Valkey / Redis) every process builds: one place for its failover options.
|
|
2
|
+
|
|
3
|
+
A managed store (ElastiCache) fails over by promoting the replica and moving the endpoint's DNS
|
|
4
|
+
name. Pooled connections still reach the old primary, now a replica, and every write gets
|
|
5
|
+
`READONLY`; redis-py keeps such a connection, so without these options the queue stayed down
|
|
6
|
+
after a failover until the process restarted (measured, CHANGELOG 0.3.0). Here `READONLY`, like
|
|
7
|
+
a dropped connection, closes the connection and retries on a new one, which resolves the name
|
|
8
|
+
again. Used by `LLMQueue.from_env` and the `llm-queue` CLI.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import redis.asyncio
|
|
16
|
+
from redis.asyncio.retry import Retry
|
|
17
|
+
from redis.backoff import EqualJitterBackoff # in every redis-py the bounds allow (>=5)
|
|
18
|
+
from redis.credentials import CredentialProvider
|
|
19
|
+
from redis.exceptions import ConnectionError as StoreConnectionError
|
|
20
|
+
from redis.exceptions import ReadOnlyError
|
|
21
|
+
|
|
22
|
+
from llm_queue.tuning import StoreOptions
|
|
23
|
+
|
|
24
|
+
MAX_CONNECTIONS = 200 # LLMQ_STORE_MAX_CONNECTIONS; about worker slots + callers + 3 readers
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def store_client(
|
|
28
|
+
url: str,
|
|
29
|
+
credential_provider: CredentialProvider | None = None,
|
|
30
|
+
*,
|
|
31
|
+
max_connections: int = MAX_CONNECTIONS,
|
|
32
|
+
options: StoreOptions | None = None,
|
|
33
|
+
) -> redis.asyncio.Redis:
|
|
34
|
+
"""An async client (bytes replies) for `url` that reconnects on drops and on `READONLY`.
|
|
35
|
+
|
|
36
|
+
The pool blocks when every connection is in use: redis-py's default pool raised at once,
|
|
37
|
+
and 150 concurrent calls on 100 connections were reported as the store being down
|
|
38
|
+
(measured in the 0.3.0 review). `valkey://` / `valkeys://` are read as `redis(s)://`.
|
|
39
|
+
"""
|
|
40
|
+
opts = options or StoreOptions() # timeouts, retries: `Tuning.store`
|
|
41
|
+
if url.startswith("valkey"):
|
|
42
|
+
url = "redis" + url[len("valkey") :]
|
|
43
|
+
kwargs: dict[str, Any] = {
|
|
44
|
+
# Not on a read timeout: the command may have run (a Lua write would run twice), and
|
|
45
|
+
# each retry waited the whole timeout again: 4 x 15 s, a 61 s call in the slow-DNS
|
|
46
|
+
# failover drill. redis-py's Retry retries TimeoutError by default and
|
|
47
|
+
# `retry_on_error` only adds to that list, so the list is given to Retry itself
|
|
48
|
+
# (measured: a half-open store hung a command 60 s with the default, 15 s without).
|
|
49
|
+
"retry": Retry(
|
|
50
|
+
EqualJitterBackoff(cap=1.0, base=0.05),
|
|
51
|
+
opts.retries,
|
|
52
|
+
supported_errors=(StoreConnectionError, ReadOnlyError),
|
|
53
|
+
),
|
|
54
|
+
"retry_on_error": [StoreConnectionError, ReadOnlyError],
|
|
55
|
+
"socket_connect_timeout": opts.connect_timeout_secs,
|
|
56
|
+
"socket_timeout": opts.read_timeout_secs,
|
|
57
|
+
"socket_keepalive": True,
|
|
58
|
+
"health_check_interval": opts.health_check_secs,
|
|
59
|
+
}
|
|
60
|
+
if credential_provider is not None:
|
|
61
|
+
kwargs["credential_provider"] = credential_provider
|
|
62
|
+
pool = redis.asyncio.BlockingConnectionPool.from_url(
|
|
63
|
+
url, max_connections=max_connections, timeout=opts.pool_wait_secs, **kwargs
|
|
64
|
+
)
|
|
65
|
+
return redis.asyncio.Redis.from_pool(pool) # owns the pool: aclose() closes it
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""`store_op`: admin operations raise `StoreUnavailable`, never redis-py's own errors.
|
|
2
|
+
|
|
3
|
+
Callers catch one family (`LLMQueueError`). Calls already turn store trouble into `CallFailed`;
|
|
4
|
+
admin operations (cancel, status, registry edits) let redis-py's `ConnectionError` reach the
|
|
5
|
+
app when the store was down (scenario X2: a live edit during a store block).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import functools
|
|
11
|
+
from collections.abc import Awaitable, Callable
|
|
12
|
+
from typing import ParamSpec, TypeVar
|
|
13
|
+
|
|
14
|
+
from redis.exceptions import ConnectionError as StoreConnectionError
|
|
15
|
+
from redis.exceptions import OutOfMemoryError, ReadOnlyError
|
|
16
|
+
from redis.exceptions import TimeoutError as StoreTimeoutError
|
|
17
|
+
|
|
18
|
+
from llm_queue.errors import StoreUnavailable
|
|
19
|
+
|
|
20
|
+
P = ParamSpec("P")
|
|
21
|
+
R = TypeVar("R")
|
|
22
|
+
|
|
23
|
+
_DOWN = (StoreConnectionError, StoreTimeoutError, ReadOnlyError, OSError)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def store_op(fn: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[R]]:
|
|
27
|
+
"""Wrap an async admin method: store trouble becomes `StoreUnavailable`."""
|
|
28
|
+
|
|
29
|
+
@functools.wraps(fn)
|
|
30
|
+
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
31
|
+
try:
|
|
32
|
+
return await fn(*args, **kwargs)
|
|
33
|
+
except OutOfMemoryError as exc:
|
|
34
|
+
raise StoreUnavailable("the state store is out of memory") from exc
|
|
35
|
+
except _DOWN as exc:
|
|
36
|
+
if isinstance(exc, StoreUnavailable):
|
|
37
|
+
raise
|
|
38
|
+
raise StoreUnavailable(f"the state store is unreachable: {exc}") from exc
|
|
39
|
+
|
|
40
|
+
return wrapper
|