llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,186 @@
1
+ """Process-level settings (plan §3.18 infrastructure table), read from the environment.
2
+
3
+ Everything about models lives in the registry; these are the stores, prefixes, the registry's
4
+ source and a few process limits. Every problem is reported at once.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import math
10
+ import os
11
+ from collections.abc import Callable, Mapping
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+ from typing import Any
15
+ from urllib.parse import parse_qs, urlsplit
16
+
17
+ from llm_queue.errors import ConfigError
18
+
19
+ PREFIX = "LLMQ_"
20
+ LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1")
21
+ STORE_SCHEMES = ("redis", "rediss", "valkey", "valkeys")
22
+ TLS_SCHEMES = ("rediss", "valkeys")
23
+ METRICS_SINKS = ("emf", "prometheus", "none")
24
+ # Environments where the store must use verified TLS and ids a keyed hash (fail closed: any
25
+ # name that is not clearly a developer's own is treated as holding real data).
26
+ DEV_ENVIRONMENTS = ("dev", "local", "test", "ci")
27
+
28
+
29
+ def _store_url_problems(name: str, url: str, strict: bool) -> list[str]:
30
+ """What is wrong with one store URL; `strict` (not a dev environment) requires verified TLS."""
31
+ parts = urlsplit(url)
32
+ if parts.scheme not in STORE_SCHEMES:
33
+ return [f"{name} must be a redis:// or rediss:// URL"]
34
+ if not strict:
35
+ return []
36
+ query = {k.lower(): v for k, v in parse_qs(parts.query).items()}
37
+ if parts.scheme not in TLS_SCHEMES and (parts.hostname or "") not in LOCAL_HOSTS:
38
+ return [f"{name} must use TLS (rediss://) outside dev"]
39
+ if query.get("ssl_cert_reqs", ["required"])[-1].lower() in ("none", "optional"):
40
+ return [f"{name} must verify the store's certificate outside dev (ssl_cert_reqs)"]
41
+ if query.get("ssl_check_hostname", ["true"])[-1].lower() in ("false", "0", "no"):
42
+ return [f"{name} must check the store's host name outside dev (ssl_check_hostname)"]
43
+ return []
44
+
45
+
46
+ @dataclass(frozen=True, slots=True)
47
+ class InfraSettings:
48
+ """The process settings of one environment."""
49
+
50
+ store_url: str = field(repr=False) # holds the password
51
+ queue_store_url: str | None = field(default=None, repr=False)
52
+ store_max_connections: int = 200
53
+ key_prefix: str = "llmq"
54
+ queue_prefix: str = ""
55
+ registry: str | None = None
56
+ registry_file: str | None = None
57
+ registry_live_edit: str = "safe"
58
+ base_url_allowed_hosts: tuple[str, ...] = ()
59
+ cfg_reload_secs: float = 30.0
60
+ result_ttl_secs: int = 300
61
+ result_dedupe_secs: int = 0
62
+ status_ttl_secs: int = 1800
63
+ audit_retention_days: int = 90
64
+ lease_secs: float = 60.0
65
+ caller_ttl_secs: int = 30
66
+ drain_secs: float = 45.0
67
+ local_limit_cap: int = 10
68
+ metrics_sink: str = "emf"
69
+ mode: str = "both"
70
+ bedrock_endpoint_url: str | None = None
71
+ sts_endpoint_url: str | None = None
72
+ id_hash_salt: str | None = None
73
+ environment: str = "dev"
74
+ queue_failover_secs: float = 5.0
75
+ queue_wait_replicas: int = 0
76
+ queue_wait_ms: int = 100
77
+ debug_capture_from_config: bool = False
78
+
79
+ @classmethod
80
+ def from_env(
81
+ cls,
82
+ environ: Mapping[str, str] | None = None,
83
+ *,
84
+ secrets: Callable[[str], str | None] | None = None,
85
+ ) -> InfraSettings:
86
+ """Read `LLMQ_*` from `environ` (default `os.environ`), secrets first when given."""
87
+ env = dict(os.environ if environ is None else environ)
88
+
89
+ def get(name: str, default: Any = None) -> Any:
90
+ if secrets is not None:
91
+ value = secrets(PREFIX + name)
92
+ if value:
93
+ return value
94
+ return env.get(PREFIX + name, default)
95
+
96
+ problems: list[str] = []
97
+ store = get("STORE_URL")
98
+ environment = str(get("ENVIRONMENT", "dev")).strip().lower()
99
+ if environment == "production":
100
+ environment = "prod"
101
+ strict = environment not in DEV_ENVIRONMENTS
102
+ if not store:
103
+ problems.append("LLMQ_STORE_URL is required (redis:// or rediss://)")
104
+ urls = (("LLMQ_STORE_URL", store), ("LLMQ_QUEUE_STORE_URL", get("QUEUE_STORE_URL")))
105
+ for name, url in urls:
106
+ if url:
107
+ problems += _store_url_problems(name, str(url), strict)
108
+ registry, registry_file = get("REGISTRY"), get("REGISTRY_FILE")
109
+ if registry and registry_file:
110
+ problems.append("set one of LLMQ_REGISTRY and LLMQ_REGISTRY_FILE, not both")
111
+ salt = get("ID_HASH_SALT")
112
+ if strict and not salt:
113
+ problems.append(f"LLMQ_ID_HASH_SALT is required outside dev ({environment})")
114
+ debug_capture = str(get("DEBUG_CAPTURE_FROM_CONFIG", "false")).strip().lower()
115
+ if debug_capture not in ("true", "false", "1", "0", "yes", "no", "on", "off"):
116
+ problems.append("LLMQ_DEBUG_CAPTURE_FROM_CONFIG must be true or false")
117
+ capture = debug_capture in ("true", "1", "yes", "on")
118
+ if capture and environment == "prod":
119
+ problems.append("LLMQ_DEBUG_CAPTURE_FROM_CONFIG is refused in prod (prompt text)")
120
+ sink = str(get("METRICS_SINK", "emf"))
121
+ if sink not in METRICS_SINKS:
122
+ problems.append("LLMQ_METRICS_SINK must be emf, prometheus or none")
123
+ live_edit = str(get("REGISTRY_LIVE_EDIT", "safe"))
124
+ if live_edit not in ("safe", "all", "off"):
125
+ problems.append("LLMQ_REGISTRY_LIVE_EDIT must be safe, all or off")
126
+ mode = str(get("MODE", "both"))
127
+ if mode not in ("both", "caller", "worker"):
128
+ problems.append("LLMQ_MODE must be both, caller or worker")
129
+
130
+ def number(name: str, default: float, kind: type = float, *, zero: bool = False) -> Any:
131
+ raw = get(name, default)
132
+ try:
133
+ value = kind(raw)
134
+ except (TypeError, ValueError):
135
+ problems.append(f"LLMQ_{name} must be a number")
136
+ return default
137
+ if not math.isfinite(value):
138
+ problems.append(f"LLMQ_{name} must be a finite number")
139
+ return default
140
+ if value < 0 or (value == 0 and not zero):
141
+ problems.append(f"LLMQ_{name} must be {'0 or more' if zero else 'above 0'}")
142
+ return value
143
+
144
+ settings = cls(
145
+ store_url=str(store or ""),
146
+ queue_store_url=get("QUEUE_STORE_URL"),
147
+ store_max_connections=number("STORE_MAX_CONNECTIONS", 200, int),
148
+ key_prefix=str(get("KEY_PREFIX", "llmq")),
149
+ queue_prefix=str(get("QUEUE_PREFIX", "")),
150
+ registry=registry,
151
+ registry_file=registry_file,
152
+ registry_live_edit=live_edit,
153
+ base_url_allowed_hosts=tuple(
154
+ h.strip() for h in str(get("BASE_URL_ALLOWED_HOSTS", "")).split(",") if h.strip()
155
+ ),
156
+ cfg_reload_secs=number("CFG_RELOAD_SECS", 30.0),
157
+ result_ttl_secs=number("RESULT_TTL_SECS", 300, int),
158
+ result_dedupe_secs=number("RESULT_DEDUPE_SECS", 0, int, zero=True),
159
+ status_ttl_secs=number("STATUS_TTL_SECS", 1800, int),
160
+ audit_retention_days=number("AUDIT_RETENTION_DAYS", 90, int),
161
+ lease_secs=number("LEASE_SECS", 60.0),
162
+ caller_ttl_secs=number("CALLER_TTL_SECS", 30, int),
163
+ drain_secs=number("DRAIN_SECS", 45.0),
164
+ local_limit_cap=number("LOCAL_LIMIT_CAP", 10, int),
165
+ metrics_sink=sink,
166
+ mode=mode,
167
+ bedrock_endpoint_url=get("BEDROCK_ENDPOINT_URL"),
168
+ sts_endpoint_url=get("STS_ENDPOINT_URL"),
169
+ id_hash_salt=salt,
170
+ environment=environment,
171
+ queue_failover_secs=number("QUEUE_FAILOVER_SECS", 5.0, zero=True),
172
+ queue_wait_replicas=number("QUEUE_WAIT_REPLICAS", 0, int, zero=True),
173
+ queue_wait_ms=number("QUEUE_WAIT_MS", 100, int),
174
+ debug_capture_from_config=capture,
175
+ )
176
+ if problems:
177
+ raise ConfigError("invalid LLMQ settings:\n " + "\n ".join(problems))
178
+ return settings
179
+
180
+ def registry_document(self) -> str:
181
+ """The registry JSON from `LLMQ_REGISTRY` or the file `LLMQ_REGISTRY_FILE`."""
182
+ if self.registry:
183
+ return self.registry
184
+ if self.registry_file:
185
+ return Path(self.registry_file).read_text(encoding="utf-8")
186
+ raise ConfigError("no registry: set LLMQ_REGISTRY or LLMQ_REGISTRY_FILE")
@@ -0,0 +1,267 @@
1
+ """Status of models, deployments and requests, read from the stores (plan §3.12).
2
+
3
+ `StatusReader.model()` reports, per model: waiting calls per class, calls in progress, and per
4
+ deployment the RPM / TPM used and left this minute, concurrency limit and use, breaker, health,
5
+ latency and throttles. `request()` reports one call. `AlertEvaluator` turns a report into state
6
+ alerts, sent as gauges every sample, with a log line once per change (plan §3.18 R32).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ from collections.abc import Mapping
13
+ from dataclasses import dataclass, field
14
+ from typing import TYPE_CHECKING, Any
15
+
16
+ from llm_queue._logging import get_logger
17
+ from llm_queue.engine.keys import Keys
18
+ from llm_queue.engine.queue import StreamQueues
19
+
20
+ if TYPE_CHECKING:
21
+ import redis.asyncio
22
+
23
+ from llm_queue.registry import ResolvedRegistry
24
+
25
+ logger = get_logger(__name__)
26
+
27
+ # One model's numbers in one command: TIME, calls in progress, health, and per deployment its
28
+ # rate hash, limiter, slots in use, breaker state and throttles this minute. As ~180 separate
29
+ # commands per model, client-side packing and parsing made a 600-deployment report 140 ms+.
30
+ # KEYS: none. ARGV: model key prefix (`<p>:{m:<alias>}:`), deployment names...
31
+ _MODEL_STATUS = """
32
+ local base = ARGV[1]
33
+ local t = redis.call('TIME')
34
+ local now_ms = tonumber(t[1]) * 1000
35
+ local minute = math.floor(tonumber(t[1]) / 60)
36
+ local out = {t[1], redis.call('ZCOUNT', base .. 'inprog', now_ms, '+inf'),
37
+ redis.call('HGETALL', base .. 'health')}
38
+ for i = 2, #ARGV do
39
+ local dep = ARGV[i]
40
+ table.insert(out, {
41
+ redis.call('HGETALL', base .. 'rate:' .. dep),
42
+ redis.call('HGET', base .. 'ac:' .. dep, 'limit') or '',
43
+ redis.call('ZCOUNT', base .. 'lease:' .. dep .. ':p', now_ms, '+inf')
44
+ + redis.call('ZCOUNT', base .. 'lease:' .. dep .. ':n', now_ms, '+inf'),
45
+ redis.call('HGET', base .. 'cb:' .. dep, 'state') or 'closed',
46
+ redis.call('GET', base .. 'thr:' .. dep .. ':' .. minute) or '0',
47
+ })
48
+ end
49
+ return out
50
+ """
51
+
52
+
53
+ def _text(value: Any) -> str:
54
+ return value.decode() if isinstance(value, bytes) else str(value)
55
+
56
+
57
+ def _num(value: Any, default: float = 0.0) -> float:
58
+ try:
59
+ return float(_text(value))
60
+ except (TypeError, ValueError):
61
+ return default
62
+
63
+
64
+ class StatusReader:
65
+ """Read-only view of one environment's stores."""
66
+
67
+ def __init__(
68
+ self, client: redis.asyncio.Redis, keys: Keys, queues: StreamQueues, queue_prefix: str = ""
69
+ ) -> None:
70
+ """Keep the clients; every report reads the store fresh."""
71
+ self._client = client
72
+ self.keys = keys
73
+ self.queues = queues
74
+ self.queue_prefix = queue_prefix
75
+ self._model_status = client.register_script(_MODEL_STATUS)
76
+
77
+ async def model(self, view: ResolvedRegistry, alias: str) -> dict[str, Any]:
78
+ """One model: queues, calls in progress, and every deployment's numbers.
79
+
80
+ All deployment reads are one script (one command per model): read one by one, 20
81
+ models x 30 deployments took 6.5 s (plan B91 asks for 200 ms; now 47 ms).
82
+ """
83
+ name = self.queue_prefix + view.queue_name(alias)
84
+ deps = view.deployments(alias)
85
+ base = self.keys.model(alias, "")
86
+ depths, raw = await asyncio.gather(
87
+ asyncio.gather(self.queues.depth(f"{name}-priority"), self.queues.depth(name)),
88
+ self._model_status([], [base, *(d.key for d in deps)]),
89
+ )
90
+ waiting = {
91
+ cls: max(0, total - held)
92
+ for cls, (total, held, _) in zip(("priority", "normal"), depths, strict=True)
93
+ }
94
+ dead = sum(letters for _, _, letters in depths)
95
+ minute = int(raw[0]) // 60
96
+ inprog, flat = raw[1], raw[2]
97
+ health = {_text(flat[i]): _text(flat[i + 1]) for i in range(0, len(flat), 2)}
98
+ deployments = [
99
+ self._deployment(view, alias, dep, raw[3 + i], health, minute)
100
+ for i, dep in enumerate(deps)
101
+ ]
102
+ return {
103
+ "model": alias,
104
+ "enabled": bool(view.effective(alias, "enabled").value),
105
+ "waiting": waiting,
106
+ "in_progress": int(inprog),
107
+ "dead_letters": dead,
108
+ "deployments": deployments,
109
+ }
110
+
111
+ def _deployment(
112
+ self,
113
+ view: ResolvedRegistry,
114
+ alias: str,
115
+ deployment: Any,
116
+ replies: list[Any],
117
+ health: Mapping[str, str],
118
+ minute: int,
119
+ ) -> dict[str, Any]:
120
+ flat_rate, raw_limit, in_use, breaker, throttles = replies
121
+ dep = deployment.key
122
+ rate = {_text(flat_rate[i]): _text(flat_rate[i + 1]) for i in range(0, len(flat_rate), 2)}
123
+ fresh = rate.get("seen_win") == str(minute)
124
+ used_calls = int(_num(rate.get("seen_calls"))) if fresh else 0
125
+ used_tokens = int(_num(rate.get("seen_tokens"))) if fresh else 0
126
+ rpm = view.value(alias, "rpm", dep)
127
+ tpm = view.value(alias, "tpm", dep)
128
+ cap = view.value(alias, "max_concurrency", dep) or 200
129
+ limit = _num(raw_limit, cap) if _text(raw_limit) else cap
130
+ disabled_until = health.get(f"{dep}|until")
131
+ return {
132
+ "name": dep,
133
+ "group": deployment.group,
134
+ "enabled": bool(view.value(alias, "enabled", dep)),
135
+ "healthy": disabled_until is None,
136
+ "next_probe": _num(disabled_until) / 1000 if disabled_until else None,
137
+ "outside_residency": view.outside_residency(deployment),
138
+ "rpm": {
139
+ "limit": rpm,
140
+ "used": used_calls,
141
+ "left": None if rpm is None else max(0, rpm - used_calls),
142
+ },
143
+ "tpm": {
144
+ "limit": tpm,
145
+ "used": used_tokens,
146
+ "left": None if tpm is None else max(0, tpm - used_tokens),
147
+ },
148
+ "concurrency": {"limit": round(limit, 2), "in_use": int(in_use)},
149
+ "breaker": _text(breaker),
150
+ "latency_ms": round(_num(health.get(f"{dep}|lat")), 1)
151
+ if f"{dep}|lat" in health
152
+ else None,
153
+ "samples": int(_num(health.get(f"{dep}|n"))),
154
+ "throttles_1m": int(_num(throttles)),
155
+ }
156
+
157
+ async def report(self, view: ResolvedRegistry) -> dict[str, Any]:
158
+ """Every model, the registry version and the routing switches."""
159
+ models = list(await asyncio.gather(*(self.model(view, a) for a in view.registry.models)))
160
+ routing = view.registry.routing
161
+ return {
162
+ "registry_version": view.registry.version,
163
+ "kill_switch": routing.kill_switch,
164
+ "blocked_regions": list(routing.blocked_regions),
165
+ "blocked_accounts": list(routing.blocked_accounts),
166
+ "models": models,
167
+ }
168
+
169
+ async def request(self, request_id: str) -> dict[str, Any]:
170
+ """One call: its status, and its in-progress entry while it runs."""
171
+ status = {
172
+ _text(k): _text(v)
173
+ for k, v in (await self._client.hgetall(self.keys.status(request_id))).items()
174
+ }
175
+ if not status:
176
+ return {}
177
+ model = status.get("served_model") or status.get("model")
178
+ if model and status.get("state") == "waiting":
179
+ entry = await self._client.hgetall(self.keys.model(model, f"inprog:{request_id}"))
180
+ if entry:
181
+ fields = {_text(k): _text(v) for k, v in entry.items()}
182
+ status["state"] = "in_progress"
183
+ status["deployment"] = fields.get("dep", "")
184
+ return {"request_id": request_id, **status}
185
+
186
+
187
+ @dataclass(slots=True)
188
+ class Alert:
189
+ """One alert state: on or off, with its details."""
190
+
191
+ name: str
192
+ on: bool
193
+ labels: dict[str, str] = field(default_factory=dict)
194
+ detail: str = ""
195
+
196
+
197
+ class AlertEvaluator:
198
+ """State alerts from a report (thresholds from the model's `alerts` setting).
199
+
200
+ Gauges are returned every time; `changed()` logs a line only when a state flips, with the
201
+ store marker deciding which process logs it.
202
+ """
203
+
204
+ def __init__(self, client: redis.asyncio.Redis, keys: Keys) -> None:
205
+ """Keep the client for the change markers."""
206
+ self._client = client
207
+ self.keys = keys
208
+
209
+ def evaluate(self, view: ResolvedRegistry, report: Mapping[str, Any]) -> list[Alert]:
210
+ """Every alert state of a report."""
211
+ alerts: list[Alert] = []
212
+ for model in report["models"]:
213
+ alias = model["model"]
214
+ thresholds = view.value(alias, "alerts")
215
+ deps = model["deployments"]
216
+ usable = [
217
+ d for d in deps if d["enabled"] and d["healthy"] and not d["outside_residency"]
218
+ ]
219
+ alerts.append(Alert("model_unavailable", not usable, {"model": alias}))
220
+ for dep in deps:
221
+ labels = {"model": alias, "deployment": dep["name"]}
222
+ alerts.append(Alert("deployment_disabled", not dep["healthy"], labels))
223
+ limit = dep["rpm"]["limit"]
224
+ used_pct = 100 * dep["rpm"]["used"] / limit if limit else 0
225
+ alerts.append(
226
+ Alert(
227
+ "quota_near_limit",
228
+ used_pct >= thresholds["quota_used_pct"],
229
+ labels,
230
+ f"{used_pct:.0f}%",
231
+ )
232
+ )
233
+ quota_left = limit is None or dep["rpm"]["left"] > 0
234
+ alerts.append(
235
+ Alert(
236
+ "quota_above_observed",
237
+ dep["throttles_1m"] > 0 and quota_left,
238
+ labels,
239
+ f"{dep['throttles_1m']} throttles with budget left",
240
+ )
241
+ )
242
+ alerts.append(
243
+ Alert(
244
+ "dead_letters",
245
+ model["dead_letters"] >= thresholds["dlq_messages"],
246
+ {"model": alias},
247
+ )
248
+ )
249
+ return alerts
250
+
251
+ async def changed(self, alerts: list[Alert]) -> list[Alert]:
252
+ """Alerts whose state flipped since the last evaluation (logged once, by one process)."""
253
+ flipped = []
254
+ for alert in alerts:
255
+ marker = f"{self.keys.prefix}:alert:{alert.name}:" + ",".join(
256
+ f"{k}={v}" for k, v in sorted(alert.labels.items())
257
+ )
258
+ state = "1" if alert.on else "0"
259
+ previous = await self._client.getset(marker, state)
260
+ await self._client.expire(marker, 86_400)
261
+ if (_text(previous) if previous is not None else "0") != state:
262
+ flipped.append(alert)
263
+ log = logger.error if alert.on else logger.info
264
+ log(
265
+ "llmq_alert", alert=alert.name, on=alert.on, detail=alert.detail, **alert.labels
266
+ )
267
+ return flipped
@@ -0,0 +1,65 @@
1
+ """The store client (Valkey / Redis) every process builds: one place for its failover options.
2
+
3
+ A managed store (ElastiCache) fails over by promoting the replica and moving the endpoint's DNS
4
+ name. Pooled connections still reach the old primary, now a replica, and every write gets
5
+ `READONLY`; redis-py keeps such a connection, so without these options the queue stayed down
6
+ after a failover until the process restarted (measured, CHANGELOG 0.3.0). Here `READONLY`, like
7
+ a dropped connection, closes the connection and retries on a new one, which resolves the name
8
+ again. Used by `LLMQueue.from_env` and the `llm-queue` CLI.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import Any
14
+
15
+ import redis.asyncio
16
+ from redis.asyncio.retry import Retry
17
+ from redis.backoff import EqualJitterBackoff # in every redis-py the bounds allow (>=5)
18
+ from redis.credentials import CredentialProvider
19
+ from redis.exceptions import ConnectionError as StoreConnectionError
20
+ from redis.exceptions import ReadOnlyError
21
+
22
+ from llm_queue.tuning import StoreOptions
23
+
24
+ MAX_CONNECTIONS = 200 # LLMQ_STORE_MAX_CONNECTIONS; about worker slots + callers + 3 readers
25
+
26
+
27
+ def store_client(
28
+ url: str,
29
+ credential_provider: CredentialProvider | None = None,
30
+ *,
31
+ max_connections: int = MAX_CONNECTIONS,
32
+ options: StoreOptions | None = None,
33
+ ) -> redis.asyncio.Redis:
34
+ """An async client (bytes replies) for `url` that reconnects on drops and on `READONLY`.
35
+
36
+ The pool blocks when every connection is in use: redis-py's default pool raised at once,
37
+ and 150 concurrent calls on 100 connections were reported as the store being down
38
+ (measured in the 0.3.0 review). `valkey://` / `valkeys://` are read as `redis(s)://`.
39
+ """
40
+ opts = options or StoreOptions() # timeouts, retries: `Tuning.store`
41
+ if url.startswith("valkey"):
42
+ url = "redis" + url[len("valkey") :]
43
+ kwargs: dict[str, Any] = {
44
+ # Not on a read timeout: the command may have run (a Lua write would run twice), and
45
+ # each retry waited the whole timeout again: 4 x 15 s, a 61 s call in the slow-DNS
46
+ # failover drill. redis-py's Retry retries TimeoutError by default and
47
+ # `retry_on_error` only adds to that list, so the list is given to Retry itself
48
+ # (measured: a half-open store hung a command 60 s with the default, 15 s without).
49
+ "retry": Retry(
50
+ EqualJitterBackoff(cap=1.0, base=0.05),
51
+ opts.retries,
52
+ supported_errors=(StoreConnectionError, ReadOnlyError),
53
+ ),
54
+ "retry_on_error": [StoreConnectionError, ReadOnlyError],
55
+ "socket_connect_timeout": opts.connect_timeout_secs,
56
+ "socket_timeout": opts.read_timeout_secs,
57
+ "socket_keepalive": True,
58
+ "health_check_interval": opts.health_check_secs,
59
+ }
60
+ if credential_provider is not None:
61
+ kwargs["credential_provider"] = credential_provider
62
+ pool = redis.asyncio.BlockingConnectionPool.from_url(
63
+ url, max_connections=max_connections, timeout=opts.pool_wait_secs, **kwargs
64
+ )
65
+ return redis.asyncio.Redis.from_pool(pool) # owns the pool: aclose() closes it
@@ -0,0 +1,40 @@
1
+ """`store_op`: admin operations raise `StoreUnavailable`, never redis-py's own errors.
2
+
3
+ Callers catch one family (`LLMQueueError`). Calls already turn store trouble into `CallFailed`;
4
+ admin operations (cancel, status, registry edits) let redis-py's `ConnectionError` reach the
5
+ app when the store was down (scenario X2: a live edit during a store block).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import functools
11
+ from collections.abc import Awaitable, Callable
12
+ from typing import ParamSpec, TypeVar
13
+
14
+ from redis.exceptions import ConnectionError as StoreConnectionError
15
+ from redis.exceptions import OutOfMemoryError, ReadOnlyError
16
+ from redis.exceptions import TimeoutError as StoreTimeoutError
17
+
18
+ from llm_queue.errors import StoreUnavailable
19
+
20
+ P = ParamSpec("P")
21
+ R = TypeVar("R")
22
+
23
+ _DOWN = (StoreConnectionError, StoreTimeoutError, ReadOnlyError, OSError)
24
+
25
+
26
+ def store_op(fn: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[R]]:
27
+ """Wrap an async admin method: store trouble becomes `StoreUnavailable`."""
28
+
29
+ @functools.wraps(fn)
30
+ async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
31
+ try:
32
+ return await fn(*args, **kwargs)
33
+ except OutOfMemoryError as exc:
34
+ raise StoreUnavailable("the state store is out of memory") from exc
35
+ except _DOWN as exc:
36
+ if isinstance(exc, StoreUnavailable):
37
+ raise
38
+ raise StoreUnavailable(f"the state store is unreachable: {exc}") from exc
39
+
40
+ return wrapper