llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Cross-field checks of a registry and the one entry point that loads it.
|
|
2
|
+
|
|
3
|
+
`load_registry` parses (`schema.build`), resolves and checks the document and raises one
|
|
4
|
+
`RegistryError` listing every problem. The checks here need effective values, so they run on
|
|
5
|
+
the resolved registry (plan §3.9 and §3.19).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import orjson
|
|
14
|
+
|
|
15
|
+
from llm_queue.errors import ConfigError
|
|
16
|
+
from llm_queue.registry.resolve import Deployment, ResolvedRegistry
|
|
17
|
+
from llm_queue.registry.schema import Registry, build
|
|
18
|
+
from llm_queue.registry.settings import CHAT_INFERENCE, EMBED_INFERENCE
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RegistryError(ConfigError):
|
|
22
|
+
"""The registry document is invalid; `problems` lists every problem with its path."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, problems: list[str], warnings: list[str] | None = None) -> None:
|
|
25
|
+
"""Keep the full lists; the message shows them one per line."""
|
|
26
|
+
self.problems = list(problems)
|
|
27
|
+
self.warnings = list(warnings or [])
|
|
28
|
+
lines = "\n ".join(self.problems)
|
|
29
|
+
super().__init__(f"invalid registry ({len(self.problems)} problems):\n {lines}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _budget_numbers(view: ResolvedRegistry, alias: str, dep: Deployment) -> tuple[Any, Any]:
|
|
33
|
+
return view.value(alias, "rpm", dep.key), view.value(alias, "tpm", dep.key)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _check_model(
|
|
37
|
+
view: ResolvedRegistry, alias: str, errors: list[str], warnings: list[str]
|
|
38
|
+
) -> None:
|
|
39
|
+
entry = view.registry.models[alias]
|
|
40
|
+
path = f"models.{alias}"
|
|
41
|
+
deployments = view.deployments(alias)
|
|
42
|
+
confirmed = [d for d in deployments if d.group == "confirmed"]
|
|
43
|
+
|
|
44
|
+
# Accounts: their kind must serve the model's provider; enabled flags only.
|
|
45
|
+
for dep in deployments:
|
|
46
|
+
account = view.registry.accounts.get(dep.account)
|
|
47
|
+
if account is not None and account.provider != entry.provider:
|
|
48
|
+
errors.append(
|
|
49
|
+
f"{path}.deployments.{dep.account}: a `{account.kind}` account serves "
|
|
50
|
+
f"`{account.provider}`, not `{entry.provider}`"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# Timeouts.
|
|
54
|
+
timeout = view.value(alias, "timeout_secs")
|
|
55
|
+
if timeout > view.value(alias, "timeout_max_secs"):
|
|
56
|
+
errors.append(f"{path}: `timeout_secs` is above `timeout_max_secs`")
|
|
57
|
+
for dep in deployments:
|
|
58
|
+
if view.value(alias, "attempt_timeout_secs", dep.key) >= view.value(
|
|
59
|
+
alias, "timeout_secs", dep.key
|
|
60
|
+
):
|
|
61
|
+
errors.append(
|
|
62
|
+
f"{path} @ {dep.key}: `attempt_timeout_secs` must be below `timeout_secs`"
|
|
63
|
+
)
|
|
64
|
+
max_wait = view.value(alias, "max_wait_secs")
|
|
65
|
+
for rule in ("fallback", "model_fallback"):
|
|
66
|
+
value = view.value(alias, rule)
|
|
67
|
+
if value and max_wait is not None and value["after_secs"] >= max_wait:
|
|
68
|
+
errors.append(f"{path}.{rule}.after_secs: must be below `max_wait_secs`")
|
|
69
|
+
|
|
70
|
+
# Shares.
|
|
71
|
+
if view.value(alias, "priority_reserve") + view.value(alias, "normal_min_share") >= 1:
|
|
72
|
+
errors.append(f"{path}: `priority_reserve` + `normal_min_share` must be below 1")
|
|
73
|
+
|
|
74
|
+
# Budgets named by rate_limits must have a number.
|
|
75
|
+
for dep in deployments:
|
|
76
|
+
limits = view.effective(alias, "rate_limits", dep.key)
|
|
77
|
+
rpm, tpm = _budget_numbers(view, alias, dep)
|
|
78
|
+
if limits.level != "library":
|
|
79
|
+
if limits.value == "rpm" and rpm is None:
|
|
80
|
+
errors.append(f"{path} @ {dep.key}: `rate_limits` is rpm but no `rpm` is set")
|
|
81
|
+
if limits.value == "tpm" and tpm is None:
|
|
82
|
+
errors.append(f"{path} @ {dep.key}: `rate_limits` is tpm but no `tpm` is set")
|
|
83
|
+
|
|
84
|
+
# Strategies that need quota numbers.
|
|
85
|
+
for name in ("balance", "balance_priority"):
|
|
86
|
+
if view.value(alias, name) == "headroom":
|
|
87
|
+
bare = [d.key for d in confirmed if _budget_numbers(view, alias, d) == (None, None)]
|
|
88
|
+
if bare:
|
|
89
|
+
errors.append(f"{path}.{name}: headroom needs `rpm` or `tpm` on {', '.join(bare)}")
|
|
90
|
+
if view.value(alias, "balance") == "weighted":
|
|
91
|
+
weights = [view.value(alias, "weight", d.key) for d in confirmed]
|
|
92
|
+
if weights and not any(weights):
|
|
93
|
+
errors.append(f"{path}: every confirmed deployment has weight 0")
|
|
94
|
+
|
|
95
|
+
# Inference parameters by model type and provider; defaults within their limits.
|
|
96
|
+
wrong = EMBED_INFERENCE if entry.type == "chat" else CHAT_INFERENCE
|
|
97
|
+
for dep_key in (None, *(d.key for d in deployments)):
|
|
98
|
+
inference = view.value(alias, "inference", dep_key) or {}
|
|
99
|
+
where = path if dep_key is None else f"{path} @ {dep_key}"
|
|
100
|
+
for name in inference:
|
|
101
|
+
if name in wrong:
|
|
102
|
+
errors.append(f"{where}.inference.{name}: not a parameter of a {entry.type} model")
|
|
103
|
+
if "seed" in inference and entry.provider == "bedrock":
|
|
104
|
+
errors.append(f"{where}.inference.seed: not supported by Bedrock")
|
|
105
|
+
caps = view.value(alias, "inference_limits", dep_key) or {}
|
|
106
|
+
if "max_tokens" in caps and inference.get("max_tokens", 0) > caps["max_tokens"]:
|
|
107
|
+
errors.append(f"{where}.inference.max_tokens: above `inference_limits.max_tokens`")
|
|
108
|
+
|
|
109
|
+
# Rules that depend on the model type.
|
|
110
|
+
if entry.type == "chat" and "embed_batch_size" in entry.settings:
|
|
111
|
+
errors.append(f"{path}.embed_batch_size: only for embedding models")
|
|
112
|
+
rule = view.value(alias, "model_fallback")
|
|
113
|
+
if rule and entry.type == "embedding":
|
|
114
|
+
errors.append(f"{path}.model_fallback: an embedding model cannot change model")
|
|
115
|
+
elif rule:
|
|
116
|
+
for item in rule["models"]:
|
|
117
|
+
other = item["model"]
|
|
118
|
+
if other == alias:
|
|
119
|
+
errors.append(f"{path}.model_fallback: a model cannot list itself")
|
|
120
|
+
continue
|
|
121
|
+
if other not in view.registry.models:
|
|
122
|
+
errors.append(f"{path}.model_fallback: unknown model `{other}`")
|
|
123
|
+
continue
|
|
124
|
+
target = view.registry.models[other]
|
|
125
|
+
if target.type != "chat":
|
|
126
|
+
errors.append(f"{path}.model_fallback: `{other}` is not a chat model")
|
|
127
|
+
back = view.value(other, "model_fallback")
|
|
128
|
+
if back:
|
|
129
|
+
mutual = [m for m in back["models"] if m["model"] == alias]
|
|
130
|
+
if mutual and not (item["require_headroom"] and mutual[0]["require_headroom"]):
|
|
131
|
+
errors.append(
|
|
132
|
+
f"{path}.model_fallback: `{alias}` and `{other}` list each other; "
|
|
133
|
+
"both must use require_headroom"
|
|
134
|
+
)
|
|
135
|
+
if item["require_headroom"]:
|
|
136
|
+
bare = [
|
|
137
|
+
d.key
|
|
138
|
+
for d in view.deployments(other, "confirmed")
|
|
139
|
+
if _budget_numbers(view, other, d) == (None, None)
|
|
140
|
+
]
|
|
141
|
+
if bare:
|
|
142
|
+
errors.append(
|
|
143
|
+
f"{path}.model_fallback: require_headroom on `{other}` needs `rpm` or "
|
|
144
|
+
f"`tpm` on {', '.join(bare)}"
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
# Warnings (not fatal).
|
|
148
|
+
fallback_deps = [d for d in deployments if d.group == "fallback"]
|
|
149
|
+
if fallback_deps and not view.value(alias, "fallback"):
|
|
150
|
+
warnings.append(f"{path}: fallback regions are listed but there is no `fallback` rule")
|
|
151
|
+
for dep in deployments:
|
|
152
|
+
if view.outside_residency(dep):
|
|
153
|
+
reason = "routes_to is missing" if dep.profile_id and not dep.routes_to else "region"
|
|
154
|
+
warnings.append(f"{path} @ {dep.key}: outside the residency list ({reason}); unused")
|
|
155
|
+
for dep in deployments:
|
|
156
|
+
rpm = view.value(alias, "rpm", dep.key)
|
|
157
|
+
shared = view.registry.accounts[dep.account].settings.get("shared_rpm")
|
|
158
|
+
if shared is not None and rpm is not None and shared < rpm:
|
|
159
|
+
warnings.append(f"accounts.{dep.account}: `shared_rpm` is below {alias} @ {dep.key}")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _check_registry(view: ResolvedRegistry) -> tuple[list[str], list[str]]:
|
|
163
|
+
errors: list[str] = []
|
|
164
|
+
warnings: list[str] = []
|
|
165
|
+
registry = view.registry
|
|
166
|
+
ids: dict[str, str] = {}
|
|
167
|
+
queues: dict[str, str] = {}
|
|
168
|
+
for alias, entry in registry.models.items():
|
|
169
|
+
if entry.model_id in ids:
|
|
170
|
+
errors.append(
|
|
171
|
+
f"models.{alias}.model_id: `{entry.model_id}` already belongs to "
|
|
172
|
+
f"`{ids[entry.model_id]}` (one alias per model id)"
|
|
173
|
+
)
|
|
174
|
+
ids.setdefault(entry.model_id, alias)
|
|
175
|
+
name = view.queue_name(alias)
|
|
176
|
+
for queue in (name, f"{name}-priority"):
|
|
177
|
+
if queue in queues:
|
|
178
|
+
errors.append(f"models.{alias}: queue `{queue}` is also used by {queues[queue]}")
|
|
179
|
+
queues[queue] = alias
|
|
180
|
+
for alias in registry.models:
|
|
181
|
+
if alias in ids and ids[alias] != alias:
|
|
182
|
+
errors.append(f"models.{alias}: the alias equals the model id of `{ids[alias]}`")
|
|
183
|
+
_check_model(view, alias, errors, warnings)
|
|
184
|
+
for account in registry.routing.blocked_accounts:
|
|
185
|
+
if account not in registry.accounts:
|
|
186
|
+
warnings.append(f"routing.blocked_accounts: `{account}` is not an account")
|
|
187
|
+
# One shared account budget, one window: a fixed and a smooth window (or two bursts) on the
|
|
188
|
+
# same counter each enforced their own view of it, and 4 shared RPM let 8 calls through.
|
|
189
|
+
for name, account_entry in registry.accounts.items():
|
|
190
|
+
settings = account_entry.settings
|
|
191
|
+
if settings.get("shared_rpm") is None and settings.get("shared_tpm") is None:
|
|
192
|
+
continue
|
|
193
|
+
windows = {
|
|
194
|
+
(view.value(alias, "rate_window", d.key), view.value(alias, "rate_burst", d.key))
|
|
195
|
+
for alias in registry.models
|
|
196
|
+
for d in view.deployments(alias)
|
|
197
|
+
if d.account == name
|
|
198
|
+
}
|
|
199
|
+
if len(windows) > 1:
|
|
200
|
+
errors.append(
|
|
201
|
+
f"accounts.{name}: deployments sharing `shared_rpm` / `shared_tpm` must use one "
|
|
202
|
+
"`rate_window` and `rate_burst` (set them on the account)"
|
|
203
|
+
)
|
|
204
|
+
return errors, warnings
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def check(
|
|
208
|
+
raw: Mapping[str, Any] | str | bytes,
|
|
209
|
+
*,
|
|
210
|
+
strict: bool = True,
|
|
211
|
+
allowed_hosts: tuple[str, ...] = (),
|
|
212
|
+
) -> tuple[Registry | None, list[str], list[str]]:
|
|
213
|
+
"""Parse, resolve and check a document; return (registry or None, errors, warnings)."""
|
|
214
|
+
if isinstance(raw, str | bytes):
|
|
215
|
+
try:
|
|
216
|
+
raw = orjson.loads(raw)
|
|
217
|
+
except orjson.JSONDecodeError as exc:
|
|
218
|
+
return None, [f"registry is not valid JSON: {exc}"], []
|
|
219
|
+
if not isinstance(raw, Mapping):
|
|
220
|
+
return None, ["registry must be a JSON object"], []
|
|
221
|
+
registry, errors, warnings = build(raw, strict=strict, allowed_hosts=allowed_hosts)
|
|
222
|
+
if registry is None:
|
|
223
|
+
return None, errors, warnings
|
|
224
|
+
more_errors, more_warnings = _check_registry(ResolvedRegistry(registry))
|
|
225
|
+
warnings = warnings + more_warnings
|
|
226
|
+
if more_errors:
|
|
227
|
+
return None, more_errors, warnings
|
|
228
|
+
return _with_warnings(registry, warnings), [], warnings
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _with_warnings(registry: Registry, warnings: list[str]) -> Registry:
|
|
232
|
+
return Registry(
|
|
233
|
+
version=registry.version,
|
|
234
|
+
defaults=registry.defaults,
|
|
235
|
+
routing=registry.routing,
|
|
236
|
+
accounts=registry.accounts,
|
|
237
|
+
models=registry.models,
|
|
238
|
+
residency=registry.residency,
|
|
239
|
+
min_lib_version=registry.min_lib_version,
|
|
240
|
+
warnings=tuple(warnings),
|
|
241
|
+
raw=registry.raw,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def load_registry(
|
|
246
|
+
raw: Mapping[str, Any] | str | bytes,
|
|
247
|
+
*,
|
|
248
|
+
strict: bool = True,
|
|
249
|
+
allowed_hosts: tuple[str, ...] = (),
|
|
250
|
+
) -> ResolvedRegistry:
|
|
251
|
+
"""Load and check a registry document; raise `RegistryError` with every problem.
|
|
252
|
+
|
|
253
|
+
`strict=False` is for a document read back from the store: unknown names become warnings.
|
|
254
|
+
"""
|
|
255
|
+
registry, errors, warnings = check(raw, strict=strict, allowed_hosts=allowed_hosts)
|
|
256
|
+
if registry is None:
|
|
257
|
+
raise RegistryError(errors, warnings)
|
|
258
|
+
return ResolvedRegistry(registry)
|
llm_queue/tuning.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Tuning: the library's timeouts, pool sizes and pacing, passed in code with safe defaults.
|
|
2
|
+
|
|
3
|
+
Each value used to be a constant in its module. An app passes `Tuning` to
|
|
4
|
+
`LLMQueue.from_env(tuning=...)`, and only the values it sets change; every default is today's
|
|
5
|
+
measured value (the comment next to each says why). `problems()` lists every bad value and
|
|
6
|
+
every bad combination at once, and `from_env` refuses to start on any (`ConfigError`), so a
|
|
7
|
+
bad value never reaches a live call.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
from dataclasses import dataclass, field, fields
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class StoreOptions:
|
|
19
|
+
"""The Valkey / Redis client of every process (`engine/store_client.py`).
|
|
20
|
+
|
|
21
|
+
The pool size stays `LLMQ_STORE_MAX_CONNECTIONS`: it follows the deployment's slots.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
retries: int = 3 # with the jittered backoff: about 1.5 s before a call sees the error
|
|
25
|
+
connect_timeout_secs: float = 5.0 # a node that does not answer in a failover fails fast
|
|
26
|
+
# A reply that never comes (half-open connection, packets dropped in a failover) fails the
|
|
27
|
+
# command instead of hanging its task: one-hour soak, two calls held for 50 minutes. Must
|
|
28
|
+
# stay above every blocking command (BLPOP, WAIT, XREAD BLOCK): see `problems()`.
|
|
29
|
+
read_timeout_secs: float = 15.0
|
|
30
|
+
health_check_secs: float = 30.0 # a connection idle this long is pinged before use
|
|
31
|
+
pool_wait_secs: float = 10.0 # a burst waits this long for a free connection
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class BedrockOptions:
|
|
36
|
+
"""The Bedrock and STS clients (`clients/pool.py`)."""
|
|
37
|
+
|
|
38
|
+
read_timeout_secs: float = 300.0 # a long generation; each attempt has its own timer too
|
|
39
|
+
connect_timeout_secs: float = 10.0
|
|
40
|
+
max_pool_connections: int = 100 # per client (one per account x region)
|
|
41
|
+
role_session_secs: int = 3600 # assumed-role session; 3600 is the role-chaining maximum
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class EngineTimings:
|
|
46
|
+
"""Pacing of the engine's loops (`engine/runtime.py`, `engine/streaming.py`)."""
|
|
47
|
+
|
|
48
|
+
# Receive loop when idle: every 50 ms, then every 1 s after 20 empty polls. A send rings
|
|
49
|
+
# its queue's doorbell, which wakes a blocked worker at once, so slow polls add no latency
|
|
50
|
+
# (idle polling cost ~96 receive scripts/s per pod with 4 models, measured).
|
|
51
|
+
idle_poll_secs: float = 0.05
|
|
52
|
+
idle_slow_secs: float = 1.0
|
|
53
|
+
idle_after_empty: int = 20
|
|
54
|
+
result_recheck_secs: float = 60.0 # a waiting caller's last-resort read of its result
|
|
55
|
+
# A store probe slower than this reports "down": a blackholed store made readiness take
|
|
56
|
+
# 41 s; a cold TLS connect to a remote ElastiCache took 1.2 s (measured).
|
|
57
|
+
health_probe_secs: float = 3.0
|
|
58
|
+
stream_read_block_ms: int = 100 # an opened chunk stream joins the shared read within this
|
|
59
|
+
# Admission scripts (engine/_admission_lua.py), sent with every call.
|
|
60
|
+
admission_probe_wait_ms: int = 200 # a breaker's probe is in flight: check again after this
|
|
61
|
+
admission_slot_wait_ms: int = 50 # no free slot: retry after this ...
|
|
62
|
+
admission_slot_jitter_ms: int = 150 # ... plus up to this, at random (spreads the retries)
|
|
63
|
+
admission_wait_ms: int = 100 # a refusal with no better estimate retries after this
|
|
64
|
+
limiter_cut_ms: int = 1000 # a throttle cuts the adaptive limit at most once per this
|
|
65
|
+
limiter_cut_factor: float = 0.7 # ... to this fraction of itself (0-1)
|
|
66
|
+
budget_key_ms: int = 180_000 # budget and limiter keys live this long (over 2 minutes)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class RegistryOptions:
|
|
71
|
+
"""The registry store (`registry/store.py`).
|
|
72
|
+
|
|
73
|
+
Audit retention stays `LLMQ_AUDIT_RETENTION_DAYS`.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
history_size: int = 20 # versions kept for `registry rollback`
|
|
77
|
+
edit_retries: int = 3 # a live edit that raced another one is retried this often
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class Tuning:
|
|
82
|
+
"""All of the above; `Tuning()` is the defaults. Passed to `LLMQueue.from_env(tuning=)`."""
|
|
83
|
+
|
|
84
|
+
store: StoreOptions = field(default_factory=StoreOptions)
|
|
85
|
+
bedrock: BedrockOptions = field(default_factory=BedrockOptions)
|
|
86
|
+
engine: EngineTimings = field(default_factory=EngineTimings)
|
|
87
|
+
registry: RegistryOptions = field(default_factory=RegistryOptions)
|
|
88
|
+
|
|
89
|
+
def problems(self, *, queue_wait_ms: int = 0) -> list[str]:
|
|
90
|
+
"""Every bad value and bad combination, one line each (empty when all is well).
|
|
91
|
+
|
|
92
|
+
`queue_wait_ms` is `LLMQ_QUEUE_WAIT_MS`: WAIT blocks a connection that long, so the
|
|
93
|
+
store read timeout must be above it.
|
|
94
|
+
"""
|
|
95
|
+
out: list[str] = []
|
|
96
|
+
for part in (self.store, self.bedrock, self.engine, self.registry):
|
|
97
|
+
out += _check_part(part)
|
|
98
|
+
# Each combination is checked once its own fields are valid (a bad field is already
|
|
99
|
+
# listed; comparing it would only repeat the same problem).
|
|
100
|
+
bad = {line.split(" ", 1)[0] for line in out}
|
|
101
|
+
store, engine = self.store, self.engine
|
|
102
|
+
blocking = {
|
|
103
|
+
"engine.idle_slow_secs": ("BLPOP", engine.idle_slow_secs),
|
|
104
|
+
"LLMQ_QUEUE_WAIT_MS": ("WAIT", queue_wait_ms / 1000),
|
|
105
|
+
"engine.stream_read_block_ms": ("XREAD BLOCK", engine.stream_read_block_ms / 1000),
|
|
106
|
+
}
|
|
107
|
+
for name, (command, secs) in blocking.items():
|
|
108
|
+
if {"store.read_timeout_secs", name} & bad:
|
|
109
|
+
continue
|
|
110
|
+
if store.read_timeout_secs < secs + 1:
|
|
111
|
+
out.append(
|
|
112
|
+
f"store.read_timeout_secs ({store.read_timeout_secs:g}) must be at least 1 s "
|
|
113
|
+
f"above {name} ({command}, {secs:g} s), or that command times out"
|
|
114
|
+
)
|
|
115
|
+
if "engine.limiter_cut_factor" not in bad and engine.limiter_cut_factor >= 1:
|
|
116
|
+
out.append("engine.limiter_cut_factor must be below 1 (a cut makes the limit smaller)")
|
|
117
|
+
if "engine.budget_key_ms" not in bad and engine.budget_key_ms < 130_000:
|
|
118
|
+
out.append(
|
|
119
|
+
"engine.budget_key_ms must be at least 130000 (a fixed window spans 2 minutes)"
|
|
120
|
+
)
|
|
121
|
+
idle = {"engine.idle_poll_secs", "engine.idle_slow_secs"}
|
|
122
|
+
if not idle & bad and engine.idle_poll_secs > engine.idle_slow_secs:
|
|
123
|
+
out.append("engine.idle_poll_secs must not be above engine.idle_slow_secs")
|
|
124
|
+
session = self.bedrock.role_session_secs
|
|
125
|
+
if "bedrock.role_session_secs" not in bad and not 900 <= session <= 43200:
|
|
126
|
+
out.append("bedrock.role_session_secs must be 900 to 43200 (STS limits)")
|
|
127
|
+
return out
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _check_part(part: Any) -> list[str]:
|
|
131
|
+
"""Type and range of each field: ints are whole numbers, every number finite and above 0."""
|
|
132
|
+
section = {
|
|
133
|
+
StoreOptions: "store",
|
|
134
|
+
BedrockOptions: "bedrock",
|
|
135
|
+
EngineTimings: "engine",
|
|
136
|
+
RegistryOptions: "registry",
|
|
137
|
+
}[type(part)]
|
|
138
|
+
out: list[str] = []
|
|
139
|
+
for spec in fields(part):
|
|
140
|
+
value = getattr(part, spec.name)
|
|
141
|
+
name = f"{section}.{spec.name}"
|
|
142
|
+
whole = spec.type in ("int", int)
|
|
143
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
144
|
+
out.append(f"{name} must be a number, not {type(value).__name__}")
|
|
145
|
+
elif whole and not isinstance(value, int):
|
|
146
|
+
out.append(f"{name} must be a whole number")
|
|
147
|
+
elif not math.isfinite(value):
|
|
148
|
+
out.append(f"{name} must be finite")
|
|
149
|
+
elif value < 0 or (value == 0 and spec.name != "retries"):
|
|
150
|
+
out.append(f"{name} must be {'0 or more' if spec.name == 'retries' else 'above 0'}")
|
|
151
|
+
return out
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: llm-queue
|
|
3
|
+
Version: 0.1.0b1
|
|
4
|
+
Summary: LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status.
|
|
5
|
+
Project-URL: Homepage, https://github.com/techiewonk/llm-queue
|
|
6
|
+
Project-URL: Documentation, https://techiewonk.github.io/llm-queue/
|
|
7
|
+
Project-URL: Source, https://github.com/techiewonk/llm-queue
|
|
8
|
+
Project-URL: Issues, https://github.com/techiewonk/llm-queue/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md
|
|
10
|
+
Author: The llm-queue authors
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
License-File: NOTICE
|
|
14
|
+
Keywords: bedrock,langchain,langgraph,llm,openai,queue,rate-limit,redis,valkey
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Framework :: AsyncIO
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.11
|
|
29
|
+
Requires-Dist: aiobotocore<4,>=2.23
|
|
30
|
+
Requires-Dist: hiredis<4,>=3
|
|
31
|
+
Requires-Dist: orjson<4,>=3.10
|
|
32
|
+
Requires-Dist: pydantic<3,>=2.10
|
|
33
|
+
Requires-Dist: redis<9,>=5.0.1
|
|
34
|
+
Requires-Dist: structlog<27,>=24.1
|
|
35
|
+
Requires-Dist: uuid-utils<1,>=0.19
|
|
36
|
+
Provides-Extra: http
|
|
37
|
+
Requires-Dist: fastapi<1,>=0.115.3; extra == 'http'
|
|
38
|
+
Provides-Extra: langchain-aws
|
|
39
|
+
Requires-Dist: langchain-aws<2,>=0.2.24; extra == 'langchain-aws'
|
|
40
|
+
Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-aws'
|
|
41
|
+
Provides-Extra: langchain-openai
|
|
42
|
+
Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-openai'
|
|
43
|
+
Requires-Dist: langchain-openai<2,>=0.3.26; extra == 'langchain-openai'
|
|
44
|
+
Provides-Extra: metrics
|
|
45
|
+
Requires-Dist: prometheus-client<1,>=0.20; extra == 'metrics'
|
|
46
|
+
Provides-Extra: openai
|
|
47
|
+
Requires-Dist: httpx<1,>=0.27; extra == 'openai'
|
|
48
|
+
Requires-Dist: openai<4,>=1.40; extra == 'openai'
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
# llm-queue
|
|
52
|
+
|
|
53
|
+
LLM calls through a queue on Valkey or Redis, under RPM / TPM budgets per **deployment**
|
|
54
|
+
(model x account x region), balanced across regions and accounts, with fallback regions,
|
|
55
|
+
fallback models, health checks and status. Works with Amazon Bedrock and OpenAI-compatible
|
|
56
|
+
servers (vLLM), and with LangChain. The queue, the budgets and the results live in one store, so
|
|
57
|
+
admission (deployment, RPM, TPM, concurrency slot) is one atomic step, and 1 pod behaves the
|
|
58
|
+
same as 50.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
pip install "llm-queue[langchain-aws]" # Bedrock + LangChain
|
|
64
|
+
pip install "llm-queue[openai,langchain-openai]" # OpenAI-compatible
|
|
65
|
+
pip install "llm-queue[http]" # status and health routes for FastAPI
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Example
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from llm_queue import LLMQueue, CallFailed, request_context
|
|
72
|
+
|
|
73
|
+
llmq = LLMQueue.from_env() # LLMQ_STORE_URL, LLMQ_REGISTRY, ...
|
|
74
|
+
await llmq.start() # publish / load the registry, run the workers
|
|
75
|
+
try:
|
|
76
|
+
with request_context(priority=True, user_id=uid):
|
|
77
|
+
reply = await llmq.converse("gpt120b", {"messages": [...]}, max_wait_secs=5)
|
|
78
|
+
print(reply.response, reply.meta.deployment)
|
|
79
|
+
except CallFailed as exc: # exc.kind: rate_limited, model_unavailable, ...
|
|
80
|
+
...
|
|
81
|
+
await llmq.stop()
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Documentation
|
|
85
|
+
|
|
86
|
+
Full documentation: **https://techiewonk.github.io/llm-queue/**
|
|
87
|
+
|
|
88
|
+
- [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
|
|
89
|
+
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, tune, call, operate, share models across services, test
|
|
90
|
+
- [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, failures and recovery
|
|
91
|
+
- [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): settings, error kinds, CLI and HTTP, the Python API
|
|
92
|
+
|
|
93
|
+
Project: [CHANGELOG](https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md),
|
|
94
|
+
[CONTRIBUTING](https://github.com/techiewonk/llm-queue/blob/main/CONTRIBUTING.md),
|
|
95
|
+
[STABILITY](https://github.com/techiewonk/llm-queue/blob/main/STABILITY.md),
|
|
96
|
+
[SECURITY](https://github.com/techiewonk/llm-queue/blob/main/SECURITY.md).
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
llm_queue/__init__.py,sha256=Zxs0rVMOU9kyHt6OARKrXJY4SQcpnQAxf-Km5CLdg7o,3032
|
|
2
|
+
llm_queue/_logging.py,sha256=ZVfyoCVewvBlJrsgyoOlLu_r9BCf0jxG9L2yN5jaugw,1744
|
|
3
|
+
llm_queue/_version.py,sha256=y4Ors2m0uN5McJDFlxvLwtvSXUmyn0Sa-AVtD97RekI,150
|
|
4
|
+
llm_queue/context.py,sha256=v05RkZ5QEu4WWYYKHzDhamKqb9ylRXXdtyKIRRveiPc,1776
|
|
5
|
+
llm_queue/core.py,sha256=9IpsrY1B7_yv7yKDar06B-QQGZKXVccbLMUFkt3vu3E,19112
|
|
6
|
+
llm_queue/errors.py,sha256=njaJsYbcyCzqVzHkylW8zm1fXhdq8gVxXgtwvLvUIdo,2481
|
|
7
|
+
llm_queue/http_transport.py,sha256=YIhRQgHaxl5dt3Oc8Bz3EYeF6SnAUx_0lHnd-ZTqbIA,9139
|
|
8
|
+
llm_queue/ids.py,sha256=qc7bRfXI2gpo0EqlfO2xsUKN3tLWjeNa6YFY4wEydu8,646
|
|
9
|
+
llm_queue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
llm_queue/tuning.py,sha256=x_rvifmLih05nsL1wPq_hlxYV-acp56YNsSx2dzH3lg,7481
|
|
11
|
+
llm_queue/clients/__init__.py,sha256=OptKjonubCxsPkgkRWwhHl0mO_k099vsmn4VN_J-x18,931
|
|
12
|
+
llm_queue/clients/pool.py,sha256=_Z_swepnNvkTjEfh22bxMN-u41PVD-R1w3TerWLCa9o,11935
|
|
13
|
+
llm_queue/clients/providers.py,sha256=p6OKuhT4VoWiUqAmAOCY_JwKl3TII7UR0rSK12DgPJU,13968
|
|
14
|
+
llm_queue/clients/timing.py,sha256=APB5qnfsNbOABKANivriQtevMnnQsr7JJ1tRbsScQsg,1860
|
|
15
|
+
llm_queue/engine/__init__.py,sha256=ptlS44P0NkZv31R2PIDXwvbnwjS_LAHGTziaEKZQsdE,127
|
|
16
|
+
llm_queue/engine/_admission_lua.py,sha256=P7IO17FZuXauo-QTgDCh-bsKOKFY5r_Nr-U2w7mAxQc,29452
|
|
17
|
+
llm_queue/engine/admission.py,sha256=-ax8s580Y1oMdCOhWAmnmOnyEoRY5cv7EjyRRPtYRVA,8896
|
|
18
|
+
llm_queue/engine/cli.py,sha256=8h7kcHcqGaLpzzWLL07SIm5fboE5mWuK0C8i-9grOhw,12653
|
|
19
|
+
llm_queue/engine/embed.py,sha256=b3J6apGMg3p0jE0jUdbCFSiJxqzIGL0b1PrUkXNfFPg,4043
|
|
20
|
+
llm_queue/engine/envelope.py,sha256=CI1eRJe7MxPYwwMqRyHCOV4U9zSh_mEEYMqYIldmnws,3571
|
|
21
|
+
llm_queue/engine/errors.py,sha256=PgDgsSoq1aAejClXZTnkrstOojj-yDIAsHyH0Aqcl3U,1443
|
|
22
|
+
llm_queue/engine/health.py,sha256=LnPcGksivg40fNQq1XzirDB7ZVLx4Mqyy2dkda8PCLY,5534
|
|
23
|
+
llm_queue/engine/hooks.py,sha256=uBZn41d2Kuav2yB5w7srqFatwCW2pJeQLj2aGdu0rOI,3956
|
|
24
|
+
llm_queue/engine/http.py,sha256=_uMSXA-jZn4T6P3TTjozghKDOiNAN5Zhpm85UtqQ0pw,7122
|
|
25
|
+
llm_queue/engine/inference.py,sha256=yobDfTmV4LYXKqcSQ4Wg4kqxvBULTK3eF5dQwO480bY,5098
|
|
26
|
+
llm_queue/engine/jobs.py,sha256=-3Ow11AxWeEeaUnyeKlDRpXFUI4H8EjOvAKOog1zQVA,5613
|
|
27
|
+
llm_queue/engine/keys.py,sha256=hfAtLT5-yo0pX1KPaOytnYk98MuLtBLQy4_k4liPOhM,2780
|
|
28
|
+
llm_queue/engine/metrics.py,sha256=RdA8irAcsELE13xigDSuMjp-yKS-hg710nLCSIO0pA4,4945
|
|
29
|
+
llm_queue/engine/offload.py,sha256=ccviGd0uL4w-IeHAhRrykutyGYM1c7InCbROZ2WEGOE,2787
|
|
30
|
+
llm_queue/engine/queue.py,sha256=Ji8drmV05O_i2K8idk9goSfoPZUB025LmpCEwx2qoYU,16427
|
|
31
|
+
llm_queue/engine/requests.py,sha256=GPK704xcCcBDlhi6FprH_as4thprjokHLrn2OzVtFDs,12542
|
|
32
|
+
llm_queue/engine/runtime.py,sha256=bUMeq8h-DTPqgGSQ6xp9s55fdUmzAcs1ZNjw62xixDc,107475
|
|
33
|
+
llm_queue/engine/settings.py,sha256=LdnhtRW3Cre6Iio7AvoLBqeoKsAKTHAiA8uolXPBmjo,8338
|
|
34
|
+
llm_queue/engine/status.py,sha256=ste09VeHbUa3tEXZOodqFb2qOyk9B9yrAhbQ3hdFD2Y,10866
|
|
35
|
+
llm_queue/engine/store_client.py,sha256=RlpUvdGEHIU_aKujWSPYh-WmYCxJk3ib885i8l3FR5Q,3131
|
|
36
|
+
llm_queue/engine/store_guard.py,sha256=0VbuFJpPMgIYJ7ww3rJwe7w__NAH5M9fmiCMRIGpp7I,1481
|
|
37
|
+
llm_queue/engine/streaming.py,sha256=jF1Lt8OJxxGSmMoaZ32ISZe98qZTxQGVmJwfZMEcHvA,8273
|
|
38
|
+
llm_queue/engine/tokens.py,sha256=pmTxjo5AqydXK1kwL1kjKkOMEaRmsKTC0MhTXIUR4qk,4674
|
|
39
|
+
llm_queue/langchain/__init__.py,sha256=VFBvk3FGvhzL6JXjBkou6jkmYD_DzKjEzCDetMHCPaE,2033
|
|
40
|
+
llm_queue/langchain/_client.py,sha256=QqMk-l13oAVZy7mfuFbYjM67DkAwyluu-2RHorLT274,2925
|
|
41
|
+
llm_queue/langchain/bedrock.py,sha256=t-s3u9AFPWA78g8MnfXKZwJKDv1nWxyKh9tXe14F4U8,12855
|
|
42
|
+
llm_queue/langchain/openai.py,sha256=bx0VUQy2vADKMM63GX5OSkxxbGxWOZw1D6xaqjA7bl0,4130
|
|
43
|
+
llm_queue/providers/__init__.py,sha256=fVPjKxVRQ1JJzq78oO7qQrO_dl4enHr0D6LCcKuxRzI,986
|
|
44
|
+
llm_queue/providers/classify.py,sha256=46ujPwREQwJyF7kNYJ0uXzw96kFR9B_L5YROkd-4I3M,9660
|
|
45
|
+
llm_queue/providers/embeddings.py,sha256=7i6klmXAaau_iyhFdq__ROIhw2jVEybuI4Bbvd-JId8,3371
|
|
46
|
+
llm_queue/registry/__init__.py,sha256=MEKJQKpWShJtn-YY905cn-TolCqrPChKrysQRfCudeE,945
|
|
47
|
+
llm_queue/registry/_lua.py,sha256=7gYSsjHN9_GkcyFW6JgLNUGaPbE0-661jBxU022GVEg,3367
|
|
48
|
+
llm_queue/registry/resolve.py,sha256=7B45usivJzVE45zBGOcyXJzODbUJ1RBJR_GCOOmCkAw,8461
|
|
49
|
+
llm_queue/registry/schema.py,sha256=y8-H96gnZSoehFnP-2pSj8hKJhW3PNpG2WVa9n4GjlA,20780
|
|
50
|
+
llm_queue/registry/settings.py,sha256=6yCIJieO3llcE4oKSadHmWUhOIYt7qV6LddO8MH2jHM,12826
|
|
51
|
+
llm_queue/registry/store.py,sha256=UttYtoc_1jWNPK3ICc4fX4GPKhLklwH2b9kk1UsNHls,25212
|
|
52
|
+
llm_queue/registry/validate.py,sha256=omwZl9iMPD6cYGz87SLXp2fx9oEQaY-cxFtwf3vCw4A,11833
|
|
53
|
+
llm_queue-0.1.0b1.dist-info/METADATA,sha256=l72fYvxv7JPZnLVfMyjJc61-LRBjezZlZkpVsOBG5Tg,4547
|
|
54
|
+
llm_queue-0.1.0b1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
55
|
+
llm_queue-0.1.0b1.dist-info/entry_points.txt,sha256=rObO8zmIGbJy0563G2CuUkX9S8lj2zPUs7enm-xDr9s,56
|
|
56
|
+
llm_queue-0.1.0b1.dist-info/licenses/LICENSE,sha256=OvUQrS0ASWXZWrpTi33eLZyB0SJmpWD99z-L0Lu8C8Y,1078
|
|
57
|
+
llm_queue-0.1.0b1.dist-info/licenses/NOTICE,sha256=ApdjxFQDujzw8sm8_i31dg_Hf_Tg_Lu1JBQZU6vjhZE,748
|
|
58
|
+
llm_queue-0.1.0b1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The llm-queue authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
llm-queue
|
|
2
|
+
Copyright (c) 2026 The llm-queue authors. Licensed under the MIT License.
|
|
3
|
+
|
|
4
|
+
The queue's design (visibility heartbeat, message metadata, large results in S3) builds on
|
|
5
|
+
ideas from two MIT-licensed SQS brokers. This package is an independent implementation on
|
|
6
|
+
Valkey / Redis streams; any code adapted from them keeps its original notice below.
|
|
7
|
+
|
|
8
|
+
taskiq-aio-sqs — https://github.com/vonsteer/taskiq-aio-sqs
|
|
9
|
+
Copyright (c) 2022-2024 Jesse Constante. MIT License.
|
|
10
|
+
Ideas used: visibility heartbeat, message metadata, extended (S3) messages.
|
|
11
|
+
|
|
12
|
+
taskiq-sqs — https://github.com/taskiq-python/taskiq-sqs
|
|
13
|
+
Copyright (c) 2026 Mike Shultz and Taskiq team. MIT License.
|
|
14
|
+
Ideas used: message expiry, flush on shutdown, S3 offload of large results.
|