llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,258 @@
1
+ """Cross-field checks of a registry and the one entry point that loads it.
2
+
3
+ `load_registry` parses (`schema.build`), resolves and checks the document and raises one
4
+ `RegistryError` listing every problem. The checks here need effective values, so they run on
5
+ the resolved registry (plan §3.9 and §3.19).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Mapping
11
+ from typing import Any
12
+
13
+ import orjson
14
+
15
+ from llm_queue.errors import ConfigError
16
+ from llm_queue.registry.resolve import Deployment, ResolvedRegistry
17
+ from llm_queue.registry.schema import Registry, build
18
+ from llm_queue.registry.settings import CHAT_INFERENCE, EMBED_INFERENCE
19
+
20
+
21
+ class RegistryError(ConfigError):
22
+ """The registry document is invalid; `problems` lists every problem with its path."""
23
+
24
+ def __init__(self, problems: list[str], warnings: list[str] | None = None) -> None:
25
+ """Keep the full lists; the message shows them one per line."""
26
+ self.problems = list(problems)
27
+ self.warnings = list(warnings or [])
28
+ lines = "\n ".join(self.problems)
29
+ super().__init__(f"invalid registry ({len(self.problems)} problems):\n {lines}")
30
+
31
+
32
+ def _budget_numbers(view: ResolvedRegistry, alias: str, dep: Deployment) -> tuple[Any, Any]:
33
+ return view.value(alias, "rpm", dep.key), view.value(alias, "tpm", dep.key)
34
+
35
+
36
+ def _check_model(
37
+ view: ResolvedRegistry, alias: str, errors: list[str], warnings: list[str]
38
+ ) -> None:
39
+ entry = view.registry.models[alias]
40
+ path = f"models.{alias}"
41
+ deployments = view.deployments(alias)
42
+ confirmed = [d for d in deployments if d.group == "confirmed"]
43
+
44
+ # Accounts: their kind must serve the model's provider; enabled flags only.
45
+ for dep in deployments:
46
+ account = view.registry.accounts.get(dep.account)
47
+ if account is not None and account.provider != entry.provider:
48
+ errors.append(
49
+ f"{path}.deployments.{dep.account}: a `{account.kind}` account serves "
50
+ f"`{account.provider}`, not `{entry.provider}`"
51
+ )
52
+
53
+ # Timeouts.
54
+ timeout = view.value(alias, "timeout_secs")
55
+ if timeout > view.value(alias, "timeout_max_secs"):
56
+ errors.append(f"{path}: `timeout_secs` is above `timeout_max_secs`")
57
+ for dep in deployments:
58
+ if view.value(alias, "attempt_timeout_secs", dep.key) >= view.value(
59
+ alias, "timeout_secs", dep.key
60
+ ):
61
+ errors.append(
62
+ f"{path} @ {dep.key}: `attempt_timeout_secs` must be below `timeout_secs`"
63
+ )
64
+ max_wait = view.value(alias, "max_wait_secs")
65
+ for rule in ("fallback", "model_fallback"):
66
+ value = view.value(alias, rule)
67
+ if value and max_wait is not None and value["after_secs"] >= max_wait:
68
+ errors.append(f"{path}.{rule}.after_secs: must be below `max_wait_secs`")
69
+
70
+ # Shares.
71
+ if view.value(alias, "priority_reserve") + view.value(alias, "normal_min_share") >= 1:
72
+ errors.append(f"{path}: `priority_reserve` + `normal_min_share` must be below 1")
73
+
74
+ # Budgets named by rate_limits must have a number.
75
+ for dep in deployments:
76
+ limits = view.effective(alias, "rate_limits", dep.key)
77
+ rpm, tpm = _budget_numbers(view, alias, dep)
78
+ if limits.level != "library":
79
+ if limits.value == "rpm" and rpm is None:
80
+ errors.append(f"{path} @ {dep.key}: `rate_limits` is rpm but no `rpm` is set")
81
+ if limits.value == "tpm" and tpm is None:
82
+ errors.append(f"{path} @ {dep.key}: `rate_limits` is tpm but no `tpm` is set")
83
+
84
+ # Strategies that need quota numbers.
85
+ for name in ("balance", "balance_priority"):
86
+ if view.value(alias, name) == "headroom":
87
+ bare = [d.key for d in confirmed if _budget_numbers(view, alias, d) == (None, None)]
88
+ if bare:
89
+ errors.append(f"{path}.{name}: headroom needs `rpm` or `tpm` on {', '.join(bare)}")
90
+ if view.value(alias, "balance") == "weighted":
91
+ weights = [view.value(alias, "weight", d.key) for d in confirmed]
92
+ if weights and not any(weights):
93
+ errors.append(f"{path}: every confirmed deployment has weight 0")
94
+
95
+ # Inference parameters by model type and provider; defaults within their limits.
96
+ wrong = EMBED_INFERENCE if entry.type == "chat" else CHAT_INFERENCE
97
+ for dep_key in (None, *(d.key for d in deployments)):
98
+ inference = view.value(alias, "inference", dep_key) or {}
99
+ where = path if dep_key is None else f"{path} @ {dep_key}"
100
+ for name in inference:
101
+ if name in wrong:
102
+ errors.append(f"{where}.inference.{name}: not a parameter of a {entry.type} model")
103
+ if "seed" in inference and entry.provider == "bedrock":
104
+ errors.append(f"{where}.inference.seed: not supported by Bedrock")
105
+ caps = view.value(alias, "inference_limits", dep_key) or {}
106
+ if "max_tokens" in caps and inference.get("max_tokens", 0) > caps["max_tokens"]:
107
+ errors.append(f"{where}.inference.max_tokens: above `inference_limits.max_tokens`")
108
+
109
+ # Rules that depend on the model type.
110
+ if entry.type == "chat" and "embed_batch_size" in entry.settings:
111
+ errors.append(f"{path}.embed_batch_size: only for embedding models")
112
+ rule = view.value(alias, "model_fallback")
113
+ if rule and entry.type == "embedding":
114
+ errors.append(f"{path}.model_fallback: an embedding model cannot change model")
115
+ elif rule:
116
+ for item in rule["models"]:
117
+ other = item["model"]
118
+ if other == alias:
119
+ errors.append(f"{path}.model_fallback: a model cannot list itself")
120
+ continue
121
+ if other not in view.registry.models:
122
+ errors.append(f"{path}.model_fallback: unknown model `{other}`")
123
+ continue
124
+ target = view.registry.models[other]
125
+ if target.type != "chat":
126
+ errors.append(f"{path}.model_fallback: `{other}` is not a chat model")
127
+ back = view.value(other, "model_fallback")
128
+ if back:
129
+ mutual = [m for m in back["models"] if m["model"] == alias]
130
+ if mutual and not (item["require_headroom"] and mutual[0]["require_headroom"]):
131
+ errors.append(
132
+ f"{path}.model_fallback: `{alias}` and `{other}` list each other; "
133
+ "both must use require_headroom"
134
+ )
135
+ if item["require_headroom"]:
136
+ bare = [
137
+ d.key
138
+ for d in view.deployments(other, "confirmed")
139
+ if _budget_numbers(view, other, d) == (None, None)
140
+ ]
141
+ if bare:
142
+ errors.append(
143
+ f"{path}.model_fallback: require_headroom on `{other}` needs `rpm` or "
144
+ f"`tpm` on {', '.join(bare)}"
145
+ )
146
+
147
+ # Warnings (not fatal).
148
+ fallback_deps = [d for d in deployments if d.group == "fallback"]
149
+ if fallback_deps and not view.value(alias, "fallback"):
150
+ warnings.append(f"{path}: fallback regions are listed but there is no `fallback` rule")
151
+ for dep in deployments:
152
+ if view.outside_residency(dep):
153
+ reason = "routes_to is missing" if dep.profile_id and not dep.routes_to else "region"
154
+ warnings.append(f"{path} @ {dep.key}: outside the residency list ({reason}); unused")
155
+ for dep in deployments:
156
+ rpm = view.value(alias, "rpm", dep.key)
157
+ shared = view.registry.accounts[dep.account].settings.get("shared_rpm")
158
+ if shared is not None and rpm is not None and shared < rpm:
159
+ warnings.append(f"accounts.{dep.account}: `shared_rpm` is below {alias} @ {dep.key}")
160
+
161
+
162
+ def _check_registry(view: ResolvedRegistry) -> tuple[list[str], list[str]]:
163
+ errors: list[str] = []
164
+ warnings: list[str] = []
165
+ registry = view.registry
166
+ ids: dict[str, str] = {}
167
+ queues: dict[str, str] = {}
168
+ for alias, entry in registry.models.items():
169
+ if entry.model_id in ids:
170
+ errors.append(
171
+ f"models.{alias}.model_id: `{entry.model_id}` already belongs to "
172
+ f"`{ids[entry.model_id]}` (one alias per model id)"
173
+ )
174
+ ids.setdefault(entry.model_id, alias)
175
+ name = view.queue_name(alias)
176
+ for queue in (name, f"{name}-priority"):
177
+ if queue in queues:
178
+ errors.append(f"models.{alias}: queue `{queue}` is also used by {queues[queue]}")
179
+ queues[queue] = alias
180
+ for alias in registry.models:
181
+ if alias in ids and ids[alias] != alias:
182
+ errors.append(f"models.{alias}: the alias equals the model id of `{ids[alias]}`")
183
+ _check_model(view, alias, errors, warnings)
184
+ for account in registry.routing.blocked_accounts:
185
+ if account not in registry.accounts:
186
+ warnings.append(f"routing.blocked_accounts: `{account}` is not an account")
187
+ # One shared account budget, one window: a fixed and a smooth window (or two bursts) on the
188
+ # same counter each enforced their own view of it, and 4 shared RPM let 8 calls through.
189
+ for name, account_entry in registry.accounts.items():
190
+ settings = account_entry.settings
191
+ if settings.get("shared_rpm") is None and settings.get("shared_tpm") is None:
192
+ continue
193
+ windows = {
194
+ (view.value(alias, "rate_window", d.key), view.value(alias, "rate_burst", d.key))
195
+ for alias in registry.models
196
+ for d in view.deployments(alias)
197
+ if d.account == name
198
+ }
199
+ if len(windows) > 1:
200
+ errors.append(
201
+ f"accounts.{name}: deployments sharing `shared_rpm` / `shared_tpm` must use one "
202
+ "`rate_window` and `rate_burst` (set them on the account)"
203
+ )
204
+ return errors, warnings
205
+
206
+
207
+ def check(
208
+ raw: Mapping[str, Any] | str | bytes,
209
+ *,
210
+ strict: bool = True,
211
+ allowed_hosts: tuple[str, ...] = (),
212
+ ) -> tuple[Registry | None, list[str], list[str]]:
213
+ """Parse, resolve and check a document; return (registry or None, errors, warnings)."""
214
+ if isinstance(raw, str | bytes):
215
+ try:
216
+ raw = orjson.loads(raw)
217
+ except orjson.JSONDecodeError as exc:
218
+ return None, [f"registry is not valid JSON: {exc}"], []
219
+ if not isinstance(raw, Mapping):
220
+ return None, ["registry must be a JSON object"], []
221
+ registry, errors, warnings = build(raw, strict=strict, allowed_hosts=allowed_hosts)
222
+ if registry is None:
223
+ return None, errors, warnings
224
+ more_errors, more_warnings = _check_registry(ResolvedRegistry(registry))
225
+ warnings = warnings + more_warnings
226
+ if more_errors:
227
+ return None, more_errors, warnings
228
+ return _with_warnings(registry, warnings), [], warnings
229
+
230
+
231
+ def _with_warnings(registry: Registry, warnings: list[str]) -> Registry:
232
+ return Registry(
233
+ version=registry.version,
234
+ defaults=registry.defaults,
235
+ routing=registry.routing,
236
+ accounts=registry.accounts,
237
+ models=registry.models,
238
+ residency=registry.residency,
239
+ min_lib_version=registry.min_lib_version,
240
+ warnings=tuple(warnings),
241
+ raw=registry.raw,
242
+ )
243
+
244
+
245
+ def load_registry(
246
+ raw: Mapping[str, Any] | str | bytes,
247
+ *,
248
+ strict: bool = True,
249
+ allowed_hosts: tuple[str, ...] = (),
250
+ ) -> ResolvedRegistry:
251
+ """Load and check a registry document; raise `RegistryError` with every problem.
252
+
253
+ `strict=False` is for a document read back from the store: unknown names become warnings.
254
+ """
255
+ registry, errors, warnings = check(raw, strict=strict, allowed_hosts=allowed_hosts)
256
+ if registry is None:
257
+ raise RegistryError(errors, warnings)
258
+ return ResolvedRegistry(registry)
llm_queue/tuning.py ADDED
@@ -0,0 +1,151 @@
1
+ """Tuning: the library's timeouts, pool sizes and pacing, passed in code with safe defaults.
2
+
3
+ Each value used to be a constant in its module. An app passes `Tuning` to
4
+ `LLMQueue.from_env(tuning=...)`, and only the values it sets change; every default is today's
5
+ measured value (the comment next to each says why). `problems()` lists every bad value and
6
+ every bad combination at once, and `from_env` refuses to start on any (`ConfigError`), so a
7
+ bad value never reaches a live call.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import math
13
+ from dataclasses import dataclass, field, fields
14
+ from typing import Any
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class StoreOptions:
19
+ """The Valkey / Redis client of every process (`engine/store_client.py`).
20
+
21
+ The pool size stays `LLMQ_STORE_MAX_CONNECTIONS`: it follows the deployment's slots.
22
+ """
23
+
24
+ retries: int = 3 # with the jittered backoff: about 1.5 s before a call sees the error
25
+ connect_timeout_secs: float = 5.0 # a node that does not answer in a failover fails fast
26
+ # A reply that never comes (half-open connection, packets dropped in a failover) fails the
27
+ # command instead of hanging its task: one-hour soak, two calls held for 50 minutes. Must
28
+ # stay above every blocking command (BLPOP, WAIT, XREAD BLOCK): see `problems()`.
29
+ read_timeout_secs: float = 15.0
30
+ health_check_secs: float = 30.0 # a connection idle this long is pinged before use
31
+ pool_wait_secs: float = 10.0 # a burst waits this long for a free connection
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class BedrockOptions:
36
+ """The Bedrock and STS clients (`clients/pool.py`)."""
37
+
38
+ read_timeout_secs: float = 300.0 # a long generation; each attempt has its own timer too
39
+ connect_timeout_secs: float = 10.0
40
+ max_pool_connections: int = 100 # per client (one per account x region)
41
+ role_session_secs: int = 3600 # assumed-role session; 3600 is the role-chaining maximum
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class EngineTimings:
46
+ """Pacing of the engine's loops (`engine/runtime.py`, `engine/streaming.py`)."""
47
+
48
+ # Receive loop when idle: every 50 ms, then every 1 s after 20 empty polls. A send rings
49
+ # its queue's doorbell, which wakes a blocked worker at once, so slow polls add no latency
50
+ # (idle polling cost ~96 receive scripts/s per pod with 4 models, measured).
51
+ idle_poll_secs: float = 0.05
52
+ idle_slow_secs: float = 1.0
53
+ idle_after_empty: int = 20
54
+ result_recheck_secs: float = 60.0 # a waiting caller's last-resort read of its result
55
+ # A store probe slower than this reports "down": a blackholed store made readiness take
56
+ # 41 s; a cold TLS connect to a remote ElastiCache took 1.2 s (measured).
57
+ health_probe_secs: float = 3.0
58
+ stream_read_block_ms: int = 100 # an opened chunk stream joins the shared read within this
59
+ # Admission scripts (engine/_admission_lua.py), sent with every call.
60
+ admission_probe_wait_ms: int = 200 # a breaker's probe is in flight: check again after this
61
+ admission_slot_wait_ms: int = 50 # no free slot: retry after this ...
62
+ admission_slot_jitter_ms: int = 150 # ... plus up to this, at random (spreads the retries)
63
+ admission_wait_ms: int = 100 # a refusal with no better estimate retries after this
64
+ limiter_cut_ms: int = 1000 # a throttle cuts the adaptive limit at most once per this
65
+ limiter_cut_factor: float = 0.7 # ... to this fraction of itself (0-1)
66
+ budget_key_ms: int = 180_000 # budget and limiter keys live this long (over 2 minutes)
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class RegistryOptions:
71
+ """The registry store (`registry/store.py`).
72
+
73
+ Audit retention stays `LLMQ_AUDIT_RETENTION_DAYS`.
74
+ """
75
+
76
+ history_size: int = 20 # versions kept for `registry rollback`
77
+ edit_retries: int = 3 # a live edit that raced another one is retried this often
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class Tuning:
82
+ """All of the above; `Tuning()` is the defaults. Passed to `LLMQueue.from_env(tuning=)`."""
83
+
84
+ store: StoreOptions = field(default_factory=StoreOptions)
85
+ bedrock: BedrockOptions = field(default_factory=BedrockOptions)
86
+ engine: EngineTimings = field(default_factory=EngineTimings)
87
+ registry: RegistryOptions = field(default_factory=RegistryOptions)
88
+
89
+ def problems(self, *, queue_wait_ms: int = 0) -> list[str]:
90
+ """Every bad value and bad combination, one line each (empty when all is well).
91
+
92
+ `queue_wait_ms` is `LLMQ_QUEUE_WAIT_MS`: WAIT blocks a connection that long, so the
93
+ store read timeout must be above it.
94
+ """
95
+ out: list[str] = []
96
+ for part in (self.store, self.bedrock, self.engine, self.registry):
97
+ out += _check_part(part)
98
+ # Each combination is checked once its own fields are valid (a bad field is already
99
+ # listed; comparing it would only repeat the same problem).
100
+ bad = {line.split(" ", 1)[0] for line in out}
101
+ store, engine = self.store, self.engine
102
+ blocking = {
103
+ "engine.idle_slow_secs": ("BLPOP", engine.idle_slow_secs),
104
+ "LLMQ_QUEUE_WAIT_MS": ("WAIT", queue_wait_ms / 1000),
105
+ "engine.stream_read_block_ms": ("XREAD BLOCK", engine.stream_read_block_ms / 1000),
106
+ }
107
+ for name, (command, secs) in blocking.items():
108
+ if {"store.read_timeout_secs", name} & bad:
109
+ continue
110
+ if store.read_timeout_secs < secs + 1:
111
+ out.append(
112
+ f"store.read_timeout_secs ({store.read_timeout_secs:g}) must be at least 1 s "
113
+ f"above {name} ({command}, {secs:g} s), or that command times out"
114
+ )
115
+ if "engine.limiter_cut_factor" not in bad and engine.limiter_cut_factor >= 1:
116
+ out.append("engine.limiter_cut_factor must be below 1 (a cut makes the limit smaller)")
117
+ if "engine.budget_key_ms" not in bad and engine.budget_key_ms < 130_000:
118
+ out.append(
119
+ "engine.budget_key_ms must be at least 130000 (a fixed window spans 2 minutes)"
120
+ )
121
+ idle = {"engine.idle_poll_secs", "engine.idle_slow_secs"}
122
+ if not idle & bad and engine.idle_poll_secs > engine.idle_slow_secs:
123
+ out.append("engine.idle_poll_secs must not be above engine.idle_slow_secs")
124
+ session = self.bedrock.role_session_secs
125
+ if "bedrock.role_session_secs" not in bad and not 900 <= session <= 43200:
126
+ out.append("bedrock.role_session_secs must be 900 to 43200 (STS limits)")
127
+ return out
128
+
129
+
130
+ def _check_part(part: Any) -> list[str]:
131
+ """Type and range of each field: ints are whole numbers, every number finite and above 0."""
132
+ section = {
133
+ StoreOptions: "store",
134
+ BedrockOptions: "bedrock",
135
+ EngineTimings: "engine",
136
+ RegistryOptions: "registry",
137
+ }[type(part)]
138
+ out: list[str] = []
139
+ for spec in fields(part):
140
+ value = getattr(part, spec.name)
141
+ name = f"{section}.{spec.name}"
142
+ whole = spec.type in ("int", int)
143
+ if isinstance(value, bool) or not isinstance(value, int | float):
144
+ out.append(f"{name} must be a number, not {type(value).__name__}")
145
+ elif whole and not isinstance(value, int):
146
+ out.append(f"{name} must be a whole number")
147
+ elif not math.isfinite(value):
148
+ out.append(f"{name} must be finite")
149
+ elif value < 0 or (value == 0 and spec.name != "retries"):
150
+ out.append(f"{name} must be {'0 or more' if spec.name == 'retries' else 'above 0'}")
151
+ return out
@@ -0,0 +1,96 @@
1
+ Metadata-Version: 2.5
2
+ Name: llm-queue
3
+ Version: 0.1.0b1
4
+ Summary: LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status.
5
+ Project-URL: Homepage, https://github.com/techiewonk/llm-queue
6
+ Project-URL: Documentation, https://techiewonk.github.io/llm-queue/
7
+ Project-URL: Source, https://github.com/techiewonk/llm-queue
8
+ Project-URL: Issues, https://github.com/techiewonk/llm-queue/issues
9
+ Project-URL: Changelog, https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md
10
+ Author: The llm-queue authors
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ License-File: NOTICE
14
+ Keywords: bedrock,langchain,langgraph,llm,openai,queue,rate-limit,redis,valkey
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Framework :: AsyncIO
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
25
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
26
+ Classifier: Topic :: System :: Distributed Computing
27
+ Classifier: Typing :: Typed
28
+ Requires-Python: >=3.11
29
+ Requires-Dist: aiobotocore<4,>=2.23
30
+ Requires-Dist: hiredis<4,>=3
31
+ Requires-Dist: orjson<4,>=3.10
32
+ Requires-Dist: pydantic<3,>=2.10
33
+ Requires-Dist: redis<9,>=5.0.1
34
+ Requires-Dist: structlog<27,>=24.1
35
+ Requires-Dist: uuid-utils<1,>=0.19
36
+ Provides-Extra: http
37
+ Requires-Dist: fastapi<1,>=0.115.3; extra == 'http'
38
+ Provides-Extra: langchain-aws
39
+ Requires-Dist: langchain-aws<2,>=0.2.24; extra == 'langchain-aws'
40
+ Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-aws'
41
+ Provides-Extra: langchain-openai
42
+ Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-openai'
43
+ Requires-Dist: langchain-openai<2,>=0.3.26; extra == 'langchain-openai'
44
+ Provides-Extra: metrics
45
+ Requires-Dist: prometheus-client<1,>=0.20; extra == 'metrics'
46
+ Provides-Extra: openai
47
+ Requires-Dist: httpx<1,>=0.27; extra == 'openai'
48
+ Requires-Dist: openai<4,>=1.40; extra == 'openai'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # llm-queue
52
+
53
+ LLM calls through a queue on Valkey or Redis, under RPM / TPM budgets per **deployment**
54
+ (model x account x region), balanced across regions and accounts, with fallback regions,
55
+ fallback models, health checks and status. Works with Amazon Bedrock and OpenAI-compatible
56
+ servers (vLLM), and with LangChain. The queue, the budgets and the results live in one store, so
57
+ admission (deployment, RPM, TPM, concurrency slot) is one atomic step, and 1 pod behaves the
58
+ same as 50.
59
+
60
+ ## Install
61
+
62
+ ```
63
+ pip install "llm-queue[langchain-aws]" # Bedrock + LangChain
64
+ pip install "llm-queue[openai,langchain-openai]" # OpenAI-compatible
65
+ pip install "llm-queue[http]" # status and health routes for FastAPI
66
+ ```
67
+
68
+ ## Example
69
+
70
+ ```python
71
+ from llm_queue import LLMQueue, CallFailed, request_context
72
+
73
+ llmq = LLMQueue.from_env() # LLMQ_STORE_URL, LLMQ_REGISTRY, ...
74
+ await llmq.start() # publish / load the registry, run the workers
75
+ try:
76
+ with request_context(priority=True, user_id=uid):
77
+ reply = await llmq.converse("gpt120b", {"messages": [...]}, max_wait_secs=5)
78
+ print(reply.response, reply.meta.deployment)
79
+ except CallFailed as exc: # exc.kind: rate_limited, model_unavailable, ...
80
+ ...
81
+ await llmq.stop()
82
+ ```
83
+
84
+ ## Documentation
85
+
86
+ Full documentation: **https://techiewonk.github.io/llm-queue/**
87
+
88
+ - [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
89
+ - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, tune, call, operate, share models across services, test
90
+ - [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, failures and recovery
91
+ - [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): settings, error kinds, CLI and HTTP, the Python API
92
+
93
+ Project: [CHANGELOG](https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md),
94
+ [CONTRIBUTING](https://github.com/techiewonk/llm-queue/blob/main/CONTRIBUTING.md),
95
+ [STABILITY](https://github.com/techiewonk/llm-queue/blob/main/STABILITY.md),
96
+ [SECURITY](https://github.com/techiewonk/llm-queue/blob/main/SECURITY.md).
@@ -0,0 +1,58 @@
1
+ llm_queue/__init__.py,sha256=Zxs0rVMOU9kyHt6OARKrXJY4SQcpnQAxf-Km5CLdg7o,3032
2
+ llm_queue/_logging.py,sha256=ZVfyoCVewvBlJrsgyoOlLu_r9BCf0jxG9L2yN5jaugw,1744
3
+ llm_queue/_version.py,sha256=y4Ors2m0uN5McJDFlxvLwtvSXUmyn0Sa-AVtD97RekI,150
4
+ llm_queue/context.py,sha256=v05RkZ5QEu4WWYYKHzDhamKqb9ylRXXdtyKIRRveiPc,1776
5
+ llm_queue/core.py,sha256=9IpsrY1B7_yv7yKDar06B-QQGZKXVccbLMUFkt3vu3E,19112
6
+ llm_queue/errors.py,sha256=njaJsYbcyCzqVzHkylW8zm1fXhdq8gVxXgtwvLvUIdo,2481
7
+ llm_queue/http_transport.py,sha256=YIhRQgHaxl5dt3Oc8Bz3EYeF6SnAUx_0lHnd-ZTqbIA,9139
8
+ llm_queue/ids.py,sha256=qc7bRfXI2gpo0EqlfO2xsUKN3tLWjeNa6YFY4wEydu8,646
9
+ llm_queue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ llm_queue/tuning.py,sha256=x_rvifmLih05nsL1wPq_hlxYV-acp56YNsSx2dzH3lg,7481
11
+ llm_queue/clients/__init__.py,sha256=OptKjonubCxsPkgkRWwhHl0mO_k099vsmn4VN_J-x18,931
12
+ llm_queue/clients/pool.py,sha256=_Z_swepnNvkTjEfh22bxMN-u41PVD-R1w3TerWLCa9o,11935
13
+ llm_queue/clients/providers.py,sha256=p6OKuhT4VoWiUqAmAOCY_JwKl3TII7UR0rSK12DgPJU,13968
14
+ llm_queue/clients/timing.py,sha256=APB5qnfsNbOABKANivriQtevMnnQsr7JJ1tRbsScQsg,1860
15
+ llm_queue/engine/__init__.py,sha256=ptlS44P0NkZv31R2PIDXwvbnwjS_LAHGTziaEKZQsdE,127
16
+ llm_queue/engine/_admission_lua.py,sha256=P7IO17FZuXauo-QTgDCh-bsKOKFY5r_Nr-U2w7mAxQc,29452
17
+ llm_queue/engine/admission.py,sha256=-ax8s580Y1oMdCOhWAmnmOnyEoRY5cv7EjyRRPtYRVA,8896
18
+ llm_queue/engine/cli.py,sha256=8h7kcHcqGaLpzzWLL07SIm5fboE5mWuK0C8i-9grOhw,12653
19
+ llm_queue/engine/embed.py,sha256=b3J6apGMg3p0jE0jUdbCFSiJxqzIGL0b1PrUkXNfFPg,4043
20
+ llm_queue/engine/envelope.py,sha256=CI1eRJe7MxPYwwMqRyHCOV4U9zSh_mEEYMqYIldmnws,3571
21
+ llm_queue/engine/errors.py,sha256=PgDgsSoq1aAejClXZTnkrstOojj-yDIAsHyH0Aqcl3U,1443
22
+ llm_queue/engine/health.py,sha256=LnPcGksivg40fNQq1XzirDB7ZVLx4Mqyy2dkda8PCLY,5534
23
+ llm_queue/engine/hooks.py,sha256=uBZn41d2Kuav2yB5w7srqFatwCW2pJeQLj2aGdu0rOI,3956
24
+ llm_queue/engine/http.py,sha256=_uMSXA-jZn4T6P3TTjozghKDOiNAN5Zhpm85UtqQ0pw,7122
25
+ llm_queue/engine/inference.py,sha256=yobDfTmV4LYXKqcSQ4Wg4kqxvBULTK3eF5dQwO480bY,5098
26
+ llm_queue/engine/jobs.py,sha256=-3Ow11AxWeEeaUnyeKlDRpXFUI4H8EjOvAKOog1zQVA,5613
27
+ llm_queue/engine/keys.py,sha256=hfAtLT5-yo0pX1KPaOytnYk98MuLtBLQy4_k4liPOhM,2780
28
+ llm_queue/engine/metrics.py,sha256=RdA8irAcsELE13xigDSuMjp-yKS-hg710nLCSIO0pA4,4945
29
+ llm_queue/engine/offload.py,sha256=ccviGd0uL4w-IeHAhRrykutyGYM1c7InCbROZ2WEGOE,2787
30
+ llm_queue/engine/queue.py,sha256=Ji8drmV05O_i2K8idk9goSfoPZUB025LmpCEwx2qoYU,16427
31
+ llm_queue/engine/requests.py,sha256=GPK704xcCcBDlhi6FprH_as4thprjokHLrn2OzVtFDs,12542
32
+ llm_queue/engine/runtime.py,sha256=bUMeq8h-DTPqgGSQ6xp9s55fdUmzAcs1ZNjw62xixDc,107475
33
+ llm_queue/engine/settings.py,sha256=LdnhtRW3Cre6Iio7AvoLBqeoKsAKTHAiA8uolXPBmjo,8338
34
+ llm_queue/engine/status.py,sha256=ste09VeHbUa3tEXZOodqFb2qOyk9B9yrAhbQ3hdFD2Y,10866
35
+ llm_queue/engine/store_client.py,sha256=RlpUvdGEHIU_aKujWSPYh-WmYCxJk3ib885i8l3FR5Q,3131
36
+ llm_queue/engine/store_guard.py,sha256=0VbuFJpPMgIYJ7ww3rJwe7w__NAH5M9fmiCMRIGpp7I,1481
37
+ llm_queue/engine/streaming.py,sha256=jF1Lt8OJxxGSmMoaZ32ISZe98qZTxQGVmJwfZMEcHvA,8273
38
+ llm_queue/engine/tokens.py,sha256=pmTxjo5AqydXK1kwL1kjKkOMEaRmsKTC0MhTXIUR4qk,4674
39
+ llm_queue/langchain/__init__.py,sha256=VFBvk3FGvhzL6JXjBkou6jkmYD_DzKjEzCDetMHCPaE,2033
40
+ llm_queue/langchain/_client.py,sha256=QqMk-l13oAVZy7mfuFbYjM67DkAwyluu-2RHorLT274,2925
41
+ llm_queue/langchain/bedrock.py,sha256=t-s3u9AFPWA78g8MnfXKZwJKDv1nWxyKh9tXe14F4U8,12855
42
+ llm_queue/langchain/openai.py,sha256=bx0VUQy2vADKMM63GX5OSkxxbGxWOZw1D6xaqjA7bl0,4130
43
+ llm_queue/providers/__init__.py,sha256=fVPjKxVRQ1JJzq78oO7qQrO_dl4enHr0D6LCcKuxRzI,986
44
+ llm_queue/providers/classify.py,sha256=46ujPwREQwJyF7kNYJ0uXzw96kFR9B_L5YROkd-4I3M,9660
45
+ llm_queue/providers/embeddings.py,sha256=7i6klmXAaau_iyhFdq__ROIhw2jVEybuI4Bbvd-JId8,3371
46
+ llm_queue/registry/__init__.py,sha256=MEKJQKpWShJtn-YY905cn-TolCqrPChKrysQRfCudeE,945
47
+ llm_queue/registry/_lua.py,sha256=7gYSsjHN9_GkcyFW6JgLNUGaPbE0-661jBxU022GVEg,3367
48
+ llm_queue/registry/resolve.py,sha256=7B45usivJzVE45zBGOcyXJzODbUJ1RBJR_GCOOmCkAw,8461
49
+ llm_queue/registry/schema.py,sha256=y8-H96gnZSoehFnP-2pSj8hKJhW3PNpG2WVa9n4GjlA,20780
50
+ llm_queue/registry/settings.py,sha256=6yCIJieO3llcE4oKSadHmWUhOIYt7qV6LddO8MH2jHM,12826
51
+ llm_queue/registry/store.py,sha256=UttYtoc_1jWNPK3ICc4fX4GPKhLklwH2b9kk1UsNHls,25212
52
+ llm_queue/registry/validate.py,sha256=omwZl9iMPD6cYGz87SLXp2fx9oEQaY-cxFtwf3vCw4A,11833
53
+ llm_queue-0.1.0b1.dist-info/METADATA,sha256=l72fYvxv7JPZnLVfMyjJc61-LRBjezZlZkpVsOBG5Tg,4547
54
+ llm_queue-0.1.0b1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
55
+ llm_queue-0.1.0b1.dist-info/entry_points.txt,sha256=rObO8zmIGbJy0563G2CuUkX9S8lj2zPUs7enm-xDr9s,56
56
+ llm_queue-0.1.0b1.dist-info/licenses/LICENSE,sha256=OvUQrS0ASWXZWrpTi33eLZyB0SJmpWD99z-L0Lu8C8Y,1078
57
+ llm_queue-0.1.0b1.dist-info/licenses/NOTICE,sha256=ApdjxFQDujzw8sm8_i31dg_Hf_Tg_Lu1JBQZU6vjhZE,748
58
+ llm_queue-0.1.0b1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ llm-queue = llm_queue.engine.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The llm-queue authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,14 @@
1
+ llm-queue
2
+ Copyright (c) 2026 The llm-queue authors. Licensed under the MIT License.
3
+
4
+ The queue's design (visibility heartbeat, message metadata, large results in S3) builds on
5
+ ideas from two MIT-licensed SQS brokers. This package is an independent implementation on
6
+ Valkey / Redis streams; any code adapted from them keeps its original notice below.
7
+
8
+ taskiq-aio-sqs — https://github.com/vonsteer/taskiq-aio-sqs
9
+ Copyright (c) 2022-2024 Jesse Constante. MIT License.
10
+ Ideas used: visibility heartbeat, message metadata, extended (S3) messages.
11
+
12
+ taskiq-sqs — https://github.com/taskiq-python/taskiq-sqs
13
+ Copyright (c) 2026 Mike Shultz and Taskiq team. MIT License.
14
+ Ideas used: message expiry, flush on shutdown, S3 offload of large results.