llm-queue 0.1.0b1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. llm_queue-0.1.0b1/.gitignore +42 -0
  2. llm_queue-0.1.0b1/CHANGELOG.md +66 -0
  3. llm_queue-0.1.0b1/LICENSE +21 -0
  4. llm_queue-0.1.0b1/NOTICE +14 -0
  5. llm_queue-0.1.0b1/PKG-INFO +96 -0
  6. llm_queue-0.1.0b1/README.md +46 -0
  7. llm_queue-0.1.0b1/SECURITY.md +21 -0
  8. llm_queue-0.1.0b1/STABILITY.md +16 -0
  9. llm_queue-0.1.0b1/pyproject.toml +186 -0
  10. llm_queue-0.1.0b1/samples/README.md +94 -0
  11. llm_queue-0.1.0b1/src/llm_queue/__init__.py +105 -0
  12. llm_queue-0.1.0b1/src/llm_queue/_logging.py +41 -0
  13. llm_queue-0.1.0b1/src/llm_queue/_version.py +5 -0
  14. llm_queue-0.1.0b1/src/llm_queue/clients/__init__.py +42 -0
  15. llm_queue-0.1.0b1/src/llm_queue/clients/pool.py +271 -0
  16. llm_queue-0.1.0b1/src/llm_queue/clients/providers.py +360 -0
  17. llm_queue-0.1.0b1/src/llm_queue/clients/timing.py +52 -0
  18. llm_queue-0.1.0b1/src/llm_queue/context.py +54 -0
  19. llm_queue-0.1.0b1/src/llm_queue/core.py +482 -0
  20. llm_queue-0.1.0b1/src/llm_queue/engine/__init__.py +3 -0
  21. llm_queue-0.1.0b1/src/llm_queue/engine/_admission_lua.py +696 -0
  22. llm_queue-0.1.0b1/src/llm_queue/engine/admission.py +243 -0
  23. llm_queue-0.1.0b1/src/llm_queue/engine/cli.py +292 -0
  24. llm_queue-0.1.0b1/src/llm_queue/engine/embed.py +103 -0
  25. llm_queue-0.1.0b1/src/llm_queue/engine/envelope.py +102 -0
  26. llm_queue-0.1.0b1/src/llm_queue/engine/errors.py +51 -0
  27. llm_queue-0.1.0b1/src/llm_queue/engine/health.py +139 -0
  28. llm_queue-0.1.0b1/src/llm_queue/engine/hooks.py +119 -0
  29. llm_queue-0.1.0b1/src/llm_queue/engine/http.py +180 -0
  30. llm_queue-0.1.0b1/src/llm_queue/engine/inference.py +121 -0
  31. llm_queue-0.1.0b1/src/llm_queue/engine/jobs.py +137 -0
  32. llm_queue-0.1.0b1/src/llm_queue/engine/keys.py +70 -0
  33. llm_queue-0.1.0b1/src/llm_queue/engine/metrics.py +141 -0
  34. llm_queue-0.1.0b1/src/llm_queue/engine/offload.py +82 -0
  35. llm_queue-0.1.0b1/src/llm_queue/engine/queue.py +447 -0
  36. llm_queue-0.1.0b1/src/llm_queue/engine/requests.py +317 -0
  37. llm_queue-0.1.0b1/src/llm_queue/engine/runtime.py +2357 -0
  38. llm_queue-0.1.0b1/src/llm_queue/engine/settings.py +186 -0
  39. llm_queue-0.1.0b1/src/llm_queue/engine/status.py +267 -0
  40. llm_queue-0.1.0b1/src/llm_queue/engine/store_client.py +65 -0
  41. llm_queue-0.1.0b1/src/llm_queue/engine/store_guard.py +40 -0
  42. llm_queue-0.1.0b1/src/llm_queue/engine/streaming.py +216 -0
  43. llm_queue-0.1.0b1/src/llm_queue/engine/tokens.py +129 -0
  44. llm_queue-0.1.0b1/src/llm_queue/errors.py +73 -0
  45. llm_queue-0.1.0b1/src/llm_queue/http_transport.py +211 -0
  46. llm_queue-0.1.0b1/src/llm_queue/ids.py +16 -0
  47. llm_queue-0.1.0b1/src/llm_queue/langchain/__init__.py +61 -0
  48. llm_queue-0.1.0b1/src/llm_queue/langchain/_client.py +94 -0
  49. llm_queue-0.1.0b1/src/llm_queue/langchain/bedrock.py +312 -0
  50. llm_queue-0.1.0b1/src/llm_queue/langchain/openai.py +102 -0
  51. llm_queue-0.1.0b1/src/llm_queue/providers/__init__.py +43 -0
  52. llm_queue-0.1.0b1/src/llm_queue/providers/classify.py +251 -0
  53. llm_queue-0.1.0b1/src/llm_queue/providers/embeddings.py +87 -0
  54. llm_queue-0.1.0b1/src/llm_queue/py.typed +0 -0
  55. llm_queue-0.1.0b1/src/llm_queue/registry/__init__.py +36 -0
  56. llm_queue-0.1.0b1/src/llm_queue/registry/_lua.py +78 -0
  57. llm_queue-0.1.0b1/src/llm_queue/registry/resolve.py +199 -0
  58. llm_queue-0.1.0b1/src/llm_queue/registry/schema.py +521 -0
  59. llm_queue-0.1.0b1/src/llm_queue/registry/settings.py +364 -0
  60. llm_queue-0.1.0b1/src/llm_queue/registry/store.py +614 -0
  61. llm_queue-0.1.0b1/src/llm_queue/registry/validate.py +258 -0
  62. llm_queue-0.1.0b1/src/llm_queue/tuning.py +151 -0
@@ -0,0 +1,42 @@
1
+ # Environments and secrets (never commit tokens: GitHub, PyPI, AWS)
2
+ .env
3
+ .env.*
4
+ !.env.example
5
+ .pypirc
6
+ *.pem
7
+ *.key
8
+ .venv/
9
+ venv/
10
+
11
+ # Python build and cache
12
+ __pycache__/
13
+ *.py[cod]
14
+ *.egg-info/
15
+ build/
16
+ dist/
17
+ .pytest_cache/
18
+ .ruff_cache/
19
+ .mypy_cache/
20
+ .coverage
21
+ coverage.xml
22
+ htmlcov/
23
+
24
+ # Test and tool output
25
+ .hypothesis/
26
+ .locust/
27
+ *.log
28
+ reports/
29
+
30
+ # Editors and OS
31
+ .idea/
32
+ .vscode/
33
+ *.swp
34
+ .DS_Store
35
+ Thumbs.db
36
+
37
+ # Docs site build (zensical build)
38
+ site/
39
+ .cache/
40
+
41
+ # Internal working notes (release and test plans): kept locally, never published
42
+ docs/internal/
@@ -0,0 +1,66 @@
1
+ # Changelog
2
+
3
+ All notable changes are listed here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
5
+ [semantic versioning](https://semver.org/); while it is 0.x, a minor release may break the
6
+ public API (see STABILITY.md).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.1.0b1] - 2026-11-14
11
+
12
+ First beta, for test runs (`pip install --pre llm-queue`).
13
+
14
+ ### Added
15
+
16
+ - **Queued LLM calls on Valkey / Redis.** `LLMQueue.from_env()` runs calls to Amazon Bedrock
17
+ and OpenAI-compatible servers (vLLM) through a queue per model (a normal and a priority
18
+ stream) on one Redis-protocol store. Every pod runs the same code as caller and worker
19
+ (`LLMQ_MODE`: `both`, `caller`, `worker`); budgets, waiter order and health live in the
20
+ store, so one pod and fifty behave the same.
21
+ - **Budgets per deployment** (model x account x region): RPM and TPM (`smooth` GCRA or `fixed`
22
+ per minute), account budgets shared by models, concurrency slots learned from throttles,
23
+ priority reserve and normal minimum share. Admission is one atomic store script: choose a
24
+ deployment (`ordered`, `round_robin`, `weighted`, `headroom`, `custom:`), take every budget
25
+ or none, keep waiters in arrival order.
26
+ - **Routing and recovery:** fallback regions and fallback models by cause (`exhausted`,
27
+ `broken`, `error`) after a wait; retries with backoff, a per-model retry budget (moving to a
28
+ deployment the call has not tried is free) and one deadline per call; circuit breaker per
29
+ deployment (calls wait for its probe); health with auto-disable, probes and a last-healthy
30
+ safeguard; slow deployments become last choices; credentials judged per account and region.
31
+ - **Registry** (`LLMQ_REGISTRY`): accounts (AWS by pod identity or assumed role,
32
+ OpenAI-compatible endpoints), models with confirmed and fallback deployments, residency,
33
+ routing (kill switch, `queue_pct`, blocked regions), every setting at four levels with
34
+ types, ranges and defaults, validated with every problem listed. Stored in the store with a
35
+ version rule, live edits with audit, rollback and restore after a store reset.
36
+ - **Calls:** `converse` / `chat`, `converse_stream` (events through a chunk stream;
37
+ `CallFailed.partial_text` on a cut), `embed` (batched per model, vectors in order), sync
38
+ calls (`*_sync`, `close_sync`), jobs (`register_job` / `send_job`, at least once),
39
+ `request_context` (priority, session, user, tenant), duplicate request ids run once,
40
+ cancel, large results in S3. LangChain: `QueuedChatBedrockConverse`,
41
+ `QueuedBedrockEmbeddings`, `QueuedChatOpenAI`, `QueuedOpenAIEmbeddings`.
42
+ - **Errors:** every failed call is `CallFailed` with a `kind` (`rate_limited`,
43
+ `deadline_exceeded`, `model_unavailable`, `queue_unavailable`, ...) and `retry_after_secs`
44
+ where a retry can work; admin operations raise `StoreUnavailable`; bad configuration is
45
+ `ConfigError` (`UnknownModel`, `MissingExtra`). Everything raised is an `LLMQueueError`.
46
+ - **Store resilience:** reconnect on `READONLY` and dropped connections (failover), a read
47
+ timeout that is never retried, a blocking connection pool, `store_down` rules (`pause`,
48
+ `static_cap`, `fail_fast`), callers kept alive through a store outage, calls held past their
49
+ deadline let go, optional separate queue store with failover to the direct path, optional
50
+ `WAIT` for replicas.
51
+ - **Tuning in code:** `Tuning` (`StoreOptions`, `BedrockOptions`, `EngineTimings`,
52
+ `RegistryOptions`) passed to `LLMQueue.from_env(tuning=...)`: every timeout, pool size and
53
+ pacing value (admission scripts included) with a measured default, validated at start-up.
54
+ - **Operations:** status and request status, alerts, metrics (CloudWatch EMF, Prometheus), one
55
+ `llmq_call_finished` record per call, `health()` with per-store state, `pause_receiving()`
56
+ for a preStop hook and a bounded `stop(drain_secs=)`, the `llm-queue` CLI (status, registry,
57
+ settings, block, drain, dead letters, queues) and a FastAPI router (`http` extra).
58
+ - **Testing:** fakellm (a fake Bedrock / OpenAI server), `scripts/check.py` (lint, types, tests
59
+ per step), fault scenarios by call count (`scripts/scenarios.py`: worker, store, provider,
60
+ registry, caller and multi-client faults under three loads), load and chaos gates, a soak
61
+ with chaos, a store failover drill, and runnable samples on fakellm or real Bedrock.
62
+ - **Docs site** (Zensical on GitHub Pages): tutorial, how-to guides, explanation with
63
+ diagrams, an API reference from the docstrings.
64
+
65
+ [Unreleased]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b1...HEAD
66
+ [0.1.0b1]: https://github.com/techiewonk/llm-queue/releases/tag/v0.1.0b1
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The llm-queue authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,14 @@
1
+ llm-queue
2
+ Copyright (c) 2026 The llm-queue authors. Licensed under the MIT License.
3
+
4
+ The queue's design (visibility heartbeat, message metadata, large results in S3) builds on
5
+ ideas from two MIT-licensed SQS brokers. This package is an independent implementation on
6
+ Valkey / Redis streams; any code adapted from them keeps its original notice below.
7
+
8
+ taskiq-aio-sqs — https://github.com/vonsteer/taskiq-aio-sqs
9
+ Copyright (c) 2022-2024 Jesse Constante. MIT License.
10
+ Ideas used: visibility heartbeat, message metadata, extended (S3) messages.
11
+
12
+ taskiq-sqs — https://github.com/taskiq-python/taskiq-sqs
13
+ Copyright (c) 2026 Mike Shultz and Taskiq team. MIT License.
14
+ Ideas used: message expiry, flush on shutdown, S3 offload of large results.
@@ -0,0 +1,96 @@
1
+ Metadata-Version: 2.5
2
+ Name: llm-queue
3
+ Version: 0.1.0b1
4
+ Summary: LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status.
5
+ Project-URL: Homepage, https://github.com/techiewonk/llm-queue
6
+ Project-URL: Documentation, https://techiewonk.github.io/llm-queue/
7
+ Project-URL: Source, https://github.com/techiewonk/llm-queue
8
+ Project-URL: Issues, https://github.com/techiewonk/llm-queue/issues
9
+ Project-URL: Changelog, https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md
10
+ Author: The llm-queue authors
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ License-File: NOTICE
14
+ Keywords: bedrock,langchain,langgraph,llm,openai,queue,rate-limit,redis,valkey
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Framework :: AsyncIO
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
25
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
26
+ Classifier: Topic :: System :: Distributed Computing
27
+ Classifier: Typing :: Typed
28
+ Requires-Python: >=3.11
29
+ Requires-Dist: aiobotocore<4,>=2.23
30
+ Requires-Dist: hiredis<4,>=3
31
+ Requires-Dist: orjson<4,>=3.10
32
+ Requires-Dist: pydantic<3,>=2.10
33
+ Requires-Dist: redis<9,>=5.0.1
34
+ Requires-Dist: structlog<27,>=24.1
35
+ Requires-Dist: uuid-utils<1,>=0.19
36
+ Provides-Extra: http
37
+ Requires-Dist: fastapi<1,>=0.115.3; extra == 'http'
38
+ Provides-Extra: langchain-aws
39
+ Requires-Dist: langchain-aws<2,>=0.2.24; extra == 'langchain-aws'
40
+ Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-aws'
41
+ Provides-Extra: langchain-openai
42
+ Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-openai'
43
+ Requires-Dist: langchain-openai<2,>=0.3.26; extra == 'langchain-openai'
44
+ Provides-Extra: metrics
45
+ Requires-Dist: prometheus-client<1,>=0.20; extra == 'metrics'
46
+ Provides-Extra: openai
47
+ Requires-Dist: httpx<1,>=0.27; extra == 'openai'
48
+ Requires-Dist: openai<4,>=1.40; extra == 'openai'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # llm-queue
52
+
53
+ LLM calls through a queue on Valkey or Redis, under RPM / TPM budgets per **deployment**
54
+ (model x account x region), balanced across regions and accounts, with fallback regions,
55
+ fallback models, health checks and status. Works with Amazon Bedrock and OpenAI-compatible
56
+ servers (vLLM), and with LangChain. The queue, the budgets and the results live in one store, so
57
+ admission (deployment, RPM, TPM, concurrency slot) is one atomic step, and 1 pod behaves the
58
+ same as 50.
59
+
60
+ ## Install
61
+
62
+ ```
63
+ pip install "llm-queue[langchain-aws]" # Bedrock + LangChain
64
+ pip install "llm-queue[openai,langchain-openai]" # OpenAI-compatible
65
+ pip install "llm-queue[http]" # status and health routes for FastAPI
66
+ ```
67
+
68
+ ## Example
69
+
70
+ ```python
71
+ from llm_queue import LLMQueue, CallFailed, request_context
72
+
73
+ llmq = LLMQueue.from_env() # LLMQ_STORE_URL, LLMQ_REGISTRY, ...
74
+ await llmq.start() # publish / load the registry, run the workers
75
+ try:
76
+ with request_context(priority=True, user_id=uid):
77
+ reply = await llmq.converse("gpt120b", {"messages": [...]}, max_wait_secs=5)
78
+ print(reply.response, reply.meta.deployment)
79
+ except CallFailed as exc: # exc.kind: rate_limited, model_unavailable, ...
80
+ ...
81
+ await llmq.stop()
82
+ ```
83
+
84
+ ## Documentation
85
+
86
+ Full documentation: **https://techiewonk.github.io/llm-queue/**
87
+
88
+ - [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
89
+ - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, tune, call, operate, share models across services, test
90
+ - [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, failures and recovery
91
+ - [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): settings, error kinds, CLI and HTTP, the Python API
92
+
93
+ Project: [CHANGELOG](https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md),
94
+ [CONTRIBUTING](https://github.com/techiewonk/llm-queue/blob/main/CONTRIBUTING.md),
95
+ [STABILITY](https://github.com/techiewonk/llm-queue/blob/main/STABILITY.md),
96
+ [SECURITY](https://github.com/techiewonk/llm-queue/blob/main/SECURITY.md).
@@ -0,0 +1,46 @@
1
+ # llm-queue
2
+
3
+ LLM calls through a queue on Valkey or Redis, under RPM / TPM budgets per **deployment**
4
+ (model x account x region), balanced across regions and accounts, with fallback regions,
5
+ fallback models, health checks and status. Works with Amazon Bedrock and OpenAI-compatible
6
+ servers (vLLM), and with LangChain. The queue, the budgets and the results live in one store, so
7
+ admission (deployment, RPM, TPM, concurrency slot) is one atomic step, and 1 pod behaves the
8
+ same as 50.
9
+
10
+ ## Install
11
+
12
+ ```
13
+ pip install "llm-queue[langchain-aws]" # Bedrock + LangChain
14
+ pip install "llm-queue[openai,langchain-openai]" # OpenAI-compatible
15
+ pip install "llm-queue[http]" # status and health routes for FastAPI
16
+ ```
17
+
18
+ ## Example
19
+
20
+ ```python
21
+ from llm_queue import LLMQueue, CallFailed, request_context
22
+
23
+ llmq = LLMQueue.from_env() # LLMQ_STORE_URL, LLMQ_REGISTRY, ...
24
+ await llmq.start() # publish / load the registry, run the workers
25
+ try:
26
+ with request_context(priority=True, user_id=uid):
27
+ reply = await llmq.converse("gpt120b", {"messages": [...]}, max_wait_secs=5)
28
+ print(reply.response, reply.meta.deployment)
29
+ except CallFailed as exc: # exc.kind: rate_limited, model_unavailable, ...
30
+ ...
31
+ await llmq.stop()
32
+ ```
33
+
34
+ ## Documentation
35
+
36
+ Full documentation: **https://techiewonk.github.io/llm-queue/**
37
+
38
+ - [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
39
+ - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, tune, call, operate, share models across services, test
40
+ - [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, failures and recovery
41
+ - [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): settings, error kinds, CLI and HTTP, the Python API
42
+
43
+ Project: [CHANGELOG](https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md),
44
+ [CONTRIBUTING](https://github.com/techiewonk/llm-queue/blob/main/CONTRIBUTING.md),
45
+ [STABILITY](https://github.com/techiewonk/llm-queue/blob/main/STABILITY.md),
46
+ [SECURITY](https://github.com/techiewonk/llm-queue/blob/main/SECURITY.md).
@@ -0,0 +1,21 @@
1
+ # Security policy
2
+
3
+ ## Supported versions
4
+
5
+ | Version | Supported |
6
+ | ------- | --------- |
7
+ | 0.3.x | Yes |
8
+ | < 0.3 | No |
9
+
10
+ ## Reporting a vulnerability
11
+
12
+ Please do not open a public issue for a security problem.
13
+
14
+ Report it privately through GitHub's private vulnerability reporting:
15
+ [Security > Report a vulnerability](https://github.com/techiewonk/llm-queue/security/advisories/new)
16
+ on the `techiewonk/llm-queue` repository. This opens a draft security advisory that only the
17
+ maintainers can see.
18
+
19
+ Include the affected version, a description of the problem, and steps or code to reproduce it.
20
+ We aim to acknowledge a report within a few working days, and will coordinate a fix and a
21
+ GitHub Security Advisory (with a CVE where it applies) before any public disclosure.
@@ -0,0 +1,16 @@
1
+ # Stability policy
2
+
3
+ - **Public API** = the names in `llm_queue.__all__`, the registry document's setting
4
+ names and the `llm-queue` commands. Anything else may change without notice.
5
+ - **0.x releases:** a minor release may break the public API; every break is in CHANGELOG.md.
6
+ - **Message format:** queue messages carry a format version `v`. A worker returns a message newer
7
+ than it understands to the queue instead of dropping it.
8
+ - **Store schema:** the key layout has a schema number; a change to it ships with a migration note.
9
+ - **Supported versions:** Python 3.11-3.13 (3.14 runs in CI, not yet supported); Valkey 8
10
+ and Redis 7.2+ as one primary with replicas (cluster mode is not supported: one admission
11
+ touches keys of several hash tags). The store must allow Lua scripts, pub/sub and blocking
12
+ list reads (`BLPOP`, the queue doorbell); ElastiCache for Valkey / Redis OSS does.
13
+ - **Dependencies:** ranges with upper bounds in `pyproject.toml` (`aiobotocore<4`, `redis<9`,
14
+ ...); a release may raise a bound, never drop one without a test on the new major.
15
+ - **Settings:** the `LLMQ_*` names and the registry setting names are public API; removing or
16
+ renaming one is a breaking change listed in CHANGELOG.md.
@@ -0,0 +1,186 @@
1
+ [project]
2
+ name = "llm-queue"
3
+ description = "LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status."
4
+ readme = "README.md"
5
+ requires-python = ">=3.11"
6
+ dynamic = ["version"]
7
+ # PEP 639 licence expression; NOTICE credits the projects the queue design builds on.
8
+ license = "MIT"
9
+ license-files = ["LICENSE", "NOTICE"]
10
+ authors = [{ name = "The llm-queue authors" }]
11
+ keywords = [
12
+ "valkey", "redis", "llm", "bedrock", "openai", "queue", "rate-limit", "langchain", "langgraph",
13
+ ]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Framework :: AsyncIO",
17
+ "Intended Audience :: Developers",
18
+ "Operating System :: OS Independent",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3 :: Only",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
25
+ "Topic :: Software Development :: Libraries :: Python Modules",
26
+ "Topic :: System :: Distributed Computing",
27
+ "Typing :: Typed",
28
+ ]
29
+
30
+ # Ranges, not pins: a library that pins exact versions locks every user into one botocore
31
+ # patch window. Exact versions live only in the consuming app's lockfile.
32
+ dependencies = [
33
+ # Async Bedrock (and S3 offload) client. aiobotocore pins a narrow botocore window.
34
+ # Upper bound: the pool reads aiobotocore internals (refreshable credentials).
35
+ "aiobotocore>=2.23,<4",
36
+ # Queues, budgets, registry and results on any Redis-protocol store (Valkey, Redis).
37
+ "redis>=5.0.1,<9",
38
+ # C RESP parser, picked up by redis-py: Lua array replies and stream reads parse 39-75x
39
+ # faster than redis-py's pure-Python parser (measured 2026-11-13).
40
+ "hiredis>=3,<4",
41
+ "pydantic>=2.10,<3",
42
+ # JSON on every hot path (envelopes, params, provider bodies): an EMF line 0.8 us vs 6.2 us,
43
+ # a 200 KB request round trip 0.38 ms vs 0.95 ms with stdlib json (measured 2026-11-13).
44
+ "orjson>=3.10,<4",
45
+ # Structured key-value logs. The library never configures it; the app does (README).
46
+ "structlog>=24.1,<27",
47
+ # UUIDv7 ids (time-ordered), Rust: 0.49 us per id vs 2.4 us for uuid.uuid4().hex (measured).
48
+ "uuid-utils>=0.19,<1",
49
+ ]
50
+
51
+ [project.scripts]
52
+ llm-queue = "llm_queue.engine.cli:main"
53
+
54
+ [project.optional-dependencies]
55
+ openai = ["openai>=1.40,<4", "httpx>=0.27,<1"]
56
+ # Both LangChain lines are supported: 0.3 and 1.x (the adapters are tested on each).
57
+ langchain-aws = ["langchain-aws>=0.2.24,<2", "langchain-core>=0.3.68,<2"]
58
+ langchain-openai = ["langchain-openai>=0.3.26,<2", "langchain-core>=0.3.68,<2"]
59
+ metrics = ["prometheus-client>=0.20,<1"]
60
+ # Status / registry / health routes for a FastAPI app (`LLMQueue.http_router`).
61
+ # 0.115.3+: Starlette with the multipart DoS fixes (CVE-2024-47874).
62
+ http = ["fastapi>=0.115.3,<1"]
63
+
64
+ [project.urls]
65
+ Homepage = "https://github.com/techiewonk/llm-queue"
66
+ Documentation = "https://techiewonk.github.io/llm-queue/"
67
+ Source = "https://github.com/techiewonk/llm-queue"
68
+ Issues = "https://github.com/techiewonk/llm-queue/issues"
69
+ Changelog = "https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md"
70
+
71
+ [dependency-groups]
72
+ dev = [
73
+ "pytest>=8",
74
+ "pytest-asyncio>=0.24",
75
+ "ruff>=0.6",
76
+ "mypy>=1.11",
77
+ "types-aiobotocore[s3,bedrock-runtime,sts]",
78
+ "moto[server]>=5",
79
+ "fakeredis>=2.26",
80
+ # extras installed for development and tests of the provider/adapter layers
81
+ # The dev env is on the LangChain 1.x line; the 0.3 line is checked in CI (langchain-03 job).
82
+ "openai>=1.40,<4",
83
+ "httpx>=0.27,<1",
84
+ "langchain-aws>=1,<2",
85
+ "langchain-openai>=1,<2",
86
+ "langchain-core>=1,<2",
87
+ "langgraph>=1",
88
+ "fastapi>=0.115.3,<1",
89
+ ]
90
+
91
+ [build-system]
92
+ requires = ["hatchling>=1.27"] # PEP 639 license-files
93
+ build-backend = "hatchling.build"
94
+
95
+ [tool.hatch.version]
96
+ path = "src/llm_queue/_version.py"
97
+
98
+ [tool.hatch.build.targets.wheel]
99
+ packages = ["src/llm_queue"]
100
+
101
+ [tool.hatch.build.targets.sdist]
102
+ # Allow-list: secrets (.env, .pypirc) and local tooling can never reach the sdist.
103
+ # The design plan, walkthrough and release / test plans are internal: never in the sdist.
104
+ include = ["src/", "README.md", "LICENSE", "NOTICE", "CHANGELOG.md", "STABILITY.md", "SECURITY.md"]
105
+
106
+ [tool.ruff]
107
+ # target-version comes from project.requires-python.
108
+ line-length = 100
109
+ extend-exclude = ["*.md"]
110
+
111
+ [tool.ruff.lint]
112
+ select = ["ASYNC", "B", "C4", "D", "E", "F", "I", "PIE", "PT", "RET", "RUF", "SIM", "UP", "W"]
113
+ ignore = ["D105"]
114
+
115
+ [tool.ruff.lint.pydocstyle]
116
+ convention = "google"
117
+
118
+ [tool.ruff.lint.per-file-ignores]
119
+ "tests/*" = ["D"]
120
+ "samples/*" = ["D103"] # samples read top to bottom; the module docstring says it all
121
+
122
+ [tool.mypy]
123
+ strict = true
124
+ python_version = "3.11"
125
+ files = ["src", "tests", "tools", "scripts", "samples"]
126
+ # Test folders have no __init__.py (several conftest.py files): map modules from paths.
127
+ explicit_package_bases = true
128
+ mypy_path = ["src", "tools/fakellm"]
129
+ plugins = ["pydantic.mypy"]
130
+ warn_unreachable = true
131
+ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
132
+
133
+ [[tool.mypy.overrides]]
134
+ module = ["moto.*", "fastapi", "fastapi.*"]
135
+ ignore_missing_imports = true
136
+
137
+ # FastAPI is an optional extra: its decorators are untyped when it is not installed.
138
+ [[tool.mypy.overrides]]
139
+ module = ["llm_queue.engine.http"]
140
+ disallow_untyped_decorators = false
141
+
142
+ # Test helpers are imported by bare name (pytest puts their folder on sys.path); the helper
143
+ # files themselves are still type-checked as part of `files`.
144
+ [[tool.mypy.overrides]]
145
+ module = ["fakellm_testkit", "providers_fake", "clients_testkit", "engine_testkit", "gates_fleet", "failover", "gates"]
146
+ ignore_missing_imports = true
147
+
148
+ [tool.pytest.ini_options]
149
+ minversion = "8"
150
+ # -ra: a short summary of every non-passing outcome (scripts/check.py passes -rA on top).
151
+ addopts = ["-ra", "--strict-config", "--strict-markers"]
152
+ xfail_strict = true
153
+ log_level = "INFO"
154
+ filterwarnings = [
155
+ "error",
156
+ # Python 3.14: langsmith (under langchain-core) calls asyncio.iscoroutinefunction while its
157
+ # pytest plugin loads; our code does not use it.
158
+ "ignore:'asyncio.iscoroutinefunction' is deprecated:DeprecationWarning",
159
+ ]
160
+ pythonpath = ["src"]
161
+ testpaths = ["tests/unit"]
162
+ asyncio_mode = "auto"
163
+ asyncio_default_fixture_loop_scope = "function"
164
+ markers = [
165
+ "integration: needs the Docker stack (docker/compose.yaml)",
166
+ ]
167
+
168
+ [tool.repo-review]
169
+ ignore = [
170
+ # PY007 (nox/tox): scripts/check.py is the task runner (CONTRIBUTING.md); it also drives the
171
+ # Docker stack steps, which nox/tox would only wrap.
172
+ "PY007",
173
+ # GH200 (Dependabot): dependency updates come from Renovate (Renovate app on the repo);
174
+ # Dependabot as well would open duplicate update PRs.
175
+ "GH200",
176
+ # PC140 (mypy hook): strict mypy needs the full locked env (extras, stubs, plugin);
177
+ # scripts/check.py and CI run it.
178
+ "PC140",
179
+ # PC160 / PC180 (spell checker, Markdown formatter): not adopted yet; follow-up.
180
+ "PC160",
181
+ "PC180",
182
+ # PC901-PC903: pre-commit.ci is not used; CI runs scripts/check.py's steps.
183
+ "PC901",
184
+ "PC902",
185
+ "PC903",
186
+ ]
@@ -0,0 +1,94 @@
1
+ # Samples
2
+
3
+ Runnable examples of every way an app uses llm-queue. Each file is plain app code:
4
+ it builds the queue with `LLMQueue.from_env()` and reads nothing else. The runner only sets
5
+ the environment.
6
+
7
+ | Sample | Shows | Use it in |
8
+ |---|---|---|
9
+ | `01_converse.py` | `converse` with `request_context` (priority, session, user); reply text and `meta` | Async request handlers (FastAPI routes), agent steps, anything user-facing |
10
+ | `02_stream.py` | `converse_stream`: events as they arrive; `CallFailed.partial_text` on a cut | Chat UIs and server-sent events, where the first words should show at once |
11
+ | `03_embed.py` | `embed` on Titan (1 text per request) and Cohere (96 per request, `input_type`) | Indexing documents (`search_document`) and search queries (`search_query`) |
12
+ | `04_sync.py` | `converse_sync` / `embed_sync` on the direct path, `close_sync()` | Sync code only: scripts, Celery tasks, sync libraries. Async code uses 01/03 |
13
+ | `05_errors_and_fallback.py` | Model fallback when a model is off; `CallFailed.kind` to branch on | Every caller: map `kind` to a user message or a retry |
14
+ | `06_jobs.py` | `register_job` / `send_job`: work that runs at least once, LLM calls inside | Ingestion, nightly re-embedding, any batch work that should not block a request |
15
+ | `07_status_and_registry.py` | `status`, `request_status`, live `registry.set`, audit `log`, `health` | Admin pages, runbooks, readiness checks |
16
+ | `08_langchain.py` | `QueuedChatBedrockConverse` (with tools) and `QueuedBedrockEmbeddings` | LangChain / LangGraph chains, agents, RAG: drop-in for `ChatBedrockConverse` / `BedrockEmbeddings` |
17
+ | `09_http_router.py` | `http_router` on FastAPI: health without auth, status / registry behind the app's admin check, audited writes (`actor=`), `/drain` with `pause_receiving()` | Admin API and Kubernetes probes; the preStop hook |
18
+
19
+ `registry.json` defines the four models the samples use: chat `gpt120b` (falls back to `gpt20b`)
20
+ and `gpt20b`, and embeddings `titan` and `cohere`, all on one AWS account. The quotas are
21
+ small sample numbers. Use your Service Quotas numbers in a real registry.
22
+
23
+ ## Run them
24
+
25
+ Start the local stack once (Valkey, plus fakellm as a fake Bedrock):
26
+
27
+ ```
28
+ python scripts/stack.py up
29
+ ```
30
+
31
+ Then run the samples from the library root:
32
+
33
+ ```
34
+ python scripts/samples.py # all, on fakellm (no real LLM, no cost)
35
+ python scripts/samples.py 03_embed 08_langchain # some
36
+ python scripts/samples.py --live # all, on real Bedrock
37
+ python scripts/samples.py --live --env-file ../../.env # AWS keys from another .env
38
+ python scripts/samples.py --live --store env --env-file ../../.env # and the env file's store
39
+ ```
40
+
41
+ - **Stored keys:** each run uses its own key prefix on the local Valkey and deletes its keys at the end.
42
+ - **What `--live` reads:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_MODEL_REGION` from the env file. It never prints them.
43
+ - **Region:** every model is placed in that region.
44
+ - **Cost:** a full live run makes about 20 small calls, well under a cent.
45
+ - **`--store env`:** the store is the env file's `LLMQ_STORE_URL`, or `rediss://` built from
46
+ the app's `valkey_host` / `valkey_port` / `valkey_key`. Only the run's own prefix is touched
47
+ and deleted, so a shared store (ElastiCache) is safe.
48
+
49
+ `python scripts/check.py samples` runs the fakellm pass with lint and type checks, as part of
50
+ `check.py all`.
51
+
52
+ ## Use them in an app
53
+
54
+ An app sets the same environment the runner sets, from its secrets:
55
+
56
+ | Variable | Runner value | In an app |
57
+ |---|---|---|
58
+ | `LLMQ_STORE_URL` | `redis://:localdev@127.0.0.1:6379/0` | `rediss://...` of the ElastiCache Valkey |
59
+ | `LLMQ_KEY_PREFIX`, `LLMQ_QUEUE_PREFIX` | per run | per environment, e.g. `llmq:prod` |
60
+ | `LLMQ_REGISTRY` | `samples/registry.json`, region set | the app's registry document |
61
+ | `LLMQ_METRICS_SINK` | `none` | `emf` (CloudWatch) or `prometheus` |
62
+ | AWS credentials | from `.env` (or fake for fakellm) | the pod's role; `role_arn` accounts assume roles |
63
+
64
+ Start the queue once per process, in the app's startup hook, and stop it on shutdown:
65
+
66
+ ```python
67
+ @asynccontextmanager
68
+ async def lifespan(app):
69
+ await llmq.start() # a failure here should fail the pod: queued models need it
70
+ yield
71
+ llmq.pause_receiving() # not ready; no new queued work (also from a preStop /drain)
72
+ await llmq.stop(drain_secs=20) # one budget for running jobs and calls together
73
+ ```
74
+
75
+ `09_http_router.py` shows the `/drain` endpoint for the preStop hook and the health routes for
76
+ the probes. Outside dev, `LLMQ_STORE_URL` must be `rediss://` and `LLMQ_ID_HASH_SALT` set.
77
+
78
+ ## Measured on real Bedrock (us-east-1, 2026-11-12)
79
+
80
+ - **All four models** answered through the queue.
81
+ - **Embeddings:**
82
+
83
+ | Model | Similar pair | Unrelated pair | Dimensions |
84
+ |---|---|---|---|
85
+ | Titan v2 | 0.73 | 0.07 | 1024 |
86
+ | Cohere v3 | 0.83 | 0.37 | 1024 |
87
+
88
+ - **Streaming:** a warm stream of gpt-oss-20b took 0.83 s through the queue; the same prompt took 0.7 s called directly. Most of the time is the model's own.
89
+ - **Fallback:** with `gpt120b` off, the call went to `gpt20b` (`meta.fallback == "model"`).
90
+ - **Tool calls:** a LangChain tool call came back as `stock_price({"symbol": "RELIANCE"})`.
91
+ - **ElastiCache (Valkey 8.2, TLS) as the store:** all nine samples pass with `--live --store
92
+ env` (2026-11-13); the run left no keys behind.
93
+ - **ap-south-1:** gpt-oss-120b, gpt-oss-20b, Titan v2 and Cohere v3 answer there too (one call
94
+ each, 2026-11-13); the app's own paths ran end to end there through the queue.