llm-queue 0.1.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue-0.1.0b1/.gitignore +42 -0
- llm_queue-0.1.0b1/CHANGELOG.md +66 -0
- llm_queue-0.1.0b1/LICENSE +21 -0
- llm_queue-0.1.0b1/NOTICE +14 -0
- llm_queue-0.1.0b1/PKG-INFO +96 -0
- llm_queue-0.1.0b1/README.md +46 -0
- llm_queue-0.1.0b1/SECURITY.md +21 -0
- llm_queue-0.1.0b1/STABILITY.md +16 -0
- llm_queue-0.1.0b1/pyproject.toml +186 -0
- llm_queue-0.1.0b1/samples/README.md +94 -0
- llm_queue-0.1.0b1/src/llm_queue/__init__.py +105 -0
- llm_queue-0.1.0b1/src/llm_queue/_logging.py +41 -0
- llm_queue-0.1.0b1/src/llm_queue/_version.py +5 -0
- llm_queue-0.1.0b1/src/llm_queue/clients/__init__.py +42 -0
- llm_queue-0.1.0b1/src/llm_queue/clients/pool.py +271 -0
- llm_queue-0.1.0b1/src/llm_queue/clients/providers.py +360 -0
- llm_queue-0.1.0b1/src/llm_queue/clients/timing.py +52 -0
- llm_queue-0.1.0b1/src/llm_queue/context.py +54 -0
- llm_queue-0.1.0b1/src/llm_queue/core.py +482 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/__init__.py +3 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/_admission_lua.py +696 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/admission.py +243 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/cli.py +292 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/embed.py +103 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/envelope.py +102 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/errors.py +51 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/health.py +139 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/hooks.py +119 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/http.py +180 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/inference.py +121 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/jobs.py +137 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/keys.py +70 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/metrics.py +141 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/offload.py +82 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/queue.py +447 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/requests.py +317 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/runtime.py +2357 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/settings.py +186 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/status.py +267 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/store_client.py +65 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/store_guard.py +40 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/streaming.py +216 -0
- llm_queue-0.1.0b1/src/llm_queue/engine/tokens.py +129 -0
- llm_queue-0.1.0b1/src/llm_queue/errors.py +73 -0
- llm_queue-0.1.0b1/src/llm_queue/http_transport.py +211 -0
- llm_queue-0.1.0b1/src/llm_queue/ids.py +16 -0
- llm_queue-0.1.0b1/src/llm_queue/langchain/__init__.py +61 -0
- llm_queue-0.1.0b1/src/llm_queue/langchain/_client.py +94 -0
- llm_queue-0.1.0b1/src/llm_queue/langchain/bedrock.py +312 -0
- llm_queue-0.1.0b1/src/llm_queue/langchain/openai.py +102 -0
- llm_queue-0.1.0b1/src/llm_queue/providers/__init__.py +43 -0
- llm_queue-0.1.0b1/src/llm_queue/providers/classify.py +251 -0
- llm_queue-0.1.0b1/src/llm_queue/providers/embeddings.py +87 -0
- llm_queue-0.1.0b1/src/llm_queue/py.typed +0 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/__init__.py +36 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/_lua.py +78 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/resolve.py +199 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/schema.py +521 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/settings.py +364 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/store.py +614 -0
- llm_queue-0.1.0b1/src/llm_queue/registry/validate.py +258 -0
- llm_queue-0.1.0b1/src/llm_queue/tuning.py +151 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Environments and secrets (never commit tokens: GitHub, PyPI, AWS)
|
|
2
|
+
.env
|
|
3
|
+
.env.*
|
|
4
|
+
!.env.example
|
|
5
|
+
.pypirc
|
|
6
|
+
*.pem
|
|
7
|
+
*.key
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
|
|
11
|
+
# Python build and cache
|
|
12
|
+
__pycache__/
|
|
13
|
+
*.py[cod]
|
|
14
|
+
*.egg-info/
|
|
15
|
+
build/
|
|
16
|
+
dist/
|
|
17
|
+
.pytest_cache/
|
|
18
|
+
.ruff_cache/
|
|
19
|
+
.mypy_cache/
|
|
20
|
+
.coverage
|
|
21
|
+
coverage.xml
|
|
22
|
+
htmlcov/
|
|
23
|
+
|
|
24
|
+
# Test and tool output
|
|
25
|
+
.hypothesis/
|
|
26
|
+
.locust/
|
|
27
|
+
*.log
|
|
28
|
+
reports/
|
|
29
|
+
|
|
30
|
+
# Editors and OS
|
|
31
|
+
.idea/
|
|
32
|
+
.vscode/
|
|
33
|
+
*.swp
|
|
34
|
+
.DS_Store
|
|
35
|
+
Thumbs.db
|
|
36
|
+
|
|
37
|
+
# Docs site build (zensical build)
|
|
38
|
+
site/
|
|
39
|
+
.cache/
|
|
40
|
+
|
|
41
|
+
# Internal working notes (release and test plans): kept locally, never published
|
|
42
|
+
docs/internal/
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes are listed here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[semantic versioning](https://semver.org/); while it is 0.x, a minor release may break the
|
|
6
|
+
public API (see STABILITY.md).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0b1] - 2026-11-14
|
|
11
|
+
|
|
12
|
+
First beta, for test runs (`pip install --pre llm-queue`).
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- **Queued LLM calls on Valkey / Redis.** `LLMQueue.from_env()` runs calls to Amazon Bedrock
|
|
17
|
+
and OpenAI-compatible servers (vLLM) through a queue per model (a normal and a priority
|
|
18
|
+
stream) on one Redis-protocol store. Every pod runs the same code as caller and worker
|
|
19
|
+
(`LLMQ_MODE`: `both`, `caller`, `worker`); budgets, waiter order and health live in the
|
|
20
|
+
store, so one pod and fifty behave the same.
|
|
21
|
+
- **Budgets per deployment** (model x account x region): RPM and TPM (`smooth` GCRA or `fixed`
|
|
22
|
+
per minute), account budgets shared by models, concurrency slots learned from throttles,
|
|
23
|
+
priority reserve and normal minimum share. Admission is one atomic store script: choose a
|
|
24
|
+
deployment (`ordered`, `round_robin`, `weighted`, `headroom`, `custom:`), take every budget
|
|
25
|
+
or none, keep waiters in arrival order.
|
|
26
|
+
- **Routing and recovery:** fallback regions and fallback models by cause (`exhausted`,
|
|
27
|
+
`broken`, `error`) after a wait; retries with backoff, a per-model retry budget (moving to a
|
|
28
|
+
deployment the call has not tried is free) and one deadline per call; circuit breaker per
|
|
29
|
+
deployment (calls wait for its probe); health with auto-disable, probes and a last-healthy
|
|
30
|
+
safeguard; slow deployments become last choices; credentials judged per account and region.
|
|
31
|
+
- **Registry** (`LLMQ_REGISTRY`): accounts (AWS by pod identity or assumed role,
|
|
32
|
+
OpenAI-compatible endpoints), models with confirmed and fallback deployments, residency,
|
|
33
|
+
routing (kill switch, `queue_pct`, blocked regions), every setting at four levels with
|
|
34
|
+
types, ranges and defaults, validated with every problem listed. Stored in the store with a
|
|
35
|
+
version rule, live edits with audit, rollback and restore after a store reset.
|
|
36
|
+
- **Calls:** `converse` / `chat`, `converse_stream` (events through a chunk stream;
|
|
37
|
+
`CallFailed.partial_text` on a cut), `embed` (batched per model, vectors in order), sync
|
|
38
|
+
calls (`*_sync`, `close_sync`), jobs (`register_job` / `send_job`, at least once),
|
|
39
|
+
`request_context` (priority, session, user, tenant), duplicate request ids run once,
|
|
40
|
+
cancel, large results in S3. LangChain: `QueuedChatBedrockConverse`,
|
|
41
|
+
`QueuedBedrockEmbeddings`, `QueuedChatOpenAI`, `QueuedOpenAIEmbeddings`.
|
|
42
|
+
- **Errors:** every failed call is `CallFailed` with a `kind` (`rate_limited`,
|
|
43
|
+
`deadline_exceeded`, `model_unavailable`, `queue_unavailable`, ...) and `retry_after_secs`
|
|
44
|
+
where a retry can work; admin operations raise `StoreUnavailable`; bad configuration is
|
|
45
|
+
`ConfigError` (`UnknownModel`, `MissingExtra`). Everything raised is an `LLMQueueError`.
|
|
46
|
+
- **Store resilience:** reconnect on `READONLY` and dropped connections (failover), a read
|
|
47
|
+
timeout that is never retried, a blocking connection pool, `store_down` rules (`pause`,
|
|
48
|
+
`static_cap`, `fail_fast`), callers kept alive through a store outage, calls held past their
|
|
49
|
+
deadline let go, optional separate queue store with failover to the direct path, optional
|
|
50
|
+
`WAIT` for replicas.
|
|
51
|
+
- **Tuning in code:** `Tuning` (`StoreOptions`, `BedrockOptions`, `EngineTimings`,
|
|
52
|
+
`RegistryOptions`) passed to `LLMQueue.from_env(tuning=...)`: every timeout, pool size and
|
|
53
|
+
pacing value (admission scripts included) with a measured default, validated at start-up.
|
|
54
|
+
- **Operations:** status and request status, alerts, metrics (CloudWatch EMF, Prometheus), one
|
|
55
|
+
`llmq_call_finished` record per call, `health()` with per-store state, `pause_receiving()`
|
|
56
|
+
for a preStop hook and a bounded `stop(drain_secs=)`, the `llm-queue` CLI (status, registry,
|
|
57
|
+
settings, block, drain, dead letters, queues) and a FastAPI router (`http` extra).
|
|
58
|
+
- **Testing:** fakellm (a fake Bedrock / OpenAI server), `scripts/check.py` (lint, types, tests
|
|
59
|
+
per step), fault scenarios by call count (`scripts/scenarios.py`: worker, store, provider,
|
|
60
|
+
registry, caller and multi-client faults under three loads), load and chaos gates, a soak
|
|
61
|
+
with chaos, a store failover drill, and runnable samples on fakellm or real Bedrock.
|
|
62
|
+
- **Docs site** (Zensical on GitHub Pages): tutorial, how-to guides, explanation with
|
|
63
|
+
diagrams, an API reference from the docstrings.
|
|
64
|
+
|
|
65
|
+
[Unreleased]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b1...HEAD
|
|
66
|
+
[0.1.0b1]: https://github.com/techiewonk/llm-queue/releases/tag/v0.1.0b1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The llm-queue authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
llm_queue-0.1.0b1/NOTICE
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
llm-queue
|
|
2
|
+
Copyright (c) 2026 The llm-queue authors. Licensed under the MIT License.
|
|
3
|
+
|
|
4
|
+
The queue's design (visibility heartbeat, message metadata, large results in S3) builds on
|
|
5
|
+
ideas from two MIT-licensed SQS brokers. This package is an independent implementation on
|
|
6
|
+
Valkey / Redis streams; any code adapted from them keeps its original notice below.
|
|
7
|
+
|
|
8
|
+
taskiq-aio-sqs — https://github.com/vonsteer/taskiq-aio-sqs
|
|
9
|
+
Copyright (c) 2022-2024 Jesse Constante. MIT License.
|
|
10
|
+
Ideas used: visibility heartbeat, message metadata, extended (S3) messages.
|
|
11
|
+
|
|
12
|
+
taskiq-sqs — https://github.com/taskiq-python/taskiq-sqs
|
|
13
|
+
Copyright (c) 2026 Mike Shultz and Taskiq team. MIT License.
|
|
14
|
+
Ideas used: message expiry, flush on shutdown, S3 offload of large results.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: llm-queue
|
|
3
|
+
Version: 0.1.0b1
|
|
4
|
+
Summary: LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status.
|
|
5
|
+
Project-URL: Homepage, https://github.com/techiewonk/llm-queue
|
|
6
|
+
Project-URL: Documentation, https://techiewonk.github.io/llm-queue/
|
|
7
|
+
Project-URL: Source, https://github.com/techiewonk/llm-queue
|
|
8
|
+
Project-URL: Issues, https://github.com/techiewonk/llm-queue/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md
|
|
10
|
+
Author: The llm-queue authors
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
License-File: NOTICE
|
|
14
|
+
Keywords: bedrock,langchain,langgraph,llm,openai,queue,rate-limit,redis,valkey
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Framework :: AsyncIO
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.11
|
|
29
|
+
Requires-Dist: aiobotocore<4,>=2.23
|
|
30
|
+
Requires-Dist: hiredis<4,>=3
|
|
31
|
+
Requires-Dist: orjson<4,>=3.10
|
|
32
|
+
Requires-Dist: pydantic<3,>=2.10
|
|
33
|
+
Requires-Dist: redis<9,>=5.0.1
|
|
34
|
+
Requires-Dist: structlog<27,>=24.1
|
|
35
|
+
Requires-Dist: uuid-utils<1,>=0.19
|
|
36
|
+
Provides-Extra: http
|
|
37
|
+
Requires-Dist: fastapi<1,>=0.115.3; extra == 'http'
|
|
38
|
+
Provides-Extra: langchain-aws
|
|
39
|
+
Requires-Dist: langchain-aws<2,>=0.2.24; extra == 'langchain-aws'
|
|
40
|
+
Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-aws'
|
|
41
|
+
Provides-Extra: langchain-openai
|
|
42
|
+
Requires-Dist: langchain-core<2,>=0.3.68; extra == 'langchain-openai'
|
|
43
|
+
Requires-Dist: langchain-openai<2,>=0.3.26; extra == 'langchain-openai'
|
|
44
|
+
Provides-Extra: metrics
|
|
45
|
+
Requires-Dist: prometheus-client<1,>=0.20; extra == 'metrics'
|
|
46
|
+
Provides-Extra: openai
|
|
47
|
+
Requires-Dist: httpx<1,>=0.27; extra == 'openai'
|
|
48
|
+
Requires-Dist: openai<4,>=1.40; extra == 'openai'
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
# llm-queue
|
|
52
|
+
|
|
53
|
+
LLM calls through a queue on Valkey or Redis, under RPM / TPM budgets per **deployment**
|
|
54
|
+
(model x account x region), balanced across regions and accounts, with fallback regions,
|
|
55
|
+
fallback models, health checks and status. Works with Amazon Bedrock and OpenAI-compatible
|
|
56
|
+
servers (vLLM), and with LangChain. The queue, the budgets and the results live in one store, so
|
|
57
|
+
admission (deployment, RPM, TPM, concurrency slot) is one atomic step, and 1 pod behaves the
|
|
58
|
+
same as 50.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
pip install "llm-queue[langchain-aws]" # Bedrock + LangChain
|
|
64
|
+
pip install "llm-queue[openai,langchain-openai]" # OpenAI-compatible
|
|
65
|
+
pip install "llm-queue[http]" # status and health routes for FastAPI
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Example
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from llm_queue import LLMQueue, CallFailed, request_context
|
|
72
|
+
|
|
73
|
+
llmq = LLMQueue.from_env() # LLMQ_STORE_URL, LLMQ_REGISTRY, ...
|
|
74
|
+
await llmq.start() # publish / load the registry, run the workers
|
|
75
|
+
try:
|
|
76
|
+
with request_context(priority=True, user_id=uid):
|
|
77
|
+
reply = await llmq.converse("gpt120b", {"messages": [...]}, max_wait_secs=5)
|
|
78
|
+
print(reply.response, reply.meta.deployment)
|
|
79
|
+
except CallFailed as exc: # exc.kind: rate_limited, model_unavailable, ...
|
|
80
|
+
...
|
|
81
|
+
await llmq.stop()
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Documentation
|
|
85
|
+
|
|
86
|
+
Full documentation: **https://techiewonk.github.io/llm-queue/**
|
|
87
|
+
|
|
88
|
+
- [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
|
|
89
|
+
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, tune, call, operate, share models across services, test
|
|
90
|
+
- [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, failures and recovery
|
|
91
|
+
- [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): settings, error kinds, CLI and HTTP, the Python API
|
|
92
|
+
|
|
93
|
+
Project: [CHANGELOG](https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md),
|
|
94
|
+
[CONTRIBUTING](https://github.com/techiewonk/llm-queue/blob/main/CONTRIBUTING.md),
|
|
95
|
+
[STABILITY](https://github.com/techiewonk/llm-queue/blob/main/STABILITY.md),
|
|
96
|
+
[SECURITY](https://github.com/techiewonk/llm-queue/blob/main/SECURITY.md).
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# llm-queue
|
|
2
|
+
|
|
3
|
+
LLM calls through a queue on Valkey or Redis, under RPM / TPM budgets per **deployment**
|
|
4
|
+
(model x account x region), balanced across regions and accounts, with fallback regions,
|
|
5
|
+
fallback models, health checks and status. Works with Amazon Bedrock and OpenAI-compatible
|
|
6
|
+
servers (vLLM), and with LangChain. The queue, the budgets and the results live in one store, so
|
|
7
|
+
admission (deployment, RPM, TPM, concurrency slot) is one atomic step, and 1 pod behaves the
|
|
8
|
+
same as 50.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
pip install "llm-queue[langchain-aws]" # Bedrock + LangChain
|
|
14
|
+
pip install "llm-queue[openai,langchain-openai]" # OpenAI-compatible
|
|
15
|
+
pip install "llm-queue[http]" # status and health routes for FastAPI
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Example
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from llm_queue import LLMQueue, CallFailed, request_context
|
|
22
|
+
|
|
23
|
+
llmq = LLMQueue.from_env() # LLMQ_STORE_URL, LLMQ_REGISTRY, ...
|
|
24
|
+
await llmq.start() # publish / load the registry, run the workers
|
|
25
|
+
try:
|
|
26
|
+
with request_context(priority=True, user_id=uid):
|
|
27
|
+
reply = await llmq.converse("gpt120b", {"messages": [...]}, max_wait_secs=5)
|
|
28
|
+
print(reply.response, reply.meta.deployment)
|
|
29
|
+
except CallFailed as exc: # exc.kind: rate_limited, model_unavailable, ...
|
|
30
|
+
...
|
|
31
|
+
await llmq.stop()
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Documentation
|
|
35
|
+
|
|
36
|
+
Full documentation: **https://techiewonk.github.io/llm-queue/**
|
|
37
|
+
|
|
38
|
+
- [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
|
|
39
|
+
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, tune, call, operate, share models across services, test
|
|
40
|
+
- [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, failures and recovery
|
|
41
|
+
- [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): settings, error kinds, CLI and HTTP, the Python API
|
|
42
|
+
|
|
43
|
+
Project: [CHANGELOG](https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md),
|
|
44
|
+
[CONTRIBUTING](https://github.com/techiewonk/llm-queue/blob/main/CONTRIBUTING.md),
|
|
45
|
+
[STABILITY](https://github.com/techiewonk/llm-queue/blob/main/STABILITY.md),
|
|
46
|
+
[SECURITY](https://github.com/techiewonk/llm-queue/blob/main/SECURITY.md).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Security policy
|
|
2
|
+
|
|
3
|
+
## Supported versions
|
|
4
|
+
|
|
5
|
+
| Version | Supported |
|
|
6
|
+
| ------- | --------- |
|
|
7
|
+
| 0.3.x | Yes |
|
|
8
|
+
| < 0.3 | No |
|
|
9
|
+
|
|
10
|
+
## Reporting a vulnerability
|
|
11
|
+
|
|
12
|
+
Please do not open a public issue for a security problem.
|
|
13
|
+
|
|
14
|
+
Report it privately through GitHub's private vulnerability reporting:
|
|
15
|
+
[Security > Report a vulnerability](https://github.com/techiewonk/llm-queue/security/advisories/new)
|
|
16
|
+
on the `techiewonk/llm-queue` repository. This opens a draft security advisory that only the
|
|
17
|
+
maintainers can see.
|
|
18
|
+
|
|
19
|
+
Include the affected version, a description of the problem, and steps or code to reproduce it.
|
|
20
|
+
We aim to acknowledge a report within a few working days, and will coordinate a fix and a
|
|
21
|
+
GitHub Security Advisory (with a CVE where it applies) before any public disclosure.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Stability policy
|
|
2
|
+
|
|
3
|
+
- **Public API** = the names in `llm_queue.__all__`, the registry document's setting
|
|
4
|
+
names and the `llm-queue` commands. Anything else may change without notice.
|
|
5
|
+
- **0.x releases:** a minor release may break the public API; every break is in CHANGELOG.md.
|
|
6
|
+
- **Message format:** queue messages carry a format version `v`. A worker returns a message newer
|
|
7
|
+
than it understands to the queue instead of dropping it.
|
|
8
|
+
- **Store schema:** the key layout has a schema number; a change to it ships with a migration note.
|
|
9
|
+
- **Supported versions:** Python 3.11-3.13 (3.14 runs in CI, not yet supported); Valkey 8
|
|
10
|
+
and Redis 7.2+ as one primary with replicas (cluster mode is not supported: one admission
|
|
11
|
+
touches keys of several hash tags). The store must allow Lua scripts, pub/sub and blocking
|
|
12
|
+
list reads (`BLPOP`, the queue doorbell); ElastiCache for Valkey / Redis OSS does.
|
|
13
|
+
- **Dependencies:** ranges with upper bounds in `pyproject.toml` (`aiobotocore<4`, `redis<9`,
|
|
14
|
+
...); a release may raise a bound, never drop one without a test on the new major.
|
|
15
|
+
- **Settings:** the `LLMQ_*` names and the registry setting names are public API; removing or
|
|
16
|
+
renaming one is a breaking change listed in CHANGELOG.md.
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "llm-queue"
|
|
3
|
+
description = "LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status."
|
|
4
|
+
readme = "README.md"
|
|
5
|
+
requires-python = ">=3.11"
|
|
6
|
+
dynamic = ["version"]
|
|
7
|
+
# PEP 639 licence expression; NOTICE credits the projects the queue design builds on.
|
|
8
|
+
license = "MIT"
|
|
9
|
+
license-files = ["LICENSE", "NOTICE"]
|
|
10
|
+
authors = [{ name = "The llm-queue authors" }]
|
|
11
|
+
keywords = [
|
|
12
|
+
"valkey", "redis", "llm", "bedrock", "openai", "queue", "rate-limit", "langchain", "langgraph",
|
|
13
|
+
]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Framework :: AsyncIO",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
25
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
26
|
+
"Topic :: System :: Distributed Computing",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# Ranges, not pins: a library that pins exact versions locks every user into one botocore
|
|
31
|
+
# patch window. Exact versions live only in the consuming app's lockfile.
|
|
32
|
+
dependencies = [
|
|
33
|
+
# Async Bedrock (and S3 offload) client. aiobotocore pins a narrow botocore window.
|
|
34
|
+
# Upper bound: the pool reads aiobotocore internals (refreshable credentials).
|
|
35
|
+
"aiobotocore>=2.23,<4",
|
|
36
|
+
# Queues, budgets, registry and results on any Redis-protocol store (Valkey, Redis).
|
|
37
|
+
"redis>=5.0.1,<9",
|
|
38
|
+
# C RESP parser, picked up by redis-py: Lua array replies and stream reads parse 39-75x
|
|
39
|
+
# faster than redis-py's pure-Python parser (measured 2026-11-13).
|
|
40
|
+
"hiredis>=3,<4",
|
|
41
|
+
"pydantic>=2.10,<3",
|
|
42
|
+
# JSON on every hot path (envelopes, params, provider bodies): an EMF line 0.8 us vs 6.2 us,
|
|
43
|
+
# a 200 KB request round trip 0.38 ms vs 0.95 ms with stdlib json (measured 2026-11-13).
|
|
44
|
+
"orjson>=3.10,<4",
|
|
45
|
+
# Structured key-value logs. The library never configures it; the app does (README).
|
|
46
|
+
"structlog>=24.1,<27",
|
|
47
|
+
# UUIDv7 ids (time-ordered), Rust: 0.49 us per id vs 2.4 us for uuid.uuid4().hex (measured).
|
|
48
|
+
"uuid-utils>=0.19,<1",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[project.scripts]
|
|
52
|
+
llm-queue = "llm_queue.engine.cli:main"
|
|
53
|
+
|
|
54
|
+
[project.optional-dependencies]
|
|
55
|
+
openai = ["openai>=1.40,<4", "httpx>=0.27,<1"]
|
|
56
|
+
# Both LangChain lines are supported: 0.3 and 1.x (the adapters are tested on each).
|
|
57
|
+
langchain-aws = ["langchain-aws>=0.2.24,<2", "langchain-core>=0.3.68,<2"]
|
|
58
|
+
langchain-openai = ["langchain-openai>=0.3.26,<2", "langchain-core>=0.3.68,<2"]
|
|
59
|
+
metrics = ["prometheus-client>=0.20,<1"]
|
|
60
|
+
# Status / registry / health routes for a FastAPI app (`LLMQueue.http_router`).
|
|
61
|
+
# 0.115.3+: Starlette with the multipart DoS fixes (CVE-2024-47874).
|
|
62
|
+
http = ["fastapi>=0.115.3,<1"]
|
|
63
|
+
|
|
64
|
+
[project.urls]
|
|
65
|
+
Homepage = "https://github.com/techiewonk/llm-queue"
|
|
66
|
+
Documentation = "https://techiewonk.github.io/llm-queue/"
|
|
67
|
+
Source = "https://github.com/techiewonk/llm-queue"
|
|
68
|
+
Issues = "https://github.com/techiewonk/llm-queue/issues"
|
|
69
|
+
Changelog = "https://github.com/techiewonk/llm-queue/blob/main/CHANGELOG.md"
|
|
70
|
+
|
|
71
|
+
[dependency-groups]
|
|
72
|
+
dev = [
|
|
73
|
+
"pytest>=8",
|
|
74
|
+
"pytest-asyncio>=0.24",
|
|
75
|
+
"ruff>=0.6",
|
|
76
|
+
"mypy>=1.11",
|
|
77
|
+
"types-aiobotocore[s3,bedrock-runtime,sts]",
|
|
78
|
+
"moto[server]>=5",
|
|
79
|
+
"fakeredis>=2.26",
|
|
80
|
+
# extras installed for development and tests of the provider/adapter layers
|
|
81
|
+
# The dev env is on the LangChain 1.x line; the 0.3 line is checked in CI (langchain-03 job).
|
|
82
|
+
"openai>=1.40,<4",
|
|
83
|
+
"httpx>=0.27,<1",
|
|
84
|
+
"langchain-aws>=1,<2",
|
|
85
|
+
"langchain-openai>=1,<2",
|
|
86
|
+
"langchain-core>=1,<2",
|
|
87
|
+
"langgraph>=1",
|
|
88
|
+
"fastapi>=0.115.3,<1",
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
[build-system]
|
|
92
|
+
requires = ["hatchling>=1.27"] # PEP 639 license-files
|
|
93
|
+
build-backend = "hatchling.build"
|
|
94
|
+
|
|
95
|
+
[tool.hatch.version]
|
|
96
|
+
path = "src/llm_queue/_version.py"
|
|
97
|
+
|
|
98
|
+
[tool.hatch.build.targets.wheel]
|
|
99
|
+
packages = ["src/llm_queue"]
|
|
100
|
+
|
|
101
|
+
[tool.hatch.build.targets.sdist]
|
|
102
|
+
# Allow-list: secrets (.env, .pypirc) and local tooling can never reach the sdist.
|
|
103
|
+
# The design plan, walkthrough and release / test plans are internal: never in the sdist.
|
|
104
|
+
include = ["src/", "README.md", "LICENSE", "NOTICE", "CHANGELOG.md", "STABILITY.md", "SECURITY.md"]
|
|
105
|
+
|
|
106
|
+
[tool.ruff]
|
|
107
|
+
# target-version comes from project.requires-python.
|
|
108
|
+
line-length = 100
|
|
109
|
+
extend-exclude = ["*.md"]
|
|
110
|
+
|
|
111
|
+
[tool.ruff.lint]
|
|
112
|
+
select = ["ASYNC", "B", "C4", "D", "E", "F", "I", "PIE", "PT", "RET", "RUF", "SIM", "UP", "W"]
|
|
113
|
+
ignore = ["D105"]
|
|
114
|
+
|
|
115
|
+
[tool.ruff.lint.pydocstyle]
|
|
116
|
+
convention = "google"
|
|
117
|
+
|
|
118
|
+
[tool.ruff.lint.per-file-ignores]
|
|
119
|
+
"tests/*" = ["D"]
|
|
120
|
+
"samples/*" = ["D103"] # samples read top to bottom; the module docstring says it all
|
|
121
|
+
|
|
122
|
+
[tool.mypy]
|
|
123
|
+
strict = true
|
|
124
|
+
python_version = "3.11"
|
|
125
|
+
files = ["src", "tests", "tools", "scripts", "samples"]
|
|
126
|
+
# Test folders have no __init__.py (several conftest.py files): map modules from paths.
|
|
127
|
+
explicit_package_bases = true
|
|
128
|
+
mypy_path = ["src", "tools/fakellm"]
|
|
129
|
+
plugins = ["pydantic.mypy"]
|
|
130
|
+
warn_unreachable = true
|
|
131
|
+
enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
|
|
132
|
+
|
|
133
|
+
[[tool.mypy.overrides]]
|
|
134
|
+
module = ["moto.*", "fastapi", "fastapi.*"]
|
|
135
|
+
ignore_missing_imports = true
|
|
136
|
+
|
|
137
|
+
# FastAPI is an optional extra: its decorators are untyped when it is not installed.
|
|
138
|
+
[[tool.mypy.overrides]]
|
|
139
|
+
module = ["llm_queue.engine.http"]
|
|
140
|
+
disallow_untyped_decorators = false
|
|
141
|
+
|
|
142
|
+
# Test helpers are imported by bare name (pytest puts their folder on sys.path); the helper
|
|
143
|
+
# files themselves are still type-checked as part of `files`.
|
|
144
|
+
[[tool.mypy.overrides]]
|
|
145
|
+
module = ["fakellm_testkit", "providers_fake", "clients_testkit", "engine_testkit", "gates_fleet", "failover", "gates"]
|
|
146
|
+
ignore_missing_imports = true
|
|
147
|
+
|
|
148
|
+
[tool.pytest.ini_options]
|
|
149
|
+
minversion = "8"
|
|
150
|
+
# -ra: a short summary of every non-passing outcome (scripts/check.py passes -rA on top).
|
|
151
|
+
addopts = ["-ra", "--strict-config", "--strict-markers"]
|
|
152
|
+
xfail_strict = true
|
|
153
|
+
log_level = "INFO"
|
|
154
|
+
filterwarnings = [
|
|
155
|
+
"error",
|
|
156
|
+
# Python 3.14: langsmith (under langchain-core) calls asyncio.iscoroutinefunction while its
|
|
157
|
+
# pytest plugin loads; our code does not use it.
|
|
158
|
+
"ignore:'asyncio.iscoroutinefunction' is deprecated:DeprecationWarning",
|
|
159
|
+
]
|
|
160
|
+
pythonpath = ["src"]
|
|
161
|
+
testpaths = ["tests/unit"]
|
|
162
|
+
asyncio_mode = "auto"
|
|
163
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
164
|
+
markers = [
|
|
165
|
+
"integration: needs the Docker stack (docker/compose.yaml)",
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
[tool.repo-review]
|
|
169
|
+
ignore = [
|
|
170
|
+
# PY007 (nox/tox): scripts/check.py is the task runner (CONTRIBUTING.md); it also drives the
|
|
171
|
+
# Docker stack steps, which nox/tox would only wrap.
|
|
172
|
+
"PY007",
|
|
173
|
+
# GH200 (Dependabot): dependency updates come from Renovate (Renovate app on the repo);
|
|
174
|
+
# Dependabot as well would open duplicate update PRs.
|
|
175
|
+
"GH200",
|
|
176
|
+
# PC140 (mypy hook): strict mypy needs the full locked env (extras, stubs, plugin);
|
|
177
|
+
# scripts/check.py and CI run it.
|
|
178
|
+
"PC140",
|
|
179
|
+
# PC160 / PC180 (spell checker, Markdown formatter): not adopted yet; follow-up.
|
|
180
|
+
"PC160",
|
|
181
|
+
"PC180",
|
|
182
|
+
# PC901-PC903: pre-commit.ci is not used; CI runs scripts/check.py's steps.
|
|
183
|
+
"PC901",
|
|
184
|
+
"PC902",
|
|
185
|
+
"PC903",
|
|
186
|
+
]
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Samples
|
|
2
|
+
|
|
3
|
+
Runnable examples of every way an app uses llm-queue. Each file is plain app code:
|
|
4
|
+
it builds the queue with `LLMQueue.from_env()` and reads nothing else. The runner only sets
|
|
5
|
+
the environment.
|
|
6
|
+
|
|
7
|
+
| Sample | Shows | Use it in |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| `01_converse.py` | `converse` with `request_context` (priority, session, user); reply text and `meta` | Async request handlers (FastAPI routes), agent steps, anything user-facing |
|
|
10
|
+
| `02_stream.py` | `converse_stream`: events as they arrive; `CallFailed.partial_text` on a cut | Chat UIs and server-sent events, where the first words should show at once |
|
|
11
|
+
| `03_embed.py` | `embed` on Titan (1 text per request) and Cohere (96 per request, `input_type`) | Indexing documents (`search_document`) and search queries (`search_query`) |
|
|
12
|
+
| `04_sync.py` | `converse_sync` / `embed_sync` on the direct path, `close_sync()` | Sync code only: scripts, Celery tasks, sync libraries. Async code uses 01/03 |
|
|
13
|
+
| `05_errors_and_fallback.py` | Model fallback when a model is off; `CallFailed.kind` to branch on | Every caller: map `kind` to a user message or a retry |
|
|
14
|
+
| `06_jobs.py` | `register_job` / `send_job`: work that runs at least once, LLM calls inside | Ingestion, nightly re-embedding, any batch work that should not block a request |
|
|
15
|
+
| `07_status_and_registry.py` | `status`, `request_status`, live `registry.set`, audit `log`, `health` | Admin pages, runbooks, readiness checks |
|
|
16
|
+
| `08_langchain.py` | `QueuedChatBedrockConverse` (with tools) and `QueuedBedrockEmbeddings` | LangChain / LangGraph chains, agents, RAG: drop-in for `ChatBedrockConverse` / `BedrockEmbeddings` |
|
|
17
|
+
| `09_http_router.py` | `http_router` on FastAPI: health without auth, status / registry behind the app's admin check, audited writes (`actor=`), `/drain` with `pause_receiving()` | Admin API and Kubernetes probes; the preStop hook |
|
|
18
|
+
|
|
19
|
+
`registry.json` defines the four models the samples use: chat `gpt120b` (falls back to `gpt20b`)
|
|
20
|
+
and `gpt20b`, and embeddings `titan` and `cohere`, all on one AWS account. The quotas are
|
|
21
|
+
small sample numbers. Use your Service Quotas numbers in a real registry.
|
|
22
|
+
|
|
23
|
+
## Run them
|
|
24
|
+
|
|
25
|
+
Start the local stack once (Valkey, plus fakellm as a fake Bedrock):
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
python scripts/stack.py up
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Then run the samples from the library root:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
python scripts/samples.py # all, on fakellm (no real LLM, no cost)
|
|
35
|
+
python scripts/samples.py 03_embed 08_langchain # some
|
|
36
|
+
python scripts/samples.py --live # all, on real Bedrock
|
|
37
|
+
python scripts/samples.py --live --env-file ../../.env # AWS keys from another .env
|
|
38
|
+
python scripts/samples.py --live --store env --env-file ../../.env # and the env file's store
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
- **Stored keys:** each run uses its own key prefix on the local Valkey and deletes its keys at the end.
|
|
42
|
+
- **What `--live` reads:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_MODEL_REGION` from the env file. It never prints them.
|
|
43
|
+
- **Region:** every model is placed in that region.
|
|
44
|
+
- **Cost:** a full live run makes about 20 small calls, well under a cent.
|
|
45
|
+
- **`--store env`:** the store is the env file's `LLMQ_STORE_URL`, or `rediss://` built from
|
|
46
|
+
the app's `valkey_host` / `valkey_port` / `valkey_key`. Only the run's own prefix is touched
|
|
47
|
+
and deleted, so a shared store (ElastiCache) is safe.
|
|
48
|
+
|
|
49
|
+
`python scripts/check.py samples` runs the fakellm pass with lint and type checks, as part of
|
|
50
|
+
`check.py all`.
|
|
51
|
+
|
|
52
|
+
## Use them in an app
|
|
53
|
+
|
|
54
|
+
An app sets the same environment the runner sets, from its secrets:
|
|
55
|
+
|
|
56
|
+
| Variable | Runner value | In an app |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| `LLMQ_STORE_URL` | `redis://:localdev@127.0.0.1:6379/0` | `rediss://...` of the ElastiCache Valkey |
|
|
59
|
+
| `LLMQ_KEY_PREFIX`, `LLMQ_QUEUE_PREFIX` | per run | per environment, e.g. `llmq:prod` |
|
|
60
|
+
| `LLMQ_REGISTRY` | `samples/registry.json`, region set | the app's registry document |
|
|
61
|
+
| `LLMQ_METRICS_SINK` | `none` | `emf` (CloudWatch) or `prometheus` |
|
|
62
|
+
| AWS credentials | from `.env` (or fake for fakellm) | the pod's role; `role_arn` accounts assume roles |
|
|
63
|
+
|
|
64
|
+
Start the queue once per process, in the app's startup hook, and stop it on shutdown:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
@asynccontextmanager
|
|
68
|
+
async def lifespan(app):
|
|
69
|
+
await llmq.start() # a failure here should fail the pod: queued models need it
|
|
70
|
+
yield
|
|
71
|
+
llmq.pause_receiving() # not ready; no new queued work (also from a preStop /drain)
|
|
72
|
+
await llmq.stop(drain_secs=20) # one budget for running jobs and calls together
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`09_http_router.py` shows the `/drain` endpoint for the preStop hook and the health routes for
|
|
76
|
+
the probes. Outside dev, `LLMQ_STORE_URL` must be `rediss://` and `LLMQ_ID_HASH_SALT` set.
|
|
77
|
+
|
|
78
|
+
## Measured on real Bedrock (us-east-1, 2026-11-12)
|
|
79
|
+
|
|
80
|
+
- **All four models** answered through the queue.
|
|
81
|
+
- **Embeddings:**
|
|
82
|
+
|
|
83
|
+
| Model | Similar pair | Unrelated pair | Dimensions |
|
|
84
|
+
|---|---|---|---|
|
|
85
|
+
| Titan v2 | 0.73 | 0.07 | 1024 |
|
|
86
|
+
| Cohere v3 | 0.83 | 0.37 | 1024 |
|
|
87
|
+
|
|
88
|
+
- **Streaming:** a warm stream of gpt-oss-20b took 0.83 s through the queue; the same prompt took 0.7 s called directly. Most of the time is the model's own.
|
|
89
|
+
- **Fallback:** with `gpt120b` off, the call went to `gpt20b` (`meta.fallback == "model"`).
|
|
90
|
+
- **Tool calls:** a LangChain tool call came back as `stock_price({"symbol": "RELIANCE"})`.
|
|
91
|
+
- **ElastiCache (Valkey 8.2, TLS) as the store:** all nine samples pass with `--live --store
|
|
92
|
+
env` (2026-11-13); the run left no keys behind.
|
|
93
|
+
- **ap-south-1:** gpt-oss-120b, gpt-oss-20b, Titan v2 and Cohere v3 answer there too (one call
|
|
94
|
+
each, 2026-11-13); the app's own paths ran end to end there through the queue.
|