llm-queue 0.1.0b2__tar.gz → 0.1.0b3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/.gitignore +1 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/CHANGELOG.md +118 -1
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/PKG-INFO +11 -4
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/README.md +10 -3
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/STABILITY.md +6 -4
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/__init__.py +8 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/_version.py +1 -1
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/core.py +14 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/_admission_lua.py +18 -18
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/admission.py +9 -7
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/cli.py +73 -58
- llm_queue-0.1.0b3/src/llm_queue/engine/expiry.py +207 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/health.py +5 -3
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/http.py +160 -2
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/jobs.py +10 -5
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/metrics.py +9 -0
- llm_queue-0.1.0b3/src/llm_queue/engine/purge.py +906 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/queue.py +24 -22
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/requests.py +37 -34
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/runtime.py +162 -10
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/settings.py +60 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/status.py +6 -2
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/store_checks.py +24 -9
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/streaming.py +13 -5
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/errors.py +8 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/LICENSE +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/NOTICE +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/SECURITY.md +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/pyproject.toml +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/_logging.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/__init__.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/bedrock_http.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/hub.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/pool.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/providers.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/timing.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/context.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/__init__.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/_codec.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/cross_checks.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/embed.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/envelope.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/errors.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/fairness.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/hooks.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/inference.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/keys.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/offload.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/shapes.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/store_client.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/store_guard.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/tokens.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/http_transport.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/ids.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/__init__.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/_client.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/_stream.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/bedrock.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/openai.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/providers/__init__.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/providers/classify.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/providers/embeddings.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/py.typed +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/__init__.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/_lua.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/json_schema.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/resolve.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/schema.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/settings.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/store.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/validate.py +0 -0
- {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/tuning.py +0 -0
|
@@ -7,6 +7,56 @@ public API (see STABILITY.md).
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.1.0b3] - 2026-11-16
|
|
11
|
+
|
|
12
|
+
Key expiry modes, so llm-queue can share a `volatile-*` cache store (ElastiCache's default
|
|
13
|
+
`volatile-lru`), and a purge service for removed models, deployments and tenants.
|
|
14
|
+
|
|
15
|
+
### Upgrading from 0.1.0b2
|
|
16
|
+
|
|
17
|
+
The message format and the key layout are unchanged, so 0.1.0b2 and 0.1.0b3 pods can share a
|
|
18
|
+
queue on a `noeviction` store. [Upgrade](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
|
|
19
|
+
has the same list with more detail.
|
|
20
|
+
|
|
21
|
+
1. **Upgrade every pod.** 0.1.0b2 pods do not read the store's key expiry mode (`P:expiry`)
|
|
22
|
+
and always write native TTLs. On a `noeviction` store `auto` picks `native`, the same as
|
|
23
|
+
0.1.0b2, so a mixed fleet is fine there during a rolling upgrade. On a `volatile-*` store a
|
|
24
|
+
0.1.0b2 pod is not ready in strict mode anyway, and its keys stay evictable: finish the
|
|
25
|
+
rollout before you rely on the store being shared.
|
|
26
|
+
2. **Switching modes loses nothing.** In `swept` mode setting an expiry also clears the key's
|
|
27
|
+
native TTL (`PERSIST`); in `native` mode it also removes the key's `P:ttl` member; reading
|
|
28
|
+
the time left counts either. Keys move to the new mode on their next write; a key never
|
|
29
|
+
written again keeps its old expiry and still goes then.
|
|
30
|
+
3. **Rolling back to 0.1.0b2 after running `swept`.** 0.1.0b2 has no expiry sweep, so a key
|
|
31
|
+
whose expiry is only in `P:ttl` would never expire under it. Before rolling back, set
|
|
32
|
+
`LLMQ_KEY_EXPIRY=native` on the 0.1.0b3 pods (the store must be `noeviction` for 0.1.0b2
|
|
33
|
+
to be ready in strict mode) and keep them running for at least the longest key lifetime (1
|
|
34
|
+
day by default; longer if you raised a retention or TTL setting). In `native` mode each write
|
|
35
|
+
moves its key back to a native TTL, and the index sweep keeps running and deletes the
|
|
36
|
+
indexed keys as they come due. Roll back when `ZCARD <prefix>:ttl` is about 0 on each store
|
|
37
|
+
(a separate queue store has its own). Leftover `P:expiry`, `P:ttl` and `P:purge` keys are
|
|
38
|
+
harmless to 0.1.0b2.
|
|
39
|
+
4. **`llm-queue registry gc` output and scope.** It prints the purge report: `deleted_keys` and
|
|
40
|
+
`dry_run` instead of `would_delete`. It also purges removed deployments of live models, and
|
|
41
|
+
keeps a removed model's queue that still holds messages unless `--force`. Update scripts that
|
|
42
|
+
parse its output.
|
|
43
|
+
5. **Auto purge is on by default.** When a publish drops a model or deployment, its store state
|
|
44
|
+
is purged after `LLMQ_PURGE_GRACE_SECS` (900). Set `LLMQ_AUTO_PURGE=false` to keep the
|
|
45
|
+
0.1.0b2 behaviour (state stays until you purge it by hand).
|
|
46
|
+
|
|
47
|
+
### Changed (behaviour)
|
|
48
|
+
|
|
49
|
+
- **Auto purge on by default** (`LLMQ_AUTO_PURGE=true`): a model or deployment dropped from
|
|
50
|
+
the registry has its store state deleted after the grace (900 s) unless it comes back.
|
|
51
|
+
0.1.0b2 kept that state until `registry gc`. `LLMQ_AUTO_PURGE=false` restores that.
|
|
52
|
+
- **`auto` key expiry picks `swept` on a `volatile-*` store** (and on `allkeys-*` or an
|
|
53
|
+
unreadable policy): llm-queue's keys get no TTL and the sweep deletes them, up to 10 s late.
|
|
54
|
+
0.1.0b2 always set native TTLs. On `noeviction` nothing changes. `LLMQ_KEY_EXPIRY=native`
|
|
55
|
+
restores the 0.1.0b2 behaviour (and leaves a `volatile-*` store not ready in strict mode).
|
|
56
|
+
- **`registry gc`** also purges removed deployments of live models, keeps non-empty queues
|
|
57
|
+
unless `--force`, and prints the purge report (`deleted_keys`, `dry_run`) instead of
|
|
58
|
+
`would_delete`.
|
|
59
|
+
|
|
10
60
|
### Added
|
|
11
61
|
|
|
12
62
|
- Docs: a reference page listing every log event the library writes, with its level and
|
|
@@ -17,6 +67,72 @@ public API (see STABILITY.md).
|
|
|
17
67
|
reference page on the document's shape with a complete example (`docs/reference/registry.md`).
|
|
18
68
|
- `LLMQueue.http_router(metrics_route=True)` serves `GET /metrics` (off by default); the option
|
|
19
69
|
existed only on `status_router(engine, ...)`, which needs the now private engine.
|
|
70
|
+
- Purge controls (`docs/how-to/purge.md`): `PurgeService` (`llmq.purge`, or
|
|
71
|
+
`PurgeService.from_env()` standalone) deletes the store state of a removed model
|
|
72
|
+
(`model`), a removed deployment (`deployment`, waiting for its running calls first), a tenant
|
|
73
|
+
(`tenant`, fairness keys with the weight sum corrected; optionally cancelling its queued or
|
|
74
|
+
running calls) or everything the registry no longer has (`removed`), with dry runs
|
|
75
|
+
(`plan_*`). Each returns a `PurgeReport`; real purges are audited (`purge_<kind>` in the
|
|
76
|
+
registry audit), logged `llmq_purge` and counted `llmq_purged_keys_total{kind}`. New error
|
|
77
|
+
`PurgeRefused` (a model or deployment still in the registry without `force`; a tenant purge of
|
|
78
|
+
calls without the tenant index).
|
|
79
|
+
- Auto purge: when a publish drops a model or deployment, it is purged after
|
|
80
|
+
`LLMQ_PURGE_GRACE_SECS` (900) unless it came back meanwhile (`LLMQ_AUTO_PURGE`, default on);
|
|
81
|
+
`purge.scheduled()` / `cancel_scheduled()` list and keep pending ones.
|
|
82
|
+
- Auto purge runs only in worker processes. A process's first tick after start only records
|
|
83
|
+
the registry it sees (a baseline), so a start never schedules a purge. Before it purges, it
|
|
84
|
+
reads the stored registry strictly and skips the purge on any read error or refusal. An auto
|
|
85
|
+
deployment purge does not wait for running calls: the grace (900 s) is as long as the longest
|
|
86
|
+
call the registry allows (`timeout_max_secs`, 900 s by default).
|
|
87
|
+
- CLI `llm-queue purge model|deployment|tenant --yes|--dry-run`; `--no-wait` on `purge
|
|
88
|
+
deployment` and `registry gc`. HTTP routes `POST /purge/model`, `/purge/deployment`,
|
|
89
|
+
`/purge/tenant`, `/registry/gc`, `GET /purge/scheduled`, `POST /purge/scheduled/cancel` (only
|
|
90
|
+
with `allow_writes=True`, `X-LLMQ-Admin`; `dry_run` defaults to true). An HTTP purge waits for
|
|
91
|
+
running calls at most `LLMQ_HTTP_PURGE_WAIT_SECS` (60, 0 to 3600); a larger or omitted
|
|
92
|
+
`wait_secs` is capped to it.
|
|
93
|
+
- Settings `LLMQ_AUTO_PURGE`, `LLMQ_PURGE_GRACE_SECS`, `LLMQ_PURGE_WAIT`,
|
|
94
|
+
`LLMQ_PURGE_WAIT_SECS`, `LLMQ_HTTP_PURGE_WAIT_SECS`, `LLMQ_PURGE_BATCH`, `LLMQ_TENANT_PURGE_DEFAULT` and
|
|
95
|
+
`LLMQ_TENANT_INDEX` (off: the per-tenant call index costs one write per call).
|
|
96
|
+
- Key expiry modes (`LLMQ_KEY_EXPIRY=auto|native|swept`, default `auto`): llm-queue can share a
|
|
97
|
+
store whose `maxmemory-policy` is `volatile-*` (ElastiCache's default `volatile-lru`, an app
|
|
98
|
+
cache). In `swept` no key the library needs has a TTL: each expiry is a member of the zset
|
|
99
|
+
`P:ttl` and the sweeper deletes due keys (up to `sweep_secs`, 10 s, late), so a `volatile-*`
|
|
100
|
+
policy cannot evict them. `auto` picks `native` on `noeviction` and `swept` otherwise. The mode
|
|
101
|
+
is the store's (`P:expiry`; missing means `native`), written at start, every minute and after
|
|
102
|
+
a store reset. New settings `LLMQ_TTL_SWEEP_BATCH` (500) and `LLMQ_TTL_SWEEP_ROUNDS` (100);
|
|
103
|
+
`health()["key_expiry"]`; gauge `llmq_key_expiry_swept{store}`; counter
|
|
104
|
+
`llmq_ttl_swept_total`; log events `key_expiry_changed`, `key_expiry_publish_failed`
|
|
105
|
+
(warning), `ttl_sweep_failed`. Until a process has published its mode it reports the mode its
|
|
106
|
+
setting asks for (`native` for `auto`). How-to: `docs/how-to/shared-cache-store.md`.
|
|
107
|
+
- Switching key expiry modes loses nothing: in `swept` mode setting an expiry also clears the
|
|
108
|
+
key's native TTL (`PERSIST`), in `native` mode it also removes the key's `P:ttl` member, and
|
|
109
|
+
reading the time left counts either; keys move to the new mode on their next write.
|
|
110
|
+
- The expiry sweep has its own lock, `P:lock:ttlsweep` (a native TTL on purpose), and runs in
|
|
111
|
+
every process, workers and callers, every `sweep_secs` (10 s): up to 500 x 100 = 50000 keys
|
|
112
|
+
per 10 s by default. The worker sweep keeps `P:lock:sweep`.
|
|
113
|
+
- `scripts/evict_drill.py`: an eviction drill that fills a throwaway store under queue traffic
|
|
114
|
+
with `volatile-lru` and `noeviction` (see CONTRIBUTING.md).
|
|
115
|
+
- Docs: a reference page listing every key, stream and channel the library keeps in the store,
|
|
116
|
+
with its fields, writers, readers, lifetime in each key expiry mode, what is lost if it is
|
|
117
|
+
evicted under each policy, and code references (`docs/reference/store-keys.md`, with a JSON
|
|
118
|
+
copy `store-keys.json`).
|
|
119
|
+
|
|
120
|
+
### Changed
|
|
121
|
+
|
|
122
|
+
- Readiness and the start check accept a `volatile-*` store when key expiry is `swept`
|
|
123
|
+
(`eviction_warning(policy, mode)`): in strict mode such a process is now ready.
|
|
124
|
+
`allkeys-*` still fails readiness in either mode, and `volatile-*` with `native` still does.
|
|
125
|
+
`llmq_store_eviction_unsafe` follows the same rule.
|
|
126
|
+
- Docs: the key table in `docs/explanation/storage.md` was out of date and is corrected: the
|
|
127
|
+
limiter, breaker, leases and in-progress keys have sliding TTLs (1 day; `max(budget_key_ms,
|
|
128
|
+
2 x lease)`), rotation slides 1 day, waiter keys slide 180 s (a waiter counts for 2 s), the
|
|
129
|
+
`health` hash has a 1-day TTL only after a latency sample, the breaker's `fail_at`, `thr_n`,
|
|
130
|
+
`thr_until` fields, the tenant fairness, receive claim, offload, job and reset keys, the
|
|
131
|
+
caller TTL (45 s by default, not 30), lock lifetimes (10 s, 5 / 35 s, 45 s), stream and
|
|
132
|
+
dead-letter trimming, the audit's `MAXLEN`, what `PUBLISH` deletes, the result's TTL after
|
|
133
|
+
its first read (cut to 2 s, not deleted), the request record's lifetimes, what the queue
|
|
134
|
+
store holds, and readiness in strict mode. New section "Key expiry modes"; the store
|
|
135
|
+
requirements now read `noeviction` **or** `volatile-*` with `swept`.
|
|
20
136
|
|
|
21
137
|
## [0.1.0b2] - 2026-11-15
|
|
22
138
|
|
|
@@ -1005,6 +1121,7 @@ First beta, for test runs (`pip install --pre llm-queue`).
|
|
|
1005
1121
|
- **Docs site** (Zensical on GitHub Pages): tutorial, how-to guides, explanation with
|
|
1006
1122
|
diagrams, an API reference from the docstrings.
|
|
1007
1123
|
|
|
1008
|
-
[Unreleased]: https://github.com/techiewonk/llm-queue/compare/v0.1.
|
|
1124
|
+
[Unreleased]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b3...HEAD
|
|
1125
|
+
[0.1.0b3]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b2...v0.1.0b3
|
|
1009
1126
|
[0.1.0b2]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b1...v0.1.0b2
|
|
1010
1127
|
[0.1.0b1]: https://github.com/techiewonk/llm-queue/releases/tag/v0.1.0b1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: llm-queue
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.0b3
|
|
4
4
|
Summary: LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status.
|
|
5
5
|
Project-URL: Homepage, https://github.com/techiewonk/llm-queue
|
|
6
6
|
Project-URL: Documentation, https://techiewonk.github.io/llm-queue/
|
|
@@ -94,6 +94,13 @@ with Amazon Bedrock and OpenAI-compatible servers (such as vLLM), and with LangC
|
|
|
94
94
|
optional token streaming (`queue_streaming=True`).
|
|
95
95
|
- **Observability.** CloudWatch EMF or Prometheus metrics (`GET /metrics`), alerts, a status
|
|
96
96
|
API, health and readiness routes, a CLI, and a documented list of every log event.
|
|
97
|
+
- **Runs on a shared cache store.** With `LLMQ_KEY_EXPIRY=auto` (the default) llm-queue keeps
|
|
98
|
+
no TTL on the keys it needs when the store's policy is `volatile-*` (ElastiCache's default
|
|
99
|
+
`volatile-lru`), so it can share one Valkey with your app's cache and none of its keys is
|
|
100
|
+
evicted. See [Run on a shared cache store](https://techiewonk.github.io/llm-queue/how-to/shared-cache-store/).
|
|
101
|
+
- **Purge as a service.** Delete the store state of removed models, deployments and tenants,
|
|
102
|
+
with dry runs and an audit, from Python, the CLI or HTTP, and automatically after a registry
|
|
103
|
+
change. See [Purge store state](https://techiewonk.github.io/llm-queue/how-to/purge/).
|
|
97
104
|
- **A live registry.** Accounts, models and every setting live in one JSON document, validated
|
|
98
105
|
with every problem listed at once, published to the store, edited live with an audit trail
|
|
99
106
|
and rolled back; a JSON Schema is printed by `llm-queue registry schema`.
|
|
@@ -194,10 +201,10 @@ one). It is a beta: see [STABILITY](https://github.com/techiewonk/llm-queue/blob
|
|
|
194
201
|
|
|
195
202
|
Full documentation: **https://techiewonk.github.io/llm-queue/**
|
|
196
203
|
|
|
197
|
-
- [What's new
|
|
198
|
-
[Upgrade
|
|
204
|
+
- [What's new](https://techiewonk.github.io/llm-queue/whats-new/) and
|
|
205
|
+
[Upgrade](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
|
|
199
206
|
- [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
|
|
200
|
-
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, monitor, share models across services, secure, test
|
|
207
|
+
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, purge, run on a shared cache store, monitor, share models across services, secure, test, upgrade
|
|
201
208
|
- [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, tenant fairness, failures and recovery
|
|
202
209
|
- [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): every setting with its default and range, the registry document and its JSON Schema, environment variables, error kinds, log events, CLI and HTTP, the Python API
|
|
203
210
|
|
|
@@ -39,6 +39,13 @@ with Amazon Bedrock and OpenAI-compatible servers (such as vLLM), and with LangC
|
|
|
39
39
|
optional token streaming (`queue_streaming=True`).
|
|
40
40
|
- **Observability.** CloudWatch EMF or Prometheus metrics (`GET /metrics`), alerts, a status
|
|
41
41
|
API, health and readiness routes, a CLI, and a documented list of every log event.
|
|
42
|
+
- **Runs on a shared cache store.** With `LLMQ_KEY_EXPIRY=auto` (the default) llm-queue keeps
|
|
43
|
+
no TTL on the keys it needs when the store's policy is `volatile-*` (ElastiCache's default
|
|
44
|
+
`volatile-lru`), so it can share one Valkey with your app's cache and none of its keys is
|
|
45
|
+
evicted. See [Run on a shared cache store](https://techiewonk.github.io/llm-queue/how-to/shared-cache-store/).
|
|
46
|
+
- **Purge as a service.** Delete the store state of removed models, deployments and tenants,
|
|
47
|
+
with dry runs and an audit, from Python, the CLI or HTTP, and automatically after a registry
|
|
48
|
+
change. See [Purge store state](https://techiewonk.github.io/llm-queue/how-to/purge/).
|
|
42
49
|
- **A live registry.** Accounts, models and every setting live in one JSON document, validated
|
|
43
50
|
with every problem listed at once, published to the store, edited live with an audit trail
|
|
44
51
|
and rolled back; a JSON Schema is printed by `llm-queue registry schema`.
|
|
@@ -139,10 +146,10 @@ one). It is a beta: see [STABILITY](https://github.com/techiewonk/llm-queue/blob
|
|
|
139
146
|
|
|
140
147
|
Full documentation: **https://techiewonk.github.io/llm-queue/**
|
|
141
148
|
|
|
142
|
-
- [What's new
|
|
143
|
-
[Upgrade
|
|
149
|
+
- [What's new](https://techiewonk.github.io/llm-queue/whats-new/) and
|
|
150
|
+
[Upgrade](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
|
|
144
151
|
- [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
|
|
145
|
-
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, monitor, share models across services, secure, test
|
|
152
|
+
- [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, purge, run on a shared cache store, monitor, share models across services, secure, test, upgrade
|
|
146
153
|
- [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, tenant fairness, failures and recovery
|
|
147
154
|
- [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): every setting with its default and range, the registry document and its JSON Schema, environment variables, error kinds, log events, CLI and HTTP, the Python API
|
|
148
155
|
|
|
@@ -9,14 +9,16 @@
|
|
|
9
9
|
- **0.x releases:** a minor release may break the public API; every break is in CHANGELOG.md.
|
|
10
10
|
- **Message format:** queue messages carry a format version `v`. A worker returns a message newer
|
|
11
11
|
than it understands to the queue instead of dropping it. Workers of two releases share a queue
|
|
12
|
-
only where CHANGELOG.md says so: 0.1.0b1 and 0.1.0b2 do not (drain 0.1.0b1 first;
|
|
13
|
-
`
|
|
12
|
+
only where CHANGELOG.md says so: 0.1.0b1 and 0.1.0b2 do not (drain 0.1.0b1 first);
|
|
13
|
+
0.1.0b2 and 0.1.0b3 do on a `noeviction` store (on a `volatile-*` store, upgrade every pod).
|
|
14
|
+
See `docs/how-to/upgrade.md`.
|
|
14
15
|
- **Store schema:** the key layout has a schema number; a change to it ships with a migration note.
|
|
15
16
|
- **Supported versions:** Python 3.12-3.14 (unit and integration tests run on each in CI; 3.11 was dropped in 0.1.0b2); Valkey 8
|
|
16
17
|
and Redis 7.2+ as one primary with replicas (tested on Valkey 8.1 and Redis 7.4). Cluster mode
|
|
17
18
|
is not supported (one admission touches keys of several hash tags) and `start()` refuses it;
|
|
18
|
-
`maxmemory-policy noeviction` is recommended
|
|
19
|
-
|
|
19
|
+
`maxmemory-policy noeviction` is recommended; a `volatile-*` policy works with key expiry
|
|
20
|
+
`swept` (the default `LLMQ_KEY_EXPIRY=auto` picks it). Any other policy logs a warning at
|
|
21
|
+
start, and in strict mode readiness fails. The client is
|
|
20
22
|
redis-py 6 or later (`redis>=6,<9`). The store must allow Lua scripts, pub/sub and blocking
|
|
21
23
|
list reads (`BLPOP`, the queue doorbell); ElastiCache for Valkey / Redis OSS does.
|
|
22
24
|
- **Dependencies:** ranges with upper bounds in `pyproject.toml` (`aiobotocore<4`, `redis<9`,
|
|
@@ -29,6 +29,7 @@ from llm_queue.errors import (
|
|
|
29
29
|
ConfigError,
|
|
30
30
|
LLMQueueError,
|
|
31
31
|
MissingExtra,
|
|
32
|
+
PurgeRefused,
|
|
32
33
|
StoreUnavailable,
|
|
33
34
|
UnknownModel,
|
|
34
35
|
)
|
|
@@ -61,6 +62,7 @@ if TYPE_CHECKING:
|
|
|
61
62
|
from llm_queue.engine.jobs import JobContext, JobFn, JobSettings, current_job
|
|
62
63
|
from llm_queue.engine.metrics import EmfSink, Metrics, NullSink, PrometheusSink
|
|
63
64
|
from llm_queue.engine.offload import ResultOffload, S3Offload
|
|
65
|
+
from llm_queue.engine.purge import PurgeReport, PurgeService
|
|
64
66
|
from llm_queue.engine.store_client import Secrets
|
|
65
67
|
from llm_queue.engine.tokens import register_token_estimator
|
|
66
68
|
from llm_queue.providers.embeddings import EmbeddingCodec, register_embedding_family
|
|
@@ -113,6 +115,9 @@ _LAZY = {
|
|
|
113
115
|
"current_job": "llm_queue.engine.jobs",
|
|
114
116
|
"Secrets": "llm_queue.engine.store_client",
|
|
115
117
|
"PublishResult": "llm_queue.registry.store",
|
|
118
|
+
# Purges of removed models / deployments and tenants (0.1.0b3): `LLMQueue.purge`.
|
|
119
|
+
"PurgeService": "llm_queue.engine.purge",
|
|
120
|
+
"PurgeReport": "llm_queue.engine.purge",
|
|
116
121
|
}
|
|
117
122
|
|
|
118
123
|
__all__ = [
|
|
@@ -145,6 +150,9 @@ __all__ = [
|
|
|
145
150
|
"PrometheusSink",
|
|
146
151
|
"Provider",
|
|
147
152
|
"PublishResult",
|
|
153
|
+
"PurgeRefused",
|
|
154
|
+
"PurgeReport",
|
|
155
|
+
"PurgeService",
|
|
148
156
|
"RateStatus",
|
|
149
157
|
"RegistryConflict",
|
|
150
158
|
"RegistryError",
|
|
@@ -43,6 +43,7 @@ from llm_queue.engine.errors import CallFailed
|
|
|
43
43
|
from llm_queue.engine.jobs import JobFn, JobRunner, JobSettings
|
|
44
44
|
from llm_queue.engine.metrics import Metrics, sink_for
|
|
45
45
|
from llm_queue.engine.offload import ResultOffload
|
|
46
|
+
from llm_queue.engine.purge import PurgeService, PurgeSettings
|
|
46
47
|
from llm_queue.engine.queue import QueueFull
|
|
47
48
|
from llm_queue.engine.runtime import Engine, EngineSettings
|
|
48
49
|
from llm_queue.engine.settings import InfraSettings
|
|
@@ -185,6 +186,7 @@ class Health(TypedDict):
|
|
|
185
186
|
stores: dict[str, str | None]
|
|
186
187
|
eviction_policy: str
|
|
187
188
|
queue_eviction_policy: NotRequired[str]
|
|
189
|
+
key_expiry: str # `native` or `swept` (LLMQ_KEY_EXPIRY, engine/expiry.py)
|
|
188
190
|
|
|
189
191
|
|
|
190
192
|
@dataclass(frozen=True, slots=True)
|
|
@@ -366,9 +368,13 @@ class LLMQueue:
|
|
|
366
368
|
queue_wait_replicas=settings.queue_wait_replicas,
|
|
367
369
|
queue_wait_ms=settings.queue_wait_ms,
|
|
368
370
|
queue_wait_scopes=settings.queue_wait_scopes,
|
|
371
|
+
key_expiry=settings.key_expiry,
|
|
372
|
+
ttl_sweep_batch=settings.ttl_sweep_batch,
|
|
373
|
+
ttl_sweep_rounds=settings.ttl_sweep_rounds,
|
|
369
374
|
strict=settings.strict,
|
|
370
375
|
debug_capture_allowed=settings.debug_capture_from_config,
|
|
371
376
|
timings=tuning.engine,
|
|
377
|
+
purge=PurgeSettings.from_infra(settings),
|
|
372
378
|
)
|
|
373
379
|
return Engine(
|
|
374
380
|
state,
|
|
@@ -893,6 +899,14 @@ class LLMQueue:
|
|
|
893
899
|
"""The registry store (live edits, audit, rollback)."""
|
|
894
900
|
return self._engine.registry
|
|
895
901
|
|
|
902
|
+
@property
|
|
903
|
+
def purge(self) -> PurgeService:
|
|
904
|
+
"""Purges of removed models, deployments and tenants (dry runs, audited, auto purge).
|
|
905
|
+
|
|
906
|
+
`await llmq.purge.tenant("acme", actor=user)`; see docs/how-to/purge.md.
|
|
907
|
+
"""
|
|
908
|
+
return self._engine.purge
|
|
909
|
+
|
|
896
910
|
def http_router(
|
|
897
911
|
self,
|
|
898
912
|
*,
|
|
@@ -37,7 +37,7 @@ KEY_MS = num(T.budget_key_ms, 180000) -- budget and limiter keys live this long
|
|
|
37
37
|
-- Every key a script writes has an expiry, sliding on each write (FL-7): a removed model's or
|
|
38
38
|
-- deployment's breaker, limiter, health, rotation, leases and waiters stayed in the store for good.
|
|
39
39
|
local DAY_MS = 86400000
|
|
40
|
-
local function slide(key, ms)
|
|
40
|
+
local function slide(key, ms) X.expire(key, ms) end -- native TTL or index (expiry.py)
|
|
41
41
|
local function held_ms() return math.max(KEY_MS, 2 * num(req.lease_ms, 60000)) end
|
|
42
42
|
|
|
43
43
|
-- Budget hashes read once per script into a table (FL-11: each budget read the same fields with
|
|
@@ -98,7 +98,7 @@ local function write(key, f, mode, value)
|
|
|
98
98
|
else
|
|
99
99
|
redis.call('HSET', key, f .. '_tat', tostring(value))
|
|
100
100
|
end
|
|
101
|
-
|
|
101
|
+
X.expire(key, KEY_MS)
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
-- Give back g units (negative g charges more). Fixed counters only inside the grant's minute.
|
|
@@ -110,12 +110,12 @@ local function give(key, f, L, g, mode, granted_minute)
|
|
|
110
110
|
-- The minute turned: an overrun (g < 0) is charged to the new minute, not dropped.
|
|
111
111
|
if g < 0 then
|
|
112
112
|
redis.call('HSET', key, f .. '_win', minute, f .. '_n', -g)
|
|
113
|
-
|
|
113
|
+
X.expire(key, KEY_MS)
|
|
114
114
|
end
|
|
115
115
|
return
|
|
116
116
|
end
|
|
117
117
|
redis.call('HSET', key, f .. '_n', math.max(0, hnum(key, f .. '_n', 0) - g))
|
|
118
|
-
|
|
118
|
+
X.expire(key, KEY_MS)
|
|
119
119
|
return
|
|
120
120
|
end
|
|
121
121
|
local per = 60000 / L
|
|
@@ -123,7 +123,7 @@ local function give(key, f, L, g, mode, granted_minute)
|
|
|
123
123
|
local value = tat - g * per
|
|
124
124
|
if g > 0 then value = math.max(value, now) end
|
|
125
125
|
redis.call('HSET', key, f .. '_tat', tostring(value))
|
|
126
|
-
|
|
126
|
+
X.expire(key, KEY_MS) -- the key may have expired since the grant
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
-- The budgets of a deployment row for class cls: {key, field, limit, cost kind}.
|
|
@@ -749,7 +749,7 @@ if not chosen then
|
|
|
749
749
|
slide(wait_keys[cls], held_ms())
|
|
750
750
|
slide(info_key, held_ms())
|
|
751
751
|
redis.call('ZADD', exp_key, exp, cls .. '|' .. rid)
|
|
752
|
-
|
|
752
|
+
X.expire(exp_key, KEY_MS)
|
|
753
753
|
local reply = {r = 'wait', ms = math.max(1, math.floor(best_wait or num(T.wait_ms, 100))),
|
|
754
754
|
reason = best_reason or 'slot'}
|
|
755
755
|
if tf_st then tf_refused(alias, tf_st, rid, exp, reply) end
|
|
@@ -763,7 +763,7 @@ for _, s in ipairs(res.states) do write(s[1], s[2], s[3], s[4]) end
|
|
|
763
763
|
local rkey = mk(alias, 'rate:' .. chosen.key)
|
|
764
764
|
if hnum(rkey, 'seen_win', -1) ~= minute then
|
|
765
765
|
redis.call('HSET', rkey, 'seen_win', minute, 'seen_calls', 0, 'seen_tokens', 0, 'seen_n', 0)
|
|
766
|
-
|
|
766
|
+
X.expire(rkey, KEY_MS)
|
|
767
767
|
end
|
|
768
768
|
redis.call('HINCRBY', rkey, 'seen_calls', cost_r)
|
|
769
769
|
redis.call('HINCRBY', rkey, 'seen_tokens', cost_t)
|
|
@@ -778,7 +778,7 @@ redis.call('HSET', entry_key, 'owner', req.owner, 'worker', req.worker, 'dep', c
|
|
|
778
778
|
'cls', cls, 'minute', minute, 'cost_r', cost_r, 'cost_t', cost_t, 'account', row.account,
|
|
779
779
|
'started', now, 'request', rid, 'served', alias, 'mode', row.rate_window or 'smooth',
|
|
780
780
|
'bud', cjson.encode(res.taken), 'job_owner', job_owner)
|
|
781
|
-
|
|
781
|
+
X.expire(entry_key, lease_ms * 2)
|
|
782
782
|
if res.probe then
|
|
783
783
|
local cb = mk(alias, 'cb:' .. chosen.key)
|
|
784
784
|
redis.call('HSET', cb, 'probe', req.owner, 'probe_at', now)
|
|
@@ -789,7 +789,7 @@ end
|
|
|
789
789
|
if req.claim or redis.call('EXISTS', job_key) == 1 then
|
|
790
790
|
redis.call('HSET', job_key, 'state', 'running', 'owner', job_owner, 'hb', now)
|
|
791
791
|
-- A claim made here (the caller's record was gone) had no TTL and stayed for good.
|
|
792
|
-
if
|
|
792
|
+
if X.pttl(job_key) < 0 then X.expire(job_key, num(T.job_ttl_ms, 3600000)) end
|
|
793
793
|
end
|
|
794
794
|
redis.call('ZREM', wait_keys[cls], rid)
|
|
795
795
|
redis.call('HDEL', info_key, rid)
|
|
@@ -804,10 +804,10 @@ end
|
|
|
804
804
|
if present(req.count_first) then
|
|
805
805
|
local retries = mk(alias, 'retries:' .. req.count_first)
|
|
806
806
|
redis.call('HINCRBY', retries, 'first', 1)
|
|
807
|
-
|
|
807
|
+
X.expire(retries, 180000)
|
|
808
808
|
end
|
|
809
809
|
if sticky_key then
|
|
810
|
-
|
|
810
|
+
X.set(sticky_key, chosen.key, num(req.sticky.ttl_ms, 1800000))
|
|
811
811
|
end
|
|
812
812
|
if req.strategy == 'round_robin' then
|
|
813
813
|
local rot = mk(alias, 'rot')
|
|
@@ -883,7 +883,7 @@ local function settle(alias, owner, rid, refund_r, refund_t, learn, outcome, lat
|
|
|
883
883
|
if outcome == 'throttle' then
|
|
884
884
|
local thr = mk(alias, 'thr:' .. dep .. ':' .. minute)
|
|
885
885
|
redis.call('INCR', thr)
|
|
886
|
-
|
|
886
|
+
X.expire(thr, KEY_MS)
|
|
887
887
|
if hnum(ac, 'cut_at', 0) < now - num(T.limiter_cut_ms, 1000) then
|
|
888
888
|
redis.call('HSET', ac, 'limit', tostring(math.max(1, math.floor(limit * num(T.limiter_cut_factor, 0.7)))), 'cut_at', now)
|
|
889
889
|
end
|
|
@@ -1014,8 +1014,8 @@ local function beat(rid, jo)
|
|
|
1014
1014
|
local job = P .. ':{r:' .. rid .. '}:job'
|
|
1015
1015
|
if not jo or redis.call('HGET', job, 'owner') ~= jo then return end
|
|
1016
1016
|
redis.call('HSET', job, 'hb', now)
|
|
1017
|
-
local ttl =
|
|
1018
|
-
if ttl >= 0 and ttl < lease_ms * 2 then
|
|
1017
|
+
local ttl = X.pttl(job)
|
|
1018
|
+
if ttl >= 0 and ttl < lease_ms * 2 then X.expire(job, num(T.job_ttl_ms, 3600000)) end
|
|
1019
1019
|
end
|
|
1020
1020
|
for _, item in ipairs(req.jobs or {}) do beat(item.request_id, item.owner) end
|
|
1021
1021
|
for _, item in ipairs(req.items) do
|
|
@@ -1030,7 +1030,7 @@ for _, item in ipairs(req.items) do
|
|
|
1030
1030
|
redis.call('ZADD', mk(item.alias, 'inprog'), 'XX', until_, item.owner)
|
|
1031
1031
|
slide(mk(item.alias, 'lease:' .. dep .. ':' .. cls), held_ms())
|
|
1032
1032
|
slide(mk(item.alias, 'inprog'), held_ms())
|
|
1033
|
-
|
|
1033
|
+
X.expire(entry_key, num(req.lease_ms, 60000) * 2)
|
|
1034
1034
|
-- A tenant's slot (fairness share) lives as long as the grant's lease.
|
|
1035
1035
|
local tf_slot = redis.call('HGET', entry_key, 'tf_slot')
|
|
1036
1036
|
if tf_slot then
|
|
@@ -1046,7 +1046,7 @@ for _, item in ipairs(req.items) do
|
|
|
1046
1046
|
end
|
|
1047
1047
|
-- Folded in (FL-21), both optional: the process's caller key (a SET of its own before) and the
|
|
1048
1048
|
-- store's epoch (a GET of its own), so a beat needs fewer round trips.
|
|
1049
|
-
if present(req.caller) then
|
|
1049
|
+
if present(req.caller) then X.set(req.caller.key, '1', req.caller.ttl * 1000) end
|
|
1050
1050
|
return cjson.encode({n = renewed, lost = lost, epoch = redis.call('GET', P .. ':epoch') or ''})
|
|
1051
1051
|
"""
|
|
1052
1052
|
)
|
|
@@ -1097,10 +1097,10 @@ for _, item in ipairs(req.items) do
|
|
|
1097
1097
|
local state = redis.call('HGET', job, 'state')
|
|
1098
1098
|
if not state or state == 'queued' or state == 'running' then
|
|
1099
1099
|
redis.call('HSET', job, 'state', 'running', 'owner', jo, 'hb', now)
|
|
1100
|
-
if
|
|
1100
|
+
if X.pttl(job) < 0 then X.expire(job, num(T.job_ttl_ms, 3600000)) end
|
|
1101
1101
|
end
|
|
1102
1102
|
end
|
|
1103
|
-
|
|
1103
|
+
X.expire(entry_key, num(req.lease_ms, 60000) * 2)
|
|
1104
1104
|
done = done + 1
|
|
1105
1105
|
end
|
|
1106
1106
|
end
|
|
@@ -21,6 +21,7 @@ from redis.exceptions import TimeoutError as StoreTimeoutError
|
|
|
21
21
|
|
|
22
22
|
from llm_queue._logging import get_logger
|
|
23
23
|
from llm_queue.engine import _admission_lua as lua
|
|
24
|
+
from llm_queue.engine.expiry import prelude
|
|
24
25
|
from llm_queue.engine.fairness import Fairness
|
|
25
26
|
from llm_queue.engine.keys import Keys
|
|
26
27
|
from llm_queue.engine.metrics import Metrics, NullSink
|
|
@@ -165,13 +166,14 @@ class Admission:
|
|
|
165
166
|
"slow_stale_ms": pace.slow_stale_ms,
|
|
166
167
|
"job_ttl_ms": pace.job_ttl_ms,
|
|
167
168
|
}
|
|
168
|
-
|
|
169
|
-
self.
|
|
170
|
-
self.
|
|
171
|
-
self.
|
|
172
|
-
self.
|
|
173
|
-
self.
|
|
174
|
-
self.
|
|
169
|
+
pre = prelude(keys.prefix) # X.expire / X.pttl / X.set (engine/expiry.py)
|
|
170
|
+
self._pick = client.register_script(pre + lua.PICK_TAKE)
|
|
171
|
+
self._finish = client.register_script(pre + lua.FINISH)
|
|
172
|
+
self._undo = client.register_script(pre + lua.UNDO)
|
|
173
|
+
self._renew = client.register_script(pre + lua.RENEW)
|
|
174
|
+
self._sweep = client.register_script(pre + lua.SWEEP)
|
|
175
|
+
self._reregister = client.register_script(pre + lua.REREGISTER)
|
|
176
|
+
self._release = client.register_script(pre + lua.RELEASE)
|
|
175
177
|
|
|
176
178
|
async def _call(self, script: Any, body: dict[str, Any]) -> Any:
|
|
177
179
|
args: list[Any] = [self.keys.prefix, orjson.dumps({**body, "t": self._t})]
|