llm-queue 0.1.0b2__tar.gz → 0.1.0b3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/.gitignore +1 -0
  2. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/CHANGELOG.md +118 -1
  3. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/PKG-INFO +11 -4
  4. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/README.md +10 -3
  5. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/STABILITY.md +6 -4
  6. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/__init__.py +8 -0
  7. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/_version.py +1 -1
  8. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/core.py +14 -0
  9. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/_admission_lua.py +18 -18
  10. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/admission.py +9 -7
  11. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/cli.py +73 -58
  12. llm_queue-0.1.0b3/src/llm_queue/engine/expiry.py +207 -0
  13. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/health.py +5 -3
  14. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/http.py +160 -2
  15. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/jobs.py +10 -5
  16. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/metrics.py +9 -0
  17. llm_queue-0.1.0b3/src/llm_queue/engine/purge.py +906 -0
  18. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/queue.py +24 -22
  19. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/requests.py +37 -34
  20. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/runtime.py +162 -10
  21. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/settings.py +60 -0
  22. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/status.py +6 -2
  23. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/store_checks.py +24 -9
  24. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/streaming.py +13 -5
  25. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/errors.py +8 -0
  26. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/LICENSE +0 -0
  27. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/NOTICE +0 -0
  28. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/SECURITY.md +0 -0
  29. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/pyproject.toml +0 -0
  30. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/_logging.py +0 -0
  31. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/__init__.py +0 -0
  32. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/bedrock_http.py +0 -0
  33. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/hub.py +0 -0
  34. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/pool.py +0 -0
  35. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/providers.py +0 -0
  36. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/clients/timing.py +0 -0
  37. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/context.py +0 -0
  38. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/__init__.py +0 -0
  39. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/_codec.py +0 -0
  40. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/cross_checks.py +0 -0
  41. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/embed.py +0 -0
  42. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/envelope.py +0 -0
  43. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/errors.py +0 -0
  44. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/fairness.py +0 -0
  45. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/hooks.py +0 -0
  46. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/inference.py +0 -0
  47. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/keys.py +0 -0
  48. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/offload.py +0 -0
  49. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/shapes.py +0 -0
  50. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/store_client.py +0 -0
  51. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/store_guard.py +0 -0
  52. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/engine/tokens.py +0 -0
  53. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/http_transport.py +0 -0
  54. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/ids.py +0 -0
  55. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/__init__.py +0 -0
  56. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/_client.py +0 -0
  57. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/_stream.py +0 -0
  58. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/bedrock.py +0 -0
  59. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/langchain/openai.py +0 -0
  60. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/providers/__init__.py +0 -0
  61. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/providers/classify.py +0 -0
  62. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/providers/embeddings.py +0 -0
  63. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/py.typed +0 -0
  64. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/__init__.py +0 -0
  65. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/_lua.py +0 -0
  66. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/json_schema.py +0 -0
  67. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/resolve.py +0 -0
  68. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/schema.py +0 -0
  69. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/settings.py +0 -0
  70. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/store.py +0 -0
  71. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/registry/validate.py +0 -0
  72. {llm_queue-0.1.0b2 → llm_queue-0.1.0b3}/src/llm_queue/tuning.py +0 -0
@@ -43,3 +43,4 @@ docs/internal/
43
43
 
44
44
  # Claude Code agent worktrees
45
45
  .claude/
46
+ .planning/
@@ -7,6 +7,56 @@ public API (see STABILITY.md).
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.1.0b3] - 2026-11-16
11
+
12
+ Key expiry modes, so llm-queue can share a `volatile-*` cache store (ElastiCache's default
13
+ `volatile-lru`), and a purge service for removed models, deployments and tenants.
14
+
15
+ ### Upgrading from 0.1.0b2
16
+
17
+ The message format and the key layout are unchanged, so 0.1.0b2 and 0.1.0b3 pods can share a
18
+ queue on a `noeviction` store. [Upgrade](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
19
+ has the same list with more detail.
20
+
21
+ 1. **Upgrade every pod.** 0.1.0b2 pods do not read the store's key expiry mode (`P:expiry`)
22
+ and always write native TTLs. On a `noeviction` store `auto` picks `native`, the same as
23
+ 0.1.0b2, so a mixed fleet is fine there during a rolling upgrade. On a `volatile-*` store a
24
+ 0.1.0b2 pod is not ready in strict mode anyway, and its keys stay evictable: finish the
25
+ rollout before you rely on the store being shared.
26
+ 2. **Switching modes loses nothing.** In `swept` mode setting an expiry also clears the key's
27
+ native TTL (`PERSIST`); in `native` mode it also removes the key's `P:ttl` member; reading
28
+ the time left counts either. Keys move to the new mode on their next write; a key never
29
+ written again keeps its old expiry and still goes then.
30
+ 3. **Rolling back to 0.1.0b2 after running `swept`.** 0.1.0b2 has no expiry sweep, so a key
31
+ whose expiry is only in `P:ttl` would never expire under it. Before rolling back, set
32
+ `LLMQ_KEY_EXPIRY=native` on the 0.1.0b3 pods (the store must be `noeviction` for 0.1.0b2
33
+ to be ready in strict mode) and keep them running for at least the longest key lifetime (1
34
+ day by default; longer if you raised a retention or TTL setting). In `native` mode each write
35
+ moves its key back to a native TTL, and the index sweep keeps running and deletes the
36
+ indexed keys as they come due. Roll back when `ZCARD <prefix>:ttl` is about 0 on each store
37
+ (a separate queue store has its own). Leftover `P:expiry`, `P:ttl` and `P:purge` keys are
38
+ harmless to 0.1.0b2.
39
+ 4. **`llm-queue registry gc` output and scope.** It prints the purge report: `deleted_keys` and
40
+ `dry_run` instead of `would_delete`. It also purges removed deployments of live models, and
41
+ keeps a removed model's queue that still holds messages unless `--force`. Update scripts that
42
+ parse its output.
43
+ 5. **Auto purge is on by default.** When a publish drops a model or deployment, its store state
44
+ is purged after `LLMQ_PURGE_GRACE_SECS` (900). Set `LLMQ_AUTO_PURGE=false` to keep the
45
+ 0.1.0b2 behaviour (state stays until you purge it by hand).
46
+
47
+ ### Changed (behaviour)
48
+
49
+ - **Auto purge on by default** (`LLMQ_AUTO_PURGE=true`): a model or deployment dropped from
50
+ the registry has its store state deleted after the grace (900 s) unless it comes back.
51
+ 0.1.0b2 kept that state until `registry gc`. `LLMQ_AUTO_PURGE=false` restores that.
52
+ - **`auto` key expiry picks `swept` on a `volatile-*` store** (and on `allkeys-*` or an
53
+ unreadable policy): llm-queue's keys get no TTL and the sweep deletes them, up to 10 s late.
54
+ 0.1.0b2 always set native TTLs. On `noeviction` nothing changes. `LLMQ_KEY_EXPIRY=native`
55
+ restores the 0.1.0b2 behaviour (and leaves a `volatile-*` store not ready in strict mode).
56
+ - **`registry gc`** also purges removed deployments of live models, keeps non-empty queues
57
+ unless `--force`, and prints the purge report (`deleted_keys`, `dry_run`) instead of
58
+ `would_delete`.
59
+
10
60
  ### Added
11
61
 
12
62
  - Docs: a reference page listing every log event the library writes, with its level and
@@ -17,6 +67,72 @@ public API (see STABILITY.md).
17
67
  reference page on the document's shape with a complete example (`docs/reference/registry.md`).
18
68
  - `LLMQueue.http_router(metrics_route=True)` serves `GET /metrics` (off by default); the option
19
69
  existed only on `status_router(engine, ...)`, which needs the now private engine.
70
+ - Purge controls (`docs/how-to/purge.md`): `PurgeService` (`llmq.purge`, or
71
+ `PurgeService.from_env()` standalone) deletes the store state of a removed model
72
+ (`model`), a removed deployment (`deployment`, waiting for its running calls first), a tenant
73
+ (`tenant`, fairness keys with the weight sum corrected; optionally cancelling its queued or
74
+ running calls) or everything the registry no longer has (`removed`), with dry runs
75
+ (`plan_*`). Each returns a `PurgeReport`; real purges are audited (`purge_<kind>` in the
76
+ registry audit), logged `llmq_purge` and counted `llmq_purged_keys_total{kind}`. New error
77
+ `PurgeRefused` (a model or deployment still in the registry without `force`; a tenant purge of
78
+ calls without the tenant index).
79
+ - Auto purge: when a publish drops a model or deployment, it is purged after
80
+ `LLMQ_PURGE_GRACE_SECS` (900) unless it came back meanwhile (`LLMQ_AUTO_PURGE`, default on);
81
+ `purge.scheduled()` / `cancel_scheduled()` list and keep pending ones.
82
+ - Auto purge runs only in worker processes. A process's first tick after start only records
83
+ the registry it sees (a baseline), so a start never schedules a purge. Before it purges, it
84
+ reads the stored registry strictly and skips the purge on any read error or refusal. An auto
85
+ deployment purge does not wait for running calls: the grace (900 s) is as long as the longest
86
+ call the registry allows (`timeout_max_secs`, 900 s by default).
87
+ - CLI `llm-queue purge model|deployment|tenant --yes|--dry-run`; `--no-wait` on `purge
88
+ deployment` and `registry gc`. HTTP routes `POST /purge/model`, `/purge/deployment`,
89
+ `/purge/tenant`, `/registry/gc`, `GET /purge/scheduled`, `POST /purge/scheduled/cancel` (only
90
+ with `allow_writes=True`, `X-LLMQ-Admin`; `dry_run` defaults to true). An HTTP purge waits for
91
+ running calls at most `LLMQ_HTTP_PURGE_WAIT_SECS` (60, 0 to 3600); a larger or omitted
92
+ `wait_secs` is capped to it.
93
+ - Settings `LLMQ_AUTO_PURGE`, `LLMQ_PURGE_GRACE_SECS`, `LLMQ_PURGE_WAIT`,
94
+ `LLMQ_PURGE_WAIT_SECS`, `LLMQ_HTTP_PURGE_WAIT_SECS`, `LLMQ_PURGE_BATCH`, `LLMQ_TENANT_PURGE_DEFAULT` and
95
+ `LLMQ_TENANT_INDEX` (off: the per-tenant call index costs one write per call).
96
+ - Key expiry modes (`LLMQ_KEY_EXPIRY=auto|native|swept`, default `auto`): llm-queue can share a
97
+ store whose `maxmemory-policy` is `volatile-*` (ElastiCache's default `volatile-lru`, an app
98
+ cache). In `swept` no key the library needs has a TTL: each expiry is a member of the zset
99
+ `P:ttl` and the sweeper deletes due keys (up to `sweep_secs`, 10 s, late), so a `volatile-*`
100
+ policy cannot evict them. `auto` picks `native` on `noeviction` and `swept` otherwise. The mode
101
+ is the store's (`P:expiry`; missing means `native`), written at start, every minute and after
102
+ a store reset. New settings `LLMQ_TTL_SWEEP_BATCH` (500) and `LLMQ_TTL_SWEEP_ROUNDS` (100);
103
+ `health()["key_expiry"]`; gauge `llmq_key_expiry_swept{store}`; counter
104
+ `llmq_ttl_swept_total`; log events `key_expiry_changed`, `key_expiry_publish_failed`
105
+ (warning), `ttl_sweep_failed`. Until a process has published its mode it reports the mode its
106
+ setting asks for (`native` for `auto`). How-to: `docs/how-to/shared-cache-store.md`.
107
+ - Switching key expiry modes loses nothing: in `swept` mode setting an expiry also clears the
108
+ key's native TTL (`PERSIST`), in `native` mode it also removes the key's `P:ttl` member, and
109
+ reading the time left counts either; keys move to the new mode on their next write.
110
+ - The expiry sweep has its own lock, `P:lock:ttlsweep` (a native TTL on purpose), and runs in
111
+ every process, workers and callers, every `sweep_secs` (10 s): up to 500 x 100 = 50000 keys
112
+ per 10 s by default. The worker sweep keeps `P:lock:sweep`.
113
+ - `scripts/evict_drill.py`: an eviction drill that fills a throwaway store under queue traffic
114
+ with `volatile-lru` and `noeviction` (see CONTRIBUTING.md).
115
+ - Docs: a reference page listing every key, stream and channel the library keeps in the store,
116
+ with its fields, writers, readers, lifetime in each key expiry mode, what is lost if it is
117
+ evicted under each policy, and code references (`docs/reference/store-keys.md`, with a JSON
118
+ copy `store-keys.json`).
119
+
120
+ ### Changed
121
+
122
+ - Readiness and the start check accept a `volatile-*` store when key expiry is `swept`
123
+ (`eviction_warning(policy, mode)`): in strict mode such a process is now ready.
124
+ `allkeys-*` still fails readiness in either mode, and `volatile-*` with `native` still does.
125
+ `llmq_store_eviction_unsafe` follows the same rule.
126
+ - Docs: the key table in `docs/explanation/storage.md` was out of date and is corrected: the
127
+ limiter, breaker, leases and in-progress keys have sliding TTLs (1 day; `max(budget_key_ms,
128
+ 2 x lease)`), rotation slides 1 day, waiter keys slide 180 s (a waiter counts for 2 s), the
129
+ `health` hash has a 1-day TTL only after a latency sample, the breaker's `fail_at`, `thr_n`,
130
+ `thr_until` fields, the tenant fairness, receive claim, offload, job and reset keys, the
131
+ caller TTL (45 s by default, not 30), lock lifetimes (10 s, 5 / 35 s, 45 s), stream and
132
+ dead-letter trimming, the audit's `MAXLEN`, what `PUBLISH` deletes, the result's TTL after
133
+ its first read (cut to 2 s, not deleted), the request record's lifetimes, what the queue
134
+ store holds, and readiness in strict mode. New section "Key expiry modes"; the store
135
+ requirements now read `noeviction` **or** `volatile-*` with `swept`.
20
136
 
21
137
  ## [0.1.0b2] - 2026-11-15
22
138
 
@@ -1005,6 +1121,7 @@ First beta, for test runs (`pip install --pre llm-queue`).
1005
1121
  - **Docs site** (Zensical on GitHub Pages): tutorial, how-to guides, explanation with
1006
1122
  diagrams, an API reference from the docstrings.
1007
1123
 
1008
- [Unreleased]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b2...HEAD
1124
+ [Unreleased]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b3...HEAD
1125
+ [0.1.0b3]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b2...v0.1.0b3
1009
1126
  [0.1.0b2]: https://github.com/techiewonk/llm-queue/compare/v0.1.0b1...v0.1.0b2
1010
1127
  [0.1.0b1]: https://github.com/techiewonk/llm-queue/releases/tag/v0.1.0b1
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: llm-queue
3
- Version: 0.1.0b2
3
+ Version: 0.1.0b3
4
4
  Summary: LLM calls through a queue on Valkey / Redis: RPM and TPM budgets per account and region, balancing, fallback, health and status.
5
5
  Project-URL: Homepage, https://github.com/techiewonk/llm-queue
6
6
  Project-URL: Documentation, https://techiewonk.github.io/llm-queue/
@@ -94,6 +94,13 @@ with Amazon Bedrock and OpenAI-compatible servers (such as vLLM), and with LangC
94
94
  optional token streaming (`queue_streaming=True`).
95
95
  - **Observability.** CloudWatch EMF or Prometheus metrics (`GET /metrics`), alerts, a status
96
96
  API, health and readiness routes, a CLI, and a documented list of every log event.
97
+ - **Runs on a shared cache store.** With `LLMQ_KEY_EXPIRY=auto` (the default) llm-queue keeps
98
+ no TTL on the keys it needs when the store's policy is `volatile-*` (ElastiCache's default
99
+ `volatile-lru`), so it can share one Valkey with your app's cache and none of its keys is
100
+ evicted. See [Run on a shared cache store](https://techiewonk.github.io/llm-queue/how-to/shared-cache-store/).
101
+ - **Purge as a service.** Delete the store state of removed models, deployments and tenants,
102
+ with dry runs and an audit, from Python, the CLI or HTTP, and automatically after a registry
103
+ change. See [Purge store state](https://techiewonk.github.io/llm-queue/how-to/purge/).
97
104
  - **A live registry.** Accounts, models and every setting live in one JSON document, validated
98
105
  with every problem listed at once, published to the store, edited live with an audit trail
99
106
  and rolled back; a JSON Schema is printed by `llm-queue registry schema`.
@@ -194,10 +201,10 @@ one). It is a beta: see [STABILITY](https://github.com/techiewonk/llm-queue/blob
194
201
 
195
202
  Full documentation: **https://techiewonk.github.io/llm-queue/**
196
203
 
197
- - [What's new in 0.1.0b2](https://techiewonk.github.io/llm-queue/whats-new/) and
198
- [Upgrade from 0.1.0b1](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
204
+ - [What's new](https://techiewonk.github.io/llm-queue/whats-new/) and
205
+ [Upgrade](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
199
206
  - [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
200
- - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, monitor, share models across services, secure, test
207
+ - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, purge, run on a shared cache store, monitor, share models across services, secure, test, upgrade
201
208
  - [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, tenant fairness, failures and recovery
202
209
  - [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): every setting with its default and range, the registry document and its JSON Schema, environment variables, error kinds, log events, CLI and HTTP, the Python API
203
210
 
@@ -39,6 +39,13 @@ with Amazon Bedrock and OpenAI-compatible servers (such as vLLM), and with LangC
39
39
  optional token streaming (`queue_streaming=True`).
40
40
  - **Observability.** CloudWatch EMF or Prometheus metrics (`GET /metrics`), alerts, a status
41
41
  API, health and readiness routes, a CLI, and a documented list of every log event.
42
+ - **Runs on a shared cache store.** With `LLMQ_KEY_EXPIRY=auto` (the default) llm-queue keeps
43
+ no TTL on the keys it needs when the store's policy is `volatile-*` (ElastiCache's default
44
+ `volatile-lru`), so it can share one Valkey with your app's cache and none of its keys is
45
+ evicted. See [Run on a shared cache store](https://techiewonk.github.io/llm-queue/how-to/shared-cache-store/).
46
+ - **Purge as a service.** Delete the store state of removed models, deployments and tenants,
47
+ with dry runs and an audit, from Python, the CLI or HTTP, and automatically after a registry
48
+ change. See [Purge store state](https://techiewonk.github.io/llm-queue/how-to/purge/).
42
49
  - **A live registry.** Accounts, models and every setting live in one JSON document, validated
43
50
  with every problem listed at once, published to the store, edited live with an audit trail
44
51
  and rolled back; a JSON Schema is printed by `llm-queue registry schema`.
@@ -139,10 +146,10 @@ one). It is a beta: see [STABILITY](https://github.com/techiewonk/llm-queue/blob
139
146
 
140
147
  Full documentation: **https://techiewonk.github.io/llm-queue/**
141
148
 
142
- - [What's new in 0.1.0b2](https://techiewonk.github.io/llm-queue/whats-new/) and
143
- [Upgrade from 0.1.0b1](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
149
+ - [What's new](https://techiewonk.github.io/llm-queue/whats-new/) and
150
+ [Upgrade](https://techiewonk.github.io/llm-queue/how-to/upgrade/)
144
151
  - [Getting started](https://techiewonk.github.io/llm-queue/tutorials/getting-started/): install, local stack, a registry, the first call
145
- - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, monitor, share models across services, secure, test
152
+ - [How-to guides](https://techiewonk.github.io/llm-queue/how-to/configure/): configure, Bedrock IAM, tune, call, jobs, large results, operate, purge, run on a shared cache store, monitor, share models across services, secure, test, upgrade
146
153
  - [Explanation](https://techiewonk.github.io/llm-queue/explanation/architecture/): architecture, the life of a call, admission, tenant fairness, failures and recovery
147
154
  - [Reference](https://techiewonk.github.io/llm-queue/reference/configuration/): every setting with its default and range, the registry document and its JSON Schema, environment variables, error kinds, log events, CLI and HTTP, the Python API
148
155
 
@@ -9,14 +9,16 @@
9
9
  - **0.x releases:** a minor release may break the public API; every break is in CHANGELOG.md.
10
10
  - **Message format:** queue messages carry a format version `v`. A worker returns a message newer
11
11
  than it understands to the queue instead of dropping it. Workers of two releases share a queue
12
- only where CHANGELOG.md says so: 0.1.0b1 and 0.1.0b2 do not (drain 0.1.0b1 first; see
13
- `docs/how-to/upgrade.md`).
12
+ only where CHANGELOG.md says so: 0.1.0b1 and 0.1.0b2 do not (drain 0.1.0b1 first);
13
+ 0.1.0b2 and 0.1.0b3 do on a `noeviction` store (on a `volatile-*` store, upgrade every pod).
14
+ See `docs/how-to/upgrade.md`.
14
15
  - **Store schema:** the key layout has a schema number; a change to it ships with a migration note.
15
16
  - **Supported versions:** Python 3.12-3.14 (unit and integration tests run on each in CI; 3.11 was dropped in 0.1.0b2); Valkey 8
16
17
  and Redis 7.2+ as one primary with replicas (tested on Valkey 8.1 and Redis 7.4). Cluster mode
17
18
  is not supported (one admission touches keys of several hash tags) and `start()` refuses it;
18
- `maxmemory-policy noeviction` is recommended (a warning at start otherwise, and in strict mode
19
- readiness fails). The client is
19
+ `maxmemory-policy noeviction` is recommended; a `volatile-*` policy works with key expiry
20
+ `swept` (the default `LLMQ_KEY_EXPIRY=auto` picks it). Any other policy logs a warning at
21
+ start, and in strict mode readiness fails. The client is
20
22
  redis-py 6 or later (`redis>=6,<9`). The store must allow Lua scripts, pub/sub and blocking
21
23
  list reads (`BLPOP`, the queue doorbell); ElastiCache for Valkey / Redis OSS does.
22
24
  - **Dependencies:** ranges with upper bounds in `pyproject.toml` (`aiobotocore<4`, `redis<9`,
@@ -29,6 +29,7 @@ from llm_queue.errors import (
29
29
  ConfigError,
30
30
  LLMQueueError,
31
31
  MissingExtra,
32
+ PurgeRefused,
32
33
  StoreUnavailable,
33
34
  UnknownModel,
34
35
  )
@@ -61,6 +62,7 @@ if TYPE_CHECKING:
61
62
  from llm_queue.engine.jobs import JobContext, JobFn, JobSettings, current_job
62
63
  from llm_queue.engine.metrics import EmfSink, Metrics, NullSink, PrometheusSink
63
64
  from llm_queue.engine.offload import ResultOffload, S3Offload
65
+ from llm_queue.engine.purge import PurgeReport, PurgeService
64
66
  from llm_queue.engine.store_client import Secrets
65
67
  from llm_queue.engine.tokens import register_token_estimator
66
68
  from llm_queue.providers.embeddings import EmbeddingCodec, register_embedding_family
@@ -113,6 +115,9 @@ _LAZY = {
113
115
  "current_job": "llm_queue.engine.jobs",
114
116
  "Secrets": "llm_queue.engine.store_client",
115
117
  "PublishResult": "llm_queue.registry.store",
118
+ # Purges of removed models / deployments and tenants (0.1.0b3): `LLMQueue.purge`.
119
+ "PurgeService": "llm_queue.engine.purge",
120
+ "PurgeReport": "llm_queue.engine.purge",
116
121
  }
117
122
 
118
123
  __all__ = [
@@ -145,6 +150,9 @@ __all__ = [
145
150
  "PrometheusSink",
146
151
  "Provider",
147
152
  "PublishResult",
153
+ "PurgeRefused",
154
+ "PurgeReport",
155
+ "PurgeService",
148
156
  "RateStatus",
149
157
  "RegistryConflict",
150
158
  "RegistryError",
@@ -2,4 +2,4 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- __version__ = "0.1.0b2"
5
+ __version__ = "0.1.0b3"
@@ -43,6 +43,7 @@ from llm_queue.engine.errors import CallFailed
43
43
  from llm_queue.engine.jobs import JobFn, JobRunner, JobSettings
44
44
  from llm_queue.engine.metrics import Metrics, sink_for
45
45
  from llm_queue.engine.offload import ResultOffload
46
+ from llm_queue.engine.purge import PurgeService, PurgeSettings
46
47
  from llm_queue.engine.queue import QueueFull
47
48
  from llm_queue.engine.runtime import Engine, EngineSettings
48
49
  from llm_queue.engine.settings import InfraSettings
@@ -185,6 +186,7 @@ class Health(TypedDict):
185
186
  stores: dict[str, str | None]
186
187
  eviction_policy: str
187
188
  queue_eviction_policy: NotRequired[str]
189
+ key_expiry: str # `native` or `swept` (LLMQ_KEY_EXPIRY, engine/expiry.py)
188
190
 
189
191
 
190
192
  @dataclass(frozen=True, slots=True)
@@ -366,9 +368,13 @@ class LLMQueue:
366
368
  queue_wait_replicas=settings.queue_wait_replicas,
367
369
  queue_wait_ms=settings.queue_wait_ms,
368
370
  queue_wait_scopes=settings.queue_wait_scopes,
371
+ key_expiry=settings.key_expiry,
372
+ ttl_sweep_batch=settings.ttl_sweep_batch,
373
+ ttl_sweep_rounds=settings.ttl_sweep_rounds,
369
374
  strict=settings.strict,
370
375
  debug_capture_allowed=settings.debug_capture_from_config,
371
376
  timings=tuning.engine,
377
+ purge=PurgeSettings.from_infra(settings),
372
378
  )
373
379
  return Engine(
374
380
  state,
@@ -893,6 +899,14 @@ class LLMQueue:
893
899
  """The registry store (live edits, audit, rollback)."""
894
900
  return self._engine.registry
895
901
 
902
+ @property
903
+ def purge(self) -> PurgeService:
904
+ """Purges of removed models, deployments and tenants (dry runs, audited, auto purge).
905
+
906
+ `await llmq.purge.tenant("acme", actor=user)`; see docs/how-to/purge.md.
907
+ """
908
+ return self._engine.purge
909
+
896
910
  def http_router(
897
911
  self,
898
912
  *,
@@ -37,7 +37,7 @@ KEY_MS = num(T.budget_key_ms, 180000) -- budget and limiter keys live this long
37
37
  -- Every key a script writes has an expiry, sliding on each write (FL-7): a removed model's or
38
38
  -- deployment's breaker, limiter, health, rotation, leases and waiters stayed in the store for good.
39
39
  local DAY_MS = 86400000
40
- local function slide(key, ms) redis.call('PEXPIRE', key, ms) end
40
+ local function slide(key, ms) X.expire(key, ms) end -- native TTL or index (expiry.py)
41
41
  local function held_ms() return math.max(KEY_MS, 2 * num(req.lease_ms, 60000)) end
42
42
 
43
43
  -- Budget hashes read once per script into a table (FL-11: each budget read the same fields with
@@ -98,7 +98,7 @@ local function write(key, f, mode, value)
98
98
  else
99
99
  redis.call('HSET', key, f .. '_tat', tostring(value))
100
100
  end
101
- redis.call('PEXPIRE', key, KEY_MS)
101
+ X.expire(key, KEY_MS)
102
102
  end
103
103
 
104
104
  -- Give back g units (negative g charges more). Fixed counters only inside the grant's minute.
@@ -110,12 +110,12 @@ local function give(key, f, L, g, mode, granted_minute)
110
110
  -- The minute turned: an overrun (g < 0) is charged to the new minute, not dropped.
111
111
  if g < 0 then
112
112
  redis.call('HSET', key, f .. '_win', minute, f .. '_n', -g)
113
- redis.call('PEXPIRE', key, KEY_MS)
113
+ X.expire(key, KEY_MS)
114
114
  end
115
115
  return
116
116
  end
117
117
  redis.call('HSET', key, f .. '_n', math.max(0, hnum(key, f .. '_n', 0) - g))
118
- redis.call('PEXPIRE', key, KEY_MS)
118
+ X.expire(key, KEY_MS)
119
119
  return
120
120
  end
121
121
  local per = 60000 / L
@@ -123,7 +123,7 @@ local function give(key, f, L, g, mode, granted_minute)
123
123
  local value = tat - g * per
124
124
  if g > 0 then value = math.max(value, now) end
125
125
  redis.call('HSET', key, f .. '_tat', tostring(value))
126
- redis.call('PEXPIRE', key, KEY_MS) -- the key may have expired since the grant
126
+ X.expire(key, KEY_MS) -- the key may have expired since the grant
127
127
  end
128
128
 
129
129
  -- The budgets of a deployment row for class cls: {key, field, limit, cost kind}.
@@ -749,7 +749,7 @@ if not chosen then
749
749
  slide(wait_keys[cls], held_ms())
750
750
  slide(info_key, held_ms())
751
751
  redis.call('ZADD', exp_key, exp, cls .. '|' .. rid)
752
- redis.call('PEXPIRE', exp_key, KEY_MS)
752
+ X.expire(exp_key, KEY_MS)
753
753
  local reply = {r = 'wait', ms = math.max(1, math.floor(best_wait or num(T.wait_ms, 100))),
754
754
  reason = best_reason or 'slot'}
755
755
  if tf_st then tf_refused(alias, tf_st, rid, exp, reply) end
@@ -763,7 +763,7 @@ for _, s in ipairs(res.states) do write(s[1], s[2], s[3], s[4]) end
763
763
  local rkey = mk(alias, 'rate:' .. chosen.key)
764
764
  if hnum(rkey, 'seen_win', -1) ~= minute then
765
765
  redis.call('HSET', rkey, 'seen_win', minute, 'seen_calls', 0, 'seen_tokens', 0, 'seen_n', 0)
766
- redis.call('PEXPIRE', rkey, KEY_MS)
766
+ X.expire(rkey, KEY_MS)
767
767
  end
768
768
  redis.call('HINCRBY', rkey, 'seen_calls', cost_r)
769
769
  redis.call('HINCRBY', rkey, 'seen_tokens', cost_t)
@@ -778,7 +778,7 @@ redis.call('HSET', entry_key, 'owner', req.owner, 'worker', req.worker, 'dep', c
778
778
  'cls', cls, 'minute', minute, 'cost_r', cost_r, 'cost_t', cost_t, 'account', row.account,
779
779
  'started', now, 'request', rid, 'served', alias, 'mode', row.rate_window or 'smooth',
780
780
  'bud', cjson.encode(res.taken), 'job_owner', job_owner)
781
- redis.call('PEXPIRE', entry_key, lease_ms * 2)
781
+ X.expire(entry_key, lease_ms * 2)
782
782
  if res.probe then
783
783
  local cb = mk(alias, 'cb:' .. chosen.key)
784
784
  redis.call('HSET', cb, 'probe', req.owner, 'probe_at', now)
@@ -789,7 +789,7 @@ end
789
789
  if req.claim or redis.call('EXISTS', job_key) == 1 then
790
790
  redis.call('HSET', job_key, 'state', 'running', 'owner', job_owner, 'hb', now)
791
791
  -- A claim made here (the caller's record was gone) had no TTL and stayed for good.
792
- if redis.call('PTTL', job_key) < 0 then redis.call('PEXPIRE', job_key, num(T.job_ttl_ms, 3600000)) end
792
+ if X.pttl(job_key) < 0 then X.expire(job_key, num(T.job_ttl_ms, 3600000)) end
793
793
  end
794
794
  redis.call('ZREM', wait_keys[cls], rid)
795
795
  redis.call('HDEL', info_key, rid)
@@ -804,10 +804,10 @@ end
804
804
  if present(req.count_first) then
805
805
  local retries = mk(alias, 'retries:' .. req.count_first)
806
806
  redis.call('HINCRBY', retries, 'first', 1)
807
- redis.call('EXPIRE', retries, 180)
807
+ X.expire(retries, 180000)
808
808
  end
809
809
  if sticky_key then
810
- redis.call('SET', sticky_key, chosen.key, 'PX', num(req.sticky.ttl_ms, 1800000))
810
+ X.set(sticky_key, chosen.key, num(req.sticky.ttl_ms, 1800000))
811
811
  end
812
812
  if req.strategy == 'round_robin' then
813
813
  local rot = mk(alias, 'rot')
@@ -883,7 +883,7 @@ local function settle(alias, owner, rid, refund_r, refund_t, learn, outcome, lat
883
883
  if outcome == 'throttle' then
884
884
  local thr = mk(alias, 'thr:' .. dep .. ':' .. minute)
885
885
  redis.call('INCR', thr)
886
- redis.call('PEXPIRE', thr, KEY_MS)
886
+ X.expire(thr, KEY_MS)
887
887
  if hnum(ac, 'cut_at', 0) < now - num(T.limiter_cut_ms, 1000) then
888
888
  redis.call('HSET', ac, 'limit', tostring(math.max(1, math.floor(limit * num(T.limiter_cut_factor, 0.7)))), 'cut_at', now)
889
889
  end
@@ -1014,8 +1014,8 @@ local function beat(rid, jo)
1014
1014
  local job = P .. ':{r:' .. rid .. '}:job'
1015
1015
  if not jo or redis.call('HGET', job, 'owner') ~= jo then return end
1016
1016
  redis.call('HSET', job, 'hb', now)
1017
- local ttl = redis.call('PTTL', job)
1018
- if ttl >= 0 and ttl < lease_ms * 2 then redis.call('PEXPIRE', job, num(T.job_ttl_ms, 3600000)) end
1017
+ local ttl = X.pttl(job)
1018
+ if ttl >= 0 and ttl < lease_ms * 2 then X.expire(job, num(T.job_ttl_ms, 3600000)) end
1019
1019
  end
1020
1020
  for _, item in ipairs(req.jobs or {}) do beat(item.request_id, item.owner) end
1021
1021
  for _, item in ipairs(req.items) do
@@ -1030,7 +1030,7 @@ for _, item in ipairs(req.items) do
1030
1030
  redis.call('ZADD', mk(item.alias, 'inprog'), 'XX', until_, item.owner)
1031
1031
  slide(mk(item.alias, 'lease:' .. dep .. ':' .. cls), held_ms())
1032
1032
  slide(mk(item.alias, 'inprog'), held_ms())
1033
- redis.call('PEXPIRE', entry_key, num(req.lease_ms, 60000) * 2)
1033
+ X.expire(entry_key, num(req.lease_ms, 60000) * 2)
1034
1034
  -- A tenant's slot (fairness share) lives as long as the grant's lease.
1035
1035
  local tf_slot = redis.call('HGET', entry_key, 'tf_slot')
1036
1036
  if tf_slot then
@@ -1046,7 +1046,7 @@ for _, item in ipairs(req.items) do
1046
1046
  end
1047
1047
  -- Folded in (FL-21), both optional: the process's caller key (a SET of its own before) and the
1048
1048
  -- store's epoch (a GET of its own), so a beat needs fewer round trips.
1049
- if present(req.caller) then redis.call('SET', req.caller.key, '1', 'EX', req.caller.ttl) end
1049
+ if present(req.caller) then X.set(req.caller.key, '1', req.caller.ttl * 1000) end
1050
1050
  return cjson.encode({n = renewed, lost = lost, epoch = redis.call('GET', P .. ':epoch') or ''})
1051
1051
  """
1052
1052
  )
@@ -1097,10 +1097,10 @@ for _, item in ipairs(req.items) do
1097
1097
  local state = redis.call('HGET', job, 'state')
1098
1098
  if not state or state == 'queued' or state == 'running' then
1099
1099
  redis.call('HSET', job, 'state', 'running', 'owner', jo, 'hb', now)
1100
- if redis.call('PTTL', job) < 0 then redis.call('PEXPIRE', job, num(T.job_ttl_ms, 3600000)) end
1100
+ if X.pttl(job) < 0 then X.expire(job, num(T.job_ttl_ms, 3600000)) end
1101
1101
  end
1102
1102
  end
1103
- redis.call('PEXPIRE', entry_key, num(req.lease_ms, 60000) * 2)
1103
+ X.expire(entry_key, num(req.lease_ms, 60000) * 2)
1104
1104
  done = done + 1
1105
1105
  end
1106
1106
  end
@@ -21,6 +21,7 @@ from redis.exceptions import TimeoutError as StoreTimeoutError
21
21
 
22
22
  from llm_queue._logging import get_logger
23
23
  from llm_queue.engine import _admission_lua as lua
24
+ from llm_queue.engine.expiry import prelude
24
25
  from llm_queue.engine.fairness import Fairness
25
26
  from llm_queue.engine.keys import Keys
26
27
  from llm_queue.engine.metrics import Metrics, NullSink
@@ -165,13 +166,14 @@ class Admission:
165
166
  "slow_stale_ms": pace.slow_stale_ms,
166
167
  "job_ttl_ms": pace.job_ttl_ms,
167
168
  }
168
- self._pick = client.register_script(lua.PICK_TAKE)
169
- self._finish = client.register_script(lua.FINISH)
170
- self._undo = client.register_script(lua.UNDO)
171
- self._renew = client.register_script(lua.RENEW)
172
- self._sweep = client.register_script(lua.SWEEP)
173
- self._reregister = client.register_script(lua.REREGISTER)
174
- self._release = client.register_script(lua.RELEASE)
169
+ pre = prelude(keys.prefix) # X.expire / X.pttl / X.set (engine/expiry.py)
170
+ self._pick = client.register_script(pre + lua.PICK_TAKE)
171
+ self._finish = client.register_script(pre + lua.FINISH)
172
+ self._undo = client.register_script(pre + lua.UNDO)
173
+ self._renew = client.register_script(pre + lua.RENEW)
174
+ self._sweep = client.register_script(pre + lua.SWEEP)
175
+ self._reregister = client.register_script(pre + lua.REREGISTER)
176
+ self._release = client.register_script(pre + lua.RELEASE)
175
177
 
176
178
  async def _call(self, script: Any, body: dict[str, Any]) -> Any:
177
179
  args: list[Any] = [self.keys.prefix, orjson.dumps({**body, "t": self._t})]