redis-allocator 0.6.1__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {redis_allocator-0.6.1/redis_allocator.egg-info → redis_allocator-0.7.0}/PKG-INFO +75 -13
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/README.md +72 -10
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/pyproject.toml +3 -2
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator/__init__.py +2 -0
- redis_allocator-0.7.0/redis_allocator/_version.py +1 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator/allocator.py +31 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/__init__.py +30 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/cli.py +52 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/collector.py +300 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/dashboard.py +93 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/invariants.py +192 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/models.py +128 -0
- redis_allocator-0.7.0/redis_allocator/diagnostics/static/dashboard.html +403 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator/task_queue.py +18 -10
- {redis_allocator-0.6.1 → redis_allocator-0.7.0/redis_allocator.egg-info}/PKG-INFO +75 -13
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator.egg-info/SOURCES.txt +13 -0
- redis_allocator-0.7.0/redis_allocator.egg-info/entry_points.txt +2 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator.egg-info/requires.txt +2 -2
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/setup.py +9 -1
- redis_allocator-0.7.0/tests/_perf_real_redis.py +195 -0
- redis_allocator-0.7.0/tests/test_allocator_benchmark.py +194 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_allocator_bughunt.py +38 -0
- redis_allocator-0.7.0/tests/test_diagnostics_cli.py +31 -0
- redis_allocator-0.7.0/tests/test_diagnostics_collector.py +76 -0
- redis_allocator-0.7.0/tests/test_diagnostics_dashboard.py +58 -0
- redis_allocator-0.7.0/tests/test_diagnostics_invariants.py +96 -0
- redis_allocator-0.7.0/tests/test_diagnostics_models.py +37 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_task_queue.py +57 -11
- redis_allocator-0.6.1/redis_allocator/_version.py +0 -1
- redis_allocator-0.6.1/tests/_perf_real_redis.py +0 -115
- redis_allocator-0.6.1/tests/test_allocator_benchmark.py +0 -127
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/LICENSE +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator/lock.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator.egg-info/dependency_links.txt +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/redis_allocator.egg-info/top_level.txt +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/setup.cfg +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/__init__.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/_bughunt_helpers.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/_profile_fakeredis.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/conftest.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/stress_pool_corruption.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_allocator.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_allocator_concurrency.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_allocator_forensics.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_allocator_fuzz.py +0 -0
- {redis_allocator-0.6.1 → redis_allocator-0.7.0}/tests/test_lock.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: redis-allocator
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Redis-based resource allocation system.
|
|
5
5
|
Home-page: https://github.com/invoker-bot/RedisAllocator-python
|
|
6
6
|
Author: Invoker Bot
|
|
@@ -25,7 +25,7 @@ Requires-Dist: fakeredis[lua]>=2.35.1; extra == "test"
|
|
|
25
25
|
Requires-Dist: flake8>=6.1.0; extra == "test"
|
|
26
26
|
Requires-Dist: freezegun>=1.4.0; extra == "test"
|
|
27
27
|
Provides-Extra: docs
|
|
28
|
-
Requires-Dist: sphinx
|
|
28
|
+
Requires-Dist: sphinx<9.0.0,>=7.0.0; extra == "docs"
|
|
29
29
|
Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == "docs"
|
|
30
30
|
Requires-Dist: sphinx-git>=11.0.0; extra == "docs"
|
|
31
31
|
Requires-Dist: sphinxcontrib-mermaid>=0.7.1; extra == "docs"
|
|
@@ -36,7 +36,7 @@ Requires-Dist: pytest-mock>=3.12.0; extra == "dev"
|
|
|
36
36
|
Requires-Dist: fakeredis[lua]>=2.35.1; extra == "dev"
|
|
37
37
|
Requires-Dist: flake8>=6.1.0; extra == "dev"
|
|
38
38
|
Requires-Dist: freezegun>=1.4.0; extra == "dev"
|
|
39
|
-
Requires-Dist: sphinx
|
|
39
|
+
Requires-Dist: sphinx<9.0.0,>=7.0.0; extra == "dev"
|
|
40
40
|
Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == "dev"
|
|
41
41
|
Requires-Dist: sphinx-git>=11.0.0; extra == "dev"
|
|
42
42
|
Requires-Dist: sphinxcontrib-mermaid>=0.7.1; extra == "dev"
|
|
@@ -97,7 +97,9 @@ A few intentional behaviors that may surprise newcomers — knowing them up-fron
|
|
|
97
97
|
|
|
98
98
|
- **`malloc_key(timeout<=0)` creates a permanent (non-expiring) lock.** Useful for resources whose ownership must outlive process restarts; the caller takes responsibility for releasing via `free_keys`. GC never reclaims it (because expiry is also `-1`). Misuse causes resource leaks — only pass `timeout=0` when you mean "I will manage this lock myself."
|
|
99
99
|
- **`cache_timeout=0` (or negative) creates a permanent soft binding.** Sticks until explicitly overwritten by another `update_soft_bind` for the same name, or removed via `unbind_soft_bind`. Symmetric to the lock-timeout contract above.
|
|
100
|
-
- **`
|
|
100
|
+
- **`malloc()` and `malloc_key()` return `None` when no resource is available.** Check the return value before using it; an empty pool is not represented as a wrapper with `key=None`.
|
|
101
|
+
- **Prefer `with allocator.malloc(...) as allocation:` or `allocation.release()` for object allocations.** `release()` closes the wrapped object and returns the key to the pool. Plain `close()` only closes the wrapped object and leaves allocation ownership unchanged for callers that manage `free()` manually.
|
|
102
|
+
- **`assign()` / `shrink()` / `clear()` are not namespace scrubbers.** `assign()` and `shrink()` remove pool membership but intentionally leave lock keys for removed members; `clear()` deletes the pool hash/head/tail only. This avoids write amplification and wildcard scans, but an evicted permanent lock becomes a permanent orphan. To recover a known orphan, call `free_keys(key)`, reintroduce that same key with `extend([key])` or `assign([... key ...])`, or delete `<prefix>|<suffix>:<key>` directly. Reintroducing unrelated keys, `clear()`, and `gc()` do not remove orphan locks outside the pool.
|
|
101
103
|
- **Pool keys are validated at the API boundary.** `extend()` / `assign()` raise `ValueError` for keys that are empty, equal to the literal `"#ALLOCATED"`, contain the `"||"` separator, contain NUL bytes, or are not strings. These shapes break the on-disk encoding and would silently corrupt the linked list.
|
|
102
104
|
|
|
103
105
|
## Documentation
|
|
@@ -227,15 +229,13 @@ if key:
|
|
|
227
229
|
# Allocate a resource with object (returns a RedisAllocatorObject)
|
|
228
230
|
allocated_obj = allocator.malloc(timeout=120)
|
|
229
231
|
if allocated_obj:
|
|
230
|
-
|
|
232
|
+
with allocated_obj:
|
|
231
233
|
# The key is available as a property
|
|
232
234
|
print(f"Allocated resource: {allocated_obj.key}")
|
|
233
235
|
|
|
234
236
|
# Update the resource's lock timeout
|
|
235
237
|
allocated_obj.update(timeout=60)
|
|
236
|
-
|
|
237
|
-
# Free the resource when done
|
|
238
|
-
allocator.free(allocated_obj)
|
|
238
|
+
# Leaving the with block calls allocated_obj.release().
|
|
239
239
|
|
|
240
240
|
# Using soft binding (associates a name with a resource)
|
|
241
241
|
allocator.update_soft_bind("worker-1", "resource-1")
|
|
@@ -461,7 +461,7 @@ result = task_queue.query(
|
|
|
461
461
|
id="task-123",
|
|
462
462
|
name="example-task",
|
|
463
463
|
params={"input": "data"},
|
|
464
|
-
timeout=300, #
|
|
464
|
+
timeout=300, # End-to-end wait timeout in seconds for remote execution
|
|
465
465
|
policy=TaskExecutePolicy.Auto, # Execution policy
|
|
466
466
|
once=False # Whether to delete the result after getting it
|
|
467
467
|
)
|
|
@@ -498,10 +498,40 @@ The RedisAllocator maintains resources in a doubly-linked list structure stored
|
|
|
498
498
|
|
|
499
499
|
### Concurrency Safety
|
|
500
500
|
|
|
501
|
-
Each EVAL is atomic on Redis (single-threaded server, Lua never preempted), so no operation can be observed half-completed. Multi-process coordination is then provided by lock TTL plus the GC pass
|
|
501
|
+
Each EVAL is atomic on Redis (single-threaded server, Lua never preempted), so no operation can be observed half-completed. Multi-process coordination is then provided by lock TTL plus the GC pass. A process killed after allocation can leave a lock behind. If the resource is still present in pool metadata, the lock either expires naturally or is reconciled by GC; if the resource is later evicted by `assign()` / `shrink()` / `clear()`, the lock becomes an orphan outside GC scope and expires only by TTL unless it is permanent.
|
|
502
502
|
|
|
503
503
|
The pool linked-list invariants survive arbitrary interleaving of `malloc` / `free` / `extend` / `assign` / `gc` from concurrent processes (validated by both fakeredis multi-thread tests and a Docker-backed stress driver — see Testing below). If you observe persistent invariant violations under load, please file an issue with the operation sequence.
|
|
504
504
|
|
|
505
|
+
#### Atomic Operation Boundaries
|
|
506
|
+
|
|
507
|
+
Allocator state transitions are written as single Redis Lua scripts. Redis runs each script atomically on the server event loop, so concurrent processes cannot observe a half-updated free-list node, lock key, or soft-binding update. This guarantee depends on using one Redis instance; Redis Cluster is out of scope because the allocator mutates multiple related keys in one script.
|
|
508
|
+
|
|
509
|
+
| Operation | Atomic boundary | Notes |
|
|
510
|
+
|---|---|---|
|
|
511
|
+
| `malloc_key()` | One Lua EVAL | Checks soft binding, pops/rotates the free-list head, marks the item allocated, creates the lock key, and refreshes binding state together. |
|
|
512
|
+
| `free_keys()` | One Lua EVAL | Deletes lock keys and returns still-present pool members to the free-list tail together. Unknown or evicted keys are no-ops. |
|
|
513
|
+
| `extend()` / `shrink()` / `assign()` | One Lua EVAL per call | Batch updates are atomic as a batch. `assign()` and `shrink()` intentionally preserve lock keys for evicted members; `extend()` / `assign()` delete a residual lock only when that same key is newly added to the pool. |
|
|
514
|
+
| `clear()` | One Lua EVAL | Deletes the pool hash plus head/tail pointers only; it does not scan or delete per-resource lock keys. |
|
|
515
|
+
| `gc()` | One Lua EVAL per pass | Incrementally reconciles up to the requested scan count and saves the scan cursor. |
|
|
516
|
+
| `malloc()` / `RedisAllocatorObject.release()` | Wrapper around atomic calls | `malloc()` wraps `malloc_key()` after the atomic allocation; `release()` wraps `free_keys()` after closing the local object. |
|
|
517
|
+
|
|
518
|
+
Multi-process coordination is provided by those atomic scripts plus lock TTLs and GC. A process killed after allocation may leave a lock behind; pool-member locks either expire naturally or are reconciled by GC. Orphan locks whose key is no longer in the pool are outside GC scope. Permanent locks (`timeout <= 0`) are never reclaimed automatically and must be released explicitly with `free_keys(key)`, by reintroducing that same key with `extend([key])` or `assign([... key ...])`, or by deleting `<prefix>|<suffix>:<key>` directly.
|
|
519
|
+
|
|
520
|
+
### Operation Complexity
|
|
521
|
+
|
|
522
|
+
The hot path is designed to avoid scanning the whole pool. Use `malloc_key()` / `free_keys()` for high-frequency allocation, use `extend()` for incremental pool refreshes, and reserve `assign()` for lower-frequency wholesale replacement.
|
|
523
|
+
|
|
524
|
+
| Operation | Time complexity | Space / Redis writes | Recommended use |
|
|
525
|
+
|---|---:|---|---|
|
|
526
|
+
| `malloc_key()` | Amortized `O(1)`; worst-case `O(k)` when skipping `k` expired or locked head entries | Constant extra metadata writes | Hot path allocation. |
|
|
527
|
+
| `free_keys(n)` | `O(n)` | Constant writes per released key | Hot path release; single-key free is `O(1)`. |
|
|
528
|
+
| `extend(n)` | `O(n)`, independent of existing pool size | Constant writes per supplied key | Incremental pool additions or expiry refresh. |
|
|
529
|
+
| `shrink(n)` | `O(n)` | Constant writes per supplied key | Remove known resources. |
|
|
530
|
+
| `assign(n)` | `O(current_pool_size + n)` | Rewrites membership for removed/new keys | Low-frequency full replacement. |
|
|
531
|
+
| `gc(count)` | Approximately `O(count)` per pass | Bounded by scanned entries | Periodic incremental repair/reclaim. |
|
|
532
|
+
|
|
533
|
+
`k` in `malloc_key()` is the number of stale head entries cleaned during that allocation. Those entries are removed or marked allocated as they are skipped, so the cost is paid once and amortizes over later calls.
|
|
534
|
+
|
|
505
535
|
## Testing
|
|
506
536
|
|
|
507
537
|
The default unit suite is fast (~7s) and self-contained — `pytest` runs it without any external services:
|
|
@@ -518,7 +548,7 @@ Several test categories are **opt-in via pytest markers** (excluded from the def
|
|
|
518
548
|
```bash
|
|
519
549
|
pytest -m concurrency # multi-thread fakeredis race tests (~25s)
|
|
520
550
|
pytest -m fuzz # randomised single-thread sequences
|
|
521
|
-
pytest -m benchmark #
|
|
551
|
+
pytest -m benchmark # operation-complexity assertions for allocator commands
|
|
522
552
|
pytest -m "concurrency or fuzz or benchmark" # all of the above
|
|
523
553
|
```
|
|
524
554
|
|
|
@@ -530,9 +560,41 @@ python tests/stress_pool_corruption.py # 60s default; STRESS_DURATION/STR
|
|
|
530
560
|
docker rm -f allocator-stress
|
|
531
561
|
```
|
|
532
562
|
|
|
533
|
-
Performance contracts are validated on both backends.
|
|
563
|
+
Performance contracts are validated on both backends. The fakeredis `pytest -m benchmark` suite enforces scaling ratios across pool sizes from 10 to 5000 keys for `malloc_key() + free_keys()`, batched `free_keys(n)`, `extend(n)`, `shrink(n)`, and `gc(count)`. It covers `assign(n)` across 10 to 1000 keys because fakeredis/lupa is unreliable with 5000+ Lua ARGV on Windows; the real Redis script covers `assign(n)` through 5000. The benchmark asserts operation cost follows the documented complexity table instead of silently regressing to full-pool scans. It relies on a temporary monkey-patch in `tests/conftest.py` because of [fakeredis upstream PR #473](https://github.com/cunla/fakeredis-py/pull/473) (merged to master, not yet released). The patch is idempotent and self-removable once fakeredis ships a release containing the fix — see the cleanup checklist embedded in `tests/conftest.py`.
|
|
564
|
+
|
|
565
|
+
A standalone Python script `tests/_perf_real_redis.py` reproduces the same operation-complexity benchmark against a real Redis container for cross-checking and exits non-zero when a contract fails.
|
|
566
|
+
|
|
567
|
+
## Live Diagnostics
|
|
568
|
+
|
|
569
|
+
`redis-allocator` includes an observer-only diagnostics API and local dashboard for inspecting live allocator state under multi-process load.
|
|
570
|
+
|
|
571
|
+
Python API:
|
|
572
|
+
|
|
573
|
+
```python
|
|
574
|
+
from redis import Redis
|
|
575
|
+
from redis_allocator import RedisAllocatorDiagnostics
|
|
576
|
+
|
|
577
|
+
redis = Redis.from_url("redis://localhost:6379/0", decode_responses=True)
|
|
578
|
+
diagnostics = RedisAllocatorDiagnostics(redis, prefix="myapp", suffix="allocator")
|
|
579
|
+
snapshot = diagnostics.snapshot()
|
|
580
|
+
print(snapshot.to_json(indent=2))
|
|
581
|
+
```
|
|
582
|
+
|
|
583
|
+
One-shot JSON:
|
|
584
|
+
|
|
585
|
+
```bash
|
|
586
|
+
redis-allocator-diagnose --redis-url redis://localhost:6379/0 --prefix myapp --suffix allocator --json
|
|
587
|
+
```
|
|
588
|
+
|
|
589
|
+
Live dashboard:
|
|
590
|
+
|
|
591
|
+
```bash
|
|
592
|
+
redis-allocator-diagnose --redis-url redis://localhost:6379/0 --prefix myapp --suffix allocator --interval 1 --port 8765
|
|
593
|
+
```
|
|
594
|
+
|
|
595
|
+
The diagnostics collector reads pool structure and per-pool-key lock state atomically with one Lua script. Advisory metrics such as Redis memory, commandstats, slowlog, soft bindings, and orphan locks are sampled separately. V1 never mutates allocator state and does not repair corruption. Orphan-lock samples are reporting only: clean a known orphan with `free_keys(key)`, by reintroducing the same key through `extend()` / `assign()`, or with a precise Redis `DEL`; `clear()` and `gc()` do not repair orphan locks outside the pool.
|
|
534
596
|
|
|
535
|
-
|
|
597
|
+
Use `sample_limit` to bound returned examples and `scan_limit` to cap advisory keyspace scans during dashboard refreshes. When a scan is capped, the JSON marks that section as `truncated`.
|
|
536
598
|
|
|
537
599
|
## Roadmap
|
|
538
600
|
|
|
@@ -42,7 +42,9 @@ A few intentional behaviors that may surprise newcomers — knowing them up-fron
|
|
|
42
42
|
|
|
43
43
|
- **`malloc_key(timeout<=0)` creates a permanent (non-expiring) lock.** Useful for resources whose ownership must outlive process restarts; the caller takes responsibility for releasing via `free_keys`. GC never reclaims it (because expiry is also `-1`). Misuse causes resource leaks — only pass `timeout=0` when you mean "I will manage this lock myself."
|
|
44
44
|
- **`cache_timeout=0` (or negative) creates a permanent soft binding.** Sticks until explicitly overwritten by another `update_soft_bind` for the same name, or removed via `unbind_soft_bind`. Symmetric to the lock-timeout contract above.
|
|
45
|
-
- **`
|
|
45
|
+
- **`malloc()` and `malloc_key()` return `None` when no resource is available.** Check the return value before using it; an empty pool is not represented as a wrapper with `key=None`.
|
|
46
|
+
- **Prefer `with allocator.malloc(...) as allocation:` or `allocation.release()` for object allocations.** `release()` closes the wrapped object and returns the key to the pool. Plain `close()` only closes the wrapped object and leaves allocation ownership unchanged for callers that manage `free()` manually.
|
|
47
|
+
- **`assign()` / `shrink()` / `clear()` are not namespace scrubbers.** `assign()` and `shrink()` remove pool membership but intentionally leave lock keys for removed members; `clear()` deletes the pool hash/head/tail only. This avoids write amplification and wildcard scans, but an evicted permanent lock becomes a permanent orphan. To recover a known orphan, call `free_keys(key)`, reintroduce that same key with `extend([key])` or `assign([... key ...])`, or delete `<prefix>|<suffix>:<key>` directly. Reintroducing unrelated keys, `clear()`, and `gc()` do not remove orphan locks outside the pool.
|
|
46
48
|
- **Pool keys are validated at the API boundary.** `extend()` / `assign()` raise `ValueError` for keys that are empty, equal to the literal `"#ALLOCATED"`, contain the `"||"` separator, contain NUL bytes, or are not strings. These shapes break the on-disk encoding and would silently corrupt the linked list.
|
|
47
49
|
|
|
48
50
|
## Documentation
|
|
@@ -172,15 +174,13 @@ if key:
|
|
|
172
174
|
# Allocate a resource with object (returns a RedisAllocatorObject)
|
|
173
175
|
allocated_obj = allocator.malloc(timeout=120)
|
|
174
176
|
if allocated_obj:
|
|
175
|
-
|
|
177
|
+
with allocated_obj:
|
|
176
178
|
# The key is available as a property
|
|
177
179
|
print(f"Allocated resource: {allocated_obj.key}")
|
|
178
180
|
|
|
179
181
|
# Update the resource's lock timeout
|
|
180
182
|
allocated_obj.update(timeout=60)
|
|
181
|
-
|
|
182
|
-
# Free the resource when done
|
|
183
|
-
allocator.free(allocated_obj)
|
|
183
|
+
# Leaving the with block calls allocated_obj.release().
|
|
184
184
|
|
|
185
185
|
# Using soft binding (associates a name with a resource)
|
|
186
186
|
allocator.update_soft_bind("worker-1", "resource-1")
|
|
@@ -406,7 +406,7 @@ result = task_queue.query(
|
|
|
406
406
|
id="task-123",
|
|
407
407
|
name="example-task",
|
|
408
408
|
params={"input": "data"},
|
|
409
|
-
timeout=300, #
|
|
409
|
+
timeout=300, # End-to-end wait timeout in seconds for remote execution
|
|
410
410
|
policy=TaskExecutePolicy.Auto, # Execution policy
|
|
411
411
|
once=False # Whether to delete the result after getting it
|
|
412
412
|
)
|
|
@@ -443,10 +443,40 @@ The RedisAllocator maintains resources in a doubly-linked list structure stored
|
|
|
443
443
|
|
|
444
444
|
### Concurrency Safety
|
|
445
445
|
|
|
446
|
-
Each EVAL is atomic on Redis (single-threaded server, Lua never preempted), so no operation can be observed half-completed. Multi-process coordination is then provided by lock TTL plus the GC pass
|
|
446
|
+
Each EVAL is atomic on Redis (single-threaded server, Lua never preempted), so no operation can be observed half-completed. Multi-process coordination is then provided by lock TTL plus the GC pass. A process killed after allocation can leave a lock behind. If the resource is still present in pool metadata, the lock either expires naturally or is reconciled by GC; if the resource is later evicted by `assign()` / `shrink()` / `clear()`, the lock becomes an orphan outside GC scope and expires only by TTL unless it is permanent.
|
|
447
447
|
|
|
448
448
|
The pool linked-list invariants survive arbitrary interleaving of `malloc` / `free` / `extend` / `assign` / `gc` from concurrent processes (validated by both fakeredis multi-thread tests and a Docker-backed stress driver — see Testing below). If you observe persistent invariant violations under load, please file an issue with the operation sequence.
|
|
449
449
|
|
|
450
|
+
#### Atomic Operation Boundaries
|
|
451
|
+
|
|
452
|
+
Allocator state transitions are written as single Redis Lua scripts. Redis runs each script atomically on the server event loop, so concurrent processes cannot observe a half-updated free-list node, lock key, or soft-binding update. This guarantee depends on using one Redis instance; Redis Cluster is out of scope because the allocator mutates multiple related keys in one script.
|
|
453
|
+
|
|
454
|
+
| Operation | Atomic boundary | Notes |
|
|
455
|
+
|---|---|---|
|
|
456
|
+
| `malloc_key()` | One Lua EVAL | Checks soft binding, pops/rotates the free-list head, marks the item allocated, creates the lock key, and refreshes binding state together. |
|
|
457
|
+
| `free_keys()` | One Lua EVAL | Deletes lock keys and returns still-present pool members to the free-list tail together. Unknown or evicted keys are no-ops. |
|
|
458
|
+
| `extend()` / `shrink()` / `assign()` | One Lua EVAL per call | Batch updates are atomic as a batch. `assign()` and `shrink()` intentionally preserve lock keys for evicted members; `extend()` / `assign()` delete a residual lock only when that same key is newly added to the pool. |
|
|
459
|
+
| `clear()` | One Lua EVAL | Deletes the pool hash plus head/tail pointers only; it does not scan or delete per-resource lock keys. |
|
|
460
|
+
| `gc()` | One Lua EVAL per pass | Incrementally reconciles up to the requested scan count and saves the scan cursor. |
|
|
461
|
+
| `malloc()` / `RedisAllocatorObject.release()` | Wrapper around atomic calls | `malloc()` wraps `malloc_key()` after the atomic allocation; `release()` wraps `free_keys()` after closing the local object. |
|
|
462
|
+
|
|
463
|
+
Multi-process coordination is provided by those atomic scripts plus lock TTLs and GC. A process killed after allocation may leave a lock behind; pool-member locks either expire naturally or are reconciled by GC. Orphan locks whose key is no longer in the pool are outside GC scope. Permanent locks (`timeout <= 0`) are never reclaimed automatically and must be released explicitly with `free_keys(key)`, by reintroducing that same key with `extend([key])` or `assign([... key ...])`, or by deleting `<prefix>|<suffix>:<key>` directly.
|
|
464
|
+
|
|
465
|
+
### Operation Complexity
|
|
466
|
+
|
|
467
|
+
The hot path is designed to avoid scanning the whole pool. Use `malloc_key()` / `free_keys()` for high-frequency allocation, use `extend()` for incremental pool refreshes, and reserve `assign()` for lower-frequency wholesale replacement.
|
|
468
|
+
|
|
469
|
+
| Operation | Time complexity | Space / Redis writes | Recommended use |
|
|
470
|
+
|---|---:|---|---|
|
|
471
|
+
| `malloc_key()` | Amortized `O(1)`; worst-case `O(k)` when skipping `k` expired or locked head entries | Constant extra metadata writes | Hot path allocation. |
|
|
472
|
+
| `free_keys(n)` | `O(n)` | Constant writes per released key | Hot path release; single-key free is `O(1)`. |
|
|
473
|
+
| `extend(n)` | `O(n)`, independent of existing pool size | Constant writes per supplied key | Incremental pool additions or expiry refresh. |
|
|
474
|
+
| `shrink(n)` | `O(n)` | Constant writes per supplied key | Remove known resources. |
|
|
475
|
+
| `assign(n)` | `O(current_pool_size + n)` | Rewrites membership for removed/new keys | Low-frequency full replacement. |
|
|
476
|
+
| `gc(count)` | Approximately `O(count)` per pass | Bounded by scanned entries | Periodic incremental repair/reclaim. |
|
|
477
|
+
|
|
478
|
+
`k` in `malloc_key()` is the number of stale head entries cleaned during that allocation. Those entries are removed or marked allocated as they are skipped, so the cost is paid once and amortizes over later calls.
|
|
479
|
+
|
|
450
480
|
## Testing
|
|
451
481
|
|
|
452
482
|
The default unit suite is fast (~7s) and self-contained — `pytest` runs it without any external services:
|
|
@@ -463,7 +493,7 @@ Several test categories are **opt-in via pytest markers** (excluded from the def
|
|
|
463
493
|
```bash
|
|
464
494
|
pytest -m concurrency # multi-thread fakeredis race tests (~25s)
|
|
465
495
|
pytest -m fuzz # randomised single-thread sequences
|
|
466
|
-
pytest -m benchmark #
|
|
496
|
+
pytest -m benchmark # operation-complexity assertions for allocator commands
|
|
467
497
|
pytest -m "concurrency or fuzz or benchmark" # all of the above
|
|
468
498
|
```
|
|
469
499
|
|
|
@@ -475,9 +505,41 @@ python tests/stress_pool_corruption.py # 60s default; STRESS_DURATION/STR
|
|
|
475
505
|
docker rm -f allocator-stress
|
|
476
506
|
```
|
|
477
507
|
|
|
478
|
-
Performance contracts are validated on both backends.
|
|
508
|
+
Performance contracts are validated on both backends. The fakeredis `pytest -m benchmark` suite enforces scaling ratios across pool sizes from 10 to 5000 keys for `malloc_key() + free_keys()`, batched `free_keys(n)`, `extend(n)`, `shrink(n)`, and `gc(count)`. It covers `assign(n)` across 10 to 1000 keys because fakeredis/lupa is unreliable with 5000+ Lua ARGV on Windows; the real Redis script covers `assign(n)` through 5000. The benchmark asserts operation cost follows the documented complexity table instead of silently regressing to full-pool scans. It relies on a temporary monkey-patch in `tests/conftest.py` because of [fakeredis upstream PR #473](https://github.com/cunla/fakeredis-py/pull/473) (merged to master, not yet released). The patch is idempotent and self-removable once fakeredis ships a release containing the fix — see the cleanup checklist embedded in `tests/conftest.py`.
|
|
509
|
+
|
|
510
|
+
A standalone Python script `tests/_perf_real_redis.py` reproduces the same operation-complexity benchmark against a real Redis container for cross-checking and exits non-zero when a contract fails.
|
|
511
|
+
|
|
512
|
+
## Live Diagnostics
|
|
513
|
+
|
|
514
|
+
`redis-allocator` includes an observer-only diagnostics API and local dashboard for inspecting live allocator state under multi-process load.
|
|
515
|
+
|
|
516
|
+
Python API:
|
|
517
|
+
|
|
518
|
+
```python
|
|
519
|
+
from redis import Redis
|
|
520
|
+
from redis_allocator import RedisAllocatorDiagnostics
|
|
521
|
+
|
|
522
|
+
redis = Redis.from_url("redis://localhost:6379/0", decode_responses=True)
|
|
523
|
+
diagnostics = RedisAllocatorDiagnostics(redis, prefix="myapp", suffix="allocator")
|
|
524
|
+
snapshot = diagnostics.snapshot()
|
|
525
|
+
print(snapshot.to_json(indent=2))
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
One-shot JSON:
|
|
529
|
+
|
|
530
|
+
```bash
|
|
531
|
+
redis-allocator-diagnose --redis-url redis://localhost:6379/0 --prefix myapp --suffix allocator --json
|
|
532
|
+
```
|
|
533
|
+
|
|
534
|
+
Live dashboard:
|
|
535
|
+
|
|
536
|
+
```bash
|
|
537
|
+
redis-allocator-diagnose --redis-url redis://localhost:6379/0 --prefix myapp --suffix allocator --interval 1 --port 8765
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
The diagnostics collector reads pool structure and per-pool-key lock state atomically with one Lua script. Advisory metrics such as Redis memory, commandstats, slowlog, soft bindings, and orphan locks are sampled separately. V1 never mutates allocator state and does not repair corruption. Orphan-lock samples are reporting only: clean a known orphan with `free_keys(key)`, by reintroducing the same key through `extend()` / `assign()`, or with a precise Redis `DEL`; `clear()` and `gc()` do not repair orphan locks outside the pool.
|
|
479
541
|
|
|
480
|
-
|
|
542
|
+
Use `sample_limit` to bound returned examples and `scan_limit` to cap advisory keyspace scans during dashboard refreshes. When a scan is capped, the JSON marks that section as `truncated`.
|
|
481
543
|
|
|
482
544
|
## Roadmap
|
|
483
545
|
|
|
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
|
|
|
4
4
|
|
|
5
5
|
[tool.commitizen]
|
|
6
6
|
name = "cz_conventional_commits"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
tag_format = "v$version"
|
|
9
9
|
|
|
10
10
|
[tool.commitizen.version_files]
|
|
@@ -16,7 +16,8 @@ tag_format = "v$version"
|
|
|
16
16
|
# Run them explicitly with:
|
|
17
17
|
# pytest -m concurrency # multi-thread fakeredis race tests
|
|
18
18
|
# pytest -m fuzz # randomised single-thread sequences
|
|
19
|
-
# pytest -m
|
|
19
|
+
# pytest -m benchmark # allocator operation-complexity contracts
|
|
20
|
+
# pytest -m "concurrency or fuzz or benchmark"
|
|
20
21
|
markers = [
|
|
21
22
|
"concurrency: tests that drive the allocator from multiple threads (fakeredis)",
|
|
22
23
|
"fuzz: random-sequence exploratory tests (may exercise lupa edge cases)",
|
|
@@ -9,6 +9,7 @@ from redis_allocator.lock import (RedisLock, RedisLockPool, LockStatus,
|
|
|
9
9
|
BaseLock, BaseLockPool, ThreadLock, ThreadLockPool)
|
|
10
10
|
from redis_allocator.task_queue import TaskExecutePolicy, RedisTask, RedisTaskQueue
|
|
11
11
|
from redis_allocator.allocator import RedisAllocator, RedisAllocatorObject, RedisAllocatorUpdater, RedisAllocatorPolicy, DefaultRedisAllocatorPolicy
|
|
12
|
+
from redis_allocator.diagnostics import RedisAllocatorDiagnostics
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
__version__ = '0.0.1'
|
|
@@ -29,4 +30,5 @@ __all__ = [
|
|
|
29
30
|
'RedisAllocatorUpdater',
|
|
30
31
|
'RedisAllocatorPolicy',
|
|
31
32
|
'DefaultRedisAllocatorPolicy',
|
|
33
|
+
'RedisAllocatorDiagnostics',
|
|
32
34
|
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '0.7.0'
|
|
@@ -153,6 +153,7 @@ class RedisAllocatorObject(Generic[U]):
|
|
|
153
153
|
self.key = key
|
|
154
154
|
self.obj = obj
|
|
155
155
|
self.params = params
|
|
156
|
+
self._released = False
|
|
156
157
|
if self.obj is not None:
|
|
157
158
|
self.obj.set_config(key, params)
|
|
158
159
|
|
|
@@ -178,6 +179,32 @@ class RedisAllocatorObject(Generic[U]):
|
|
|
178
179
|
if self.obj is not None:
|
|
179
180
|
self.obj.close()
|
|
180
181
|
|
|
182
|
+
def release(self, timeout: int = -1):
|
|
183
|
+
"""Close the wrapped object and return the allocation to the pool.
|
|
184
|
+
|
|
185
|
+
This method is idempotent so callers can safely place it in cleanup
|
|
186
|
+
paths that may run more than once.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
timeout: Optional timeout in seconds for the returned pool item.
|
|
190
|
+
"""
|
|
191
|
+
if self._released:
|
|
192
|
+
return
|
|
193
|
+
try:
|
|
194
|
+
self.close()
|
|
195
|
+
finally:
|
|
196
|
+
self.allocator.free(self, timeout=timeout)
|
|
197
|
+
self._released = True
|
|
198
|
+
|
|
199
|
+
def __enter__(self):
|
|
200
|
+
"""Open and return this allocated object for ``with`` statements."""
|
|
201
|
+
return self.open()
|
|
202
|
+
|
|
203
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
204
|
+
"""Release this allocation when leaving a ``with`` statement."""
|
|
205
|
+
self.release()
|
|
206
|
+
return False
|
|
207
|
+
|
|
181
208
|
def is_healthy(self) -> bool:
|
|
182
209
|
"""Check if the object is healthy."""
|
|
183
210
|
if self.obj is not None:
|
|
@@ -456,6 +483,8 @@ class DefaultRedisAllocatorPolicy(RedisAllocatorPolicy[U]):
|
|
|
456
483
|
obj.name) else (obj.name if obj and hasattr(obj, 'name') else None)
|
|
457
484
|
key = allocator.malloc_key(timeout, obj_name,
|
|
458
485
|
cache_timeout=cache_timeout)
|
|
486
|
+
if key is None:
|
|
487
|
+
return None
|
|
459
488
|
alloc_obj = RedisAllocatorObject(allocator, key, obj, params)
|
|
460
489
|
if self.auto_close:
|
|
461
490
|
old_obj = self.objects.get(alloc_obj.unique_id, None)
|
|
@@ -1271,6 +1300,8 @@ class RedisAllocator(RedisLockPool, Generic[U]):
|
|
|
1271
1300
|
# Explicitly call obj.name if obj exists
|
|
1272
1301
|
name = obj.name if obj and hasattr(obj, 'name') else None
|
|
1273
1302
|
key = self.malloc_key(timeout, name, cache_timeout=cache_timeout)
|
|
1303
|
+
if key is None:
|
|
1304
|
+
return None
|
|
1274
1305
|
return RedisAllocatorObject(
|
|
1275
1306
|
self, key, obj, params
|
|
1276
1307
|
)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Diagnostics helpers for live RedisAllocator instances."""
|
|
2
|
+
|
|
3
|
+
from redis_allocator.diagnostics.collector import RedisAllocatorDiagnostics
|
|
4
|
+
from redis_allocator.diagnostics.dashboard import create_dashboard_server, serve_dashboard
|
|
5
|
+
from redis_allocator.diagnostics.models import (
|
|
6
|
+
AllocatorDiagnosticsSnapshot,
|
|
7
|
+
BindingDiagnostics,
|
|
8
|
+
IdentityDiagnostics,
|
|
9
|
+
IntegrityDiagnostics,
|
|
10
|
+
OrphanDiagnostics,
|
|
11
|
+
PoolDiagnostics,
|
|
12
|
+
PressureDiagnostics,
|
|
13
|
+
Recommendation,
|
|
14
|
+
RedisDiagnostics,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"AllocatorDiagnosticsSnapshot",
|
|
19
|
+
"BindingDiagnostics",
|
|
20
|
+
"IdentityDiagnostics",
|
|
21
|
+
"IntegrityDiagnostics",
|
|
22
|
+
"OrphanDiagnostics",
|
|
23
|
+
"PoolDiagnostics",
|
|
24
|
+
"PressureDiagnostics",
|
|
25
|
+
"Recommendation",
|
|
26
|
+
"RedisAllocatorDiagnostics",
|
|
27
|
+
"RedisDiagnostics",
|
|
28
|
+
"create_dashboard_server",
|
|
29
|
+
"serve_dashboard",
|
|
30
|
+
]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Command-line entry point for allocator diagnostics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
|
|
7
|
+
from redis import Redis
|
|
8
|
+
|
|
9
|
+
from redis_allocator.diagnostics.collector import RedisAllocatorDiagnostics
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
13
|
+
"""Build the diagnostics CLI parser."""
|
|
14
|
+
parser = argparse.ArgumentParser(description="Inspect a live RedisAllocator instance.")
|
|
15
|
+
parser.add_argument("--redis-url", default="redis://localhost:6379/0")
|
|
16
|
+
parser.add_argument("--prefix", required=True)
|
|
17
|
+
parser.add_argument("--suffix", default="allocator")
|
|
18
|
+
parser.add_argument("--shared", action="store_true")
|
|
19
|
+
parser.add_argument("--sample-limit", type=int, default=20)
|
|
20
|
+
parser.add_argument("--scan-limit", type=int, default=1000)
|
|
21
|
+
parser.add_argument("--json", action="store_true", help="Print one JSON snapshot and exit.")
|
|
22
|
+
parser.add_argument("--interval", type=float, default=1.0)
|
|
23
|
+
parser.add_argument("--host", default="127.0.0.1")
|
|
24
|
+
parser.add_argument("--port", type=int, default=8765)
|
|
25
|
+
return parser
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def main(argv: list[str] | None = None) -> int:
|
|
29
|
+
"""Run the diagnostics CLI."""
|
|
30
|
+
parser = build_parser()
|
|
31
|
+
args = parser.parse_args(argv)
|
|
32
|
+
redis_client = Redis.from_url(args.redis_url, decode_responses=True)
|
|
33
|
+
diagnostics = RedisAllocatorDiagnostics(
|
|
34
|
+
redis_client,
|
|
35
|
+
prefix=args.prefix,
|
|
36
|
+
suffix=args.suffix,
|
|
37
|
+
shared=args.shared,
|
|
38
|
+
sample_limit=args.sample_limit,
|
|
39
|
+
scan_limit=args.scan_limit,
|
|
40
|
+
)
|
|
41
|
+
if args.json:
|
|
42
|
+
print(diagnostics.snapshot_json(indent=2))
|
|
43
|
+
return 0
|
|
44
|
+
|
|
45
|
+
from redis_allocator.diagnostics.dashboard import serve_dashboard
|
|
46
|
+
|
|
47
|
+
serve_dashboard(diagnostics, host=args.host, port=args.port, interval=args.interval)
|
|
48
|
+
return 0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
if __name__ == "__main__":
|
|
52
|
+
raise SystemExit(main())
|