ringq 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ringq-0.1.0/CHANGELOG.md +19 -0
- ringq-0.1.0/LICENSE +21 -0
- ringq-0.1.0/MANIFEST.in +8 -0
- ringq-0.1.0/PKG-INFO +330 -0
- ringq-0.1.0/README.md +296 -0
- ringq-0.1.0/pyproject.toml +59 -0
- ringq-0.1.0/setup.cfg +4 -0
- ringq-0.1.0/setup.py +15 -0
- ringq-0.1.0/src/ringq/__init__.py +10 -0
- ringq-0.1.0/src/ringq/_core.c +13694 -0
- ringq-0.1.0/src/ringq/_core.pxd +37 -0
- ringq-0.1.0/src/ringq/_core.pyi +19 -0
- ringq-0.1.0/src/ringq/_core.pyx +324 -0
- ringq-0.1.0/src/ringq/_exceptions.py +14 -0
- ringq-0.1.0/src/ringq/_fast_validate.c +8827 -0
- ringq-0.1.0/src/ringq/_fast_validate.pyi +5 -0
- ringq-0.1.0/src/ringq/_fast_validate.pyx +89 -0
- ringq-0.1.0/src/ringq/py.typed +0 -0
- ringq-0.1.0/src/ringq/queue.py +333 -0
- ringq-0.1.0/src/ringq.egg-info/PKG-INFO +330 -0
- ringq-0.1.0/src/ringq.egg-info/SOURCES.txt +35 -0
- ringq-0.1.0/src/ringq.egg-info/dependency_links.txt +1 -0
- ringq-0.1.0/src/ringq.egg-info/requires.txt +9 -0
- ringq-0.1.0/src/ringq.egg-info/top_level.txt +1 -0
- ringq-0.1.0/tests/test_async.py +100 -0
- ringq-0.1.0/tests/test_asyncio_compat.py +673 -0
- ringq-0.1.0/tests/test_bounded.py +38 -0
- ringq-0.1.0/tests/test_combinations.py +1202 -0
- ringq-0.1.0/tests/test_coverage_gaps.py +599 -0
- ringq-0.1.0/tests/test_dedup.py +84 -0
- ringq-0.1.0/tests/test_edge_cases.py +101 -0
- ringq-0.1.0/tests/test_eviction.py +139 -0
- ringq-0.1.0/tests/test_fifo.py +59 -0
- ringq-0.1.0/tests/test_race_and_stress.py +260 -0
- ringq-0.1.0/tests/test_shutdown.py +196 -0
- ringq-0.1.0/tests/test_task_done.py +75 -0
- ringq-0.1.0/tests/test_validation.py +110 -0
ringq-0.1.0/CHANGELOG.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [0.1.0] — 2026-03-21
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- `Queue` class — drop-in `asyncio.Queue` replacement backed by a Cython ring buffer
|
|
9
|
+
- Bounded mode with `maxsize` parameter
|
|
10
|
+
- Eviction policies: `eviction="old"` (discard oldest) and `eviction="new"` (reject newest)
|
|
11
|
+
- Deduplication: `dedup=True` / `"drop"` (drop duplicates) and `dedup="replace"` (update in-place)
|
|
12
|
+
- Custom key function for dedup via `key=` parameter
|
|
13
|
+
- JSON validation on put via `validate=True` (Cython fast path) or `validate="full"` (orjson)
|
|
14
|
+
- `peek_nowait()` — inspect next item without removing
|
|
15
|
+
- `stats()` — eviction/dedup counters
|
|
16
|
+
- `shutdown(immediate=False)` — Python 3.13-compatible shutdown
|
|
17
|
+
- `task_done()` / `join()` support with Cython-side `_unfinished` counter
|
|
18
|
+
- Full async API: `put()`, `get()`, `join()`
|
|
19
|
+
- PEP 561 type stubs (`py.typed`, `.pyi` files)
|
ringq-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 cutient
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ringq-0.1.0/MANIFEST.in
ADDED
ringq-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ringq
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: High-performance async queue backed by a Cython ring buffer
|
|
5
|
+
Author: cutient
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/cutient/ringq
|
|
8
|
+
Project-URL: Repository, https://github.com/cutient/ringq
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/cutient/ringq/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/cutient/ringq/blob/main/CHANGELOG.md
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Framework :: AsyncIO
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Cython
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Provides-Extra: orjson
|
|
27
|
+
Requires-Dist: orjson>=3.9; extra == "orjson"
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
30
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
31
|
+
Requires-Dist: cython>=3.0; extra == "dev"
|
|
32
|
+
Requires-Dist: orjson>=3.9; extra == "dev"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# ringq
|
|
36
|
+
|
|
37
|
+
[](https://github.com/cutient/ringq/actions/workflows/ci.yml)
|
|
38
|
+
[](https://pypi.org/project/ringq/)
|
|
39
|
+
[](https://pypi.org/project/ringq/)
|
|
40
|
+
[](LICENSE)
|
|
41
|
+
|
|
42
|
+
High-performance async queue backed by a Cython ring buffer. Drop-in replacement for `asyncio.Queue` with optional eviction, deduplication, and JSON validation.
|
|
43
|
+
|
|
44
|
+
## Why ringq?
|
|
45
|
+
|
|
46
|
+
- **Faster than `asyncio.Queue`** — Cython ring buffer with power-of-2 bitmask indexing, zero Python overhead in the hot path. 10–50% faster depending on configuration.
|
|
47
|
+
- **Eviction policies** — bounded queues that never block: discard the oldest item or silently reject the newest.
|
|
48
|
+
- **Built-in deduplication** — drop or replace duplicates by value or custom key, without maintaining a separate set.
|
|
49
|
+
- **JSON validation** — catch non-serializable data at enqueue time, not when you try to send it downstream.
|
|
50
|
+
- **Drop-in compatible** — same interface as `asyncio.Queue`, including `shutdown()` (Python 3.13+), `task_done()`, and `join()`.
|
|
51
|
+
- **Zero runtime dependencies** — optional `orjson` for faster full validation.
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install ringq
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Optional: install `orjson` for faster `validate="full"` mode:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install ringq orjson
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Quick start
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import asyncio
|
|
69
|
+
from ringq import Queue
|
|
70
|
+
|
|
71
|
+
async def main():
|
|
72
|
+
# Basic FIFO (same as asyncio.Queue)
|
|
73
|
+
q = Queue()
|
|
74
|
+
await q.put("hello")
|
|
75
|
+
print(await q.get()) # "hello"
|
|
76
|
+
|
|
77
|
+
# Bounded with eviction (discard oldest when full)
|
|
78
|
+
q = Queue(maxsize=100, eviction=True)
|
|
79
|
+
|
|
80
|
+
# Deduplication — drop new duplicates
|
|
81
|
+
q = Queue(maxsize=100, dedup=True)
|
|
82
|
+
|
|
83
|
+
# Deduplication — replace existing with new value
|
|
84
|
+
q = Queue(maxsize=100, dedup="replace", key=lambda x: x["id"])
|
|
85
|
+
|
|
86
|
+
# JSON validation
|
|
87
|
+
q = Queue(validate=True)
|
|
88
|
+
q.put_nowait({"key": "value"}) # OK
|
|
89
|
+
# q.put_nowait(set()) # raises TypeError
|
|
90
|
+
|
|
91
|
+
asyncio.run(main())
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Features
|
|
95
|
+
|
|
96
|
+
### Eviction policies
|
|
97
|
+
|
|
98
|
+
Control what happens when a bounded queue is full.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
# Default: raise QueueFull (same as asyncio.Queue)
|
|
102
|
+
q = Queue(maxsize=100)
|
|
103
|
+
|
|
104
|
+
# Discard oldest item to make room for the new one
|
|
105
|
+
q = Queue(maxsize=100, eviction=True) # or eviction="old"
|
|
106
|
+
|
|
107
|
+
# Silently reject the new item, never blocks
|
|
108
|
+
q = Queue(maxsize=100, eviction="new")
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
With `eviction="old"`, `put()` and `put_nowait()` never block or raise `QueueFull` — the oldest item is evicted automatically. With `eviction="new"`, the new item is silently dropped and `put_nowait()` returns `False`.
|
|
112
|
+
|
|
113
|
+
### Deduplication
|
|
114
|
+
|
|
115
|
+
Prevent duplicate items from accumulating in the queue.
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
# Drop duplicates — keep the original, reject the new one
|
|
119
|
+
q = Queue(dedup=True) # or dedup="drop"
|
|
120
|
+
q.put_nowait("a") # True
|
|
121
|
+
q.put_nowait("a") # False (duplicate dropped)
|
|
122
|
+
|
|
123
|
+
# Replace duplicates — update the value in-place
|
|
124
|
+
q = Queue(dedup="replace")
|
|
125
|
+
q.put_nowait("old_value") # True
|
|
126
|
+
q.put_nowait("old_value") # True (original replaced)
|
|
127
|
+
|
|
128
|
+
# Custom key function — deduplicate by a specific field
|
|
129
|
+
q = Queue(dedup="replace", key=lambda x: x["id"])
|
|
130
|
+
q.put_nowait({"id": 1, "status": "pending"})
|
|
131
|
+
q.put_nowait({"id": 1, "status": "done"}) # replaces previous
|
|
132
|
+
print(q.get_nowait()) # {"id": 1, "status": "done"}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
`put_nowait()` returns `True` if the item was inserted, `False` if it was dropped as a duplicate.
|
|
136
|
+
|
|
137
|
+
### JSON validation
|
|
138
|
+
|
|
139
|
+
Catch non-JSON-serializable data at enqueue time.
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
# Fast mode (Cython, type checks only — no serialization)
|
|
143
|
+
q = Queue(validate=True) # or validate="fast"
|
|
144
|
+
q.put_nowait({"key": [1, 2]}) # OK
|
|
145
|
+
q.put_nowait({1: "value"}) # TypeError — dict keys must be strings
|
|
146
|
+
|
|
147
|
+
# Full mode (actual JSON serialization via orjson or stdlib json)
|
|
148
|
+
q = Queue(validate="full")
|
|
149
|
+
q.put_nowait({"key": "value"}) # OK
|
|
150
|
+
q.put_nowait(set()) # TypeError
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Fast mode checks basic types recursively via Cython (None, bool, int, float, str, list, tuple, dict with string keys). Full mode performs an actual serialization round-trip and accepts anything that `json.dumps` (or `orjson.dumps`) accepts.
|
|
154
|
+
|
|
155
|
+
### Combining features
|
|
156
|
+
|
|
157
|
+
All features compose freely:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
# Bounded queue with eviction, dedup by key, and JSON validation
|
|
161
|
+
q = Queue(
|
|
162
|
+
maxsize=1000,
|
|
163
|
+
eviction=True,
|
|
164
|
+
dedup="replace",
|
|
165
|
+
key=lambda x: x["id"],
|
|
166
|
+
validate=True,
|
|
167
|
+
)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Shutdown
|
|
171
|
+
|
|
172
|
+
Gracefully shut down a queue, compatible with Python 3.13+ `asyncio.Queue.shutdown()`.
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
# Graceful — allow consumers to drain remaining items
|
|
176
|
+
q.shutdown()
|
|
177
|
+
# q.put_nowait(item) # raises QueueShutDown
|
|
178
|
+
await q.get() # returns remaining items, then raises QueueShutDown
|
|
179
|
+
|
|
180
|
+
# Immediate — discard all items, cancel all waiters
|
|
181
|
+
q.shutdown(immediate=True)
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### Statistics
|
|
185
|
+
|
|
186
|
+
Track eviction and deduplication counters:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
q = Queue(maxsize=2, eviction=True, dedup=True)
|
|
190
|
+
q.put_nowait("a")
|
|
191
|
+
q.put_nowait("b")
|
|
192
|
+
q.put_nowait("c") # evicts "a"
|
|
193
|
+
q.put_nowait("c") # duplicate dropped
|
|
194
|
+
|
|
195
|
+
print(q.stats())
|
|
196
|
+
# {
|
|
197
|
+
# "evictions": 1,
|
|
198
|
+
# "dedup_drops": 1,
|
|
199
|
+
# "dedup_replacements": 0,
|
|
200
|
+
# "invalidated_skips": 0,
|
|
201
|
+
# "maxsize": 2,
|
|
202
|
+
# }
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
## API reference
|
|
206
|
+
|
|
207
|
+
### Constructor
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
Queue(
|
|
211
|
+
maxsize=0,
|
|
212
|
+
*,
|
|
213
|
+
eviction=False, # False | True | "old" | "new"
|
|
214
|
+
dedup=False, # False | True | "drop" | "replace"
|
|
215
|
+
key=None, # callable(item) -> hashable key
|
|
216
|
+
validate=False, # False | True | "fast" | "full"
|
|
217
|
+
)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
| Parameter | Type | Default | Description |
|
|
221
|
+
|-----------|------|---------|-------------|
|
|
222
|
+
| `maxsize` | `int` | `0` | Maximum number of items. `0` = unbounded. |
|
|
223
|
+
| `eviction` | `bool \| str` | `False` | `True`/`"old"`: evict oldest. `"new"`: reject newest. |
|
|
224
|
+
| `dedup` | `bool \| str` | `False` | `True`/`"drop"`: drop duplicates. `"replace"`: update in-place. |
|
|
225
|
+
| `key` | `callable` | `None` | Extract dedup key from items. Requires `dedup` to be enabled. |
|
|
226
|
+
| `validate` | `bool \| str` | `False` | `True`/`"fast"`: Cython basic type check. `"full"`: actual JSON serialization round-trip. |
|
|
227
|
+
|
|
228
|
+
### Methods
|
|
229
|
+
|
|
230
|
+
| Method | Returns | Raises | Description |
|
|
231
|
+
|--------|---------|--------|-------------|
|
|
232
|
+
| `put_nowait(item)` | `bool` | `QueueFull`, `QueueShutDown` | Insert item. Returns `True` if inserted, `False` if dropped. |
|
|
233
|
+
| `get_nowait()` | item | `QueueEmpty`, `QueueShutDown` | Remove and return next item. |
|
|
234
|
+
| `await put(item)` | `bool` | `QueueShutDown` | Async put. Waits if bounded and full (unless eviction enabled). |
|
|
235
|
+
| `await get()` | item | `QueueShutDown` | Async get. Waits if empty. |
|
|
236
|
+
| `peek_nowait()` | item | `QueueEmpty` | Return next item without removing it. |
|
|
237
|
+
| `task_done()` | `None` | `ValueError` | Mark a retrieved item as processed. |
|
|
238
|
+
| `await join()` | `None` | — | Wait until all items have been processed (`task_done()` called for each). |
|
|
239
|
+
| `clear()` | `None` | — | Remove all items and reset unfinished task counter. |
|
|
240
|
+
| `shutdown(immediate=False)` | `None` | — | Shut down the queue. Idempotent. |
|
|
241
|
+
| `stats()` | `dict` | — | Return `{"evictions", "dedup_drops", "dedup_replacements", "invalidated_skips", "maxsize"}`. |
|
|
242
|
+
| `qsize()` / `len(q)` | `int` | — | Number of items currently in the queue. |
|
|
243
|
+
| `empty()` | `bool` | — | `True` if the queue is empty. |
|
|
244
|
+
| `full()` | `bool` | — | `True` if bounded and at capacity. Always `False` for unbounded queues. |
|
|
245
|
+
| `maxsize` (property) | `int` | — | The queue's capacity (from constructor). |
|
|
246
|
+
|
|
247
|
+
### Exceptions
|
|
248
|
+
|
|
249
|
+
| Exception | When raised |
|
|
250
|
+
|-----------|-------------|
|
|
251
|
+
| `QueueEmpty` | `get_nowait()` on an empty queue |
|
|
252
|
+
| `QueueFull` | `put_nowait()` on a full bounded queue (without eviction) |
|
|
253
|
+
| `QueueShutDown` | `put()`/`get()` after `shutdown()` |
|
|
254
|
+
|
|
255
|
+
All exceptions are importable from `ringq`:
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
from ringq import Queue, QueueEmpty, QueueFull, QueueShutDown
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
## Migrating from asyncio.Queue
|
|
262
|
+
|
|
263
|
+
ringq is a drop-in replacement. Change one import:
|
|
264
|
+
|
|
265
|
+
```diff
|
|
266
|
+
-from asyncio import Queue
|
|
267
|
+
+from ringq import Queue
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
Existing code continues to work. To take advantage of new features, add keyword arguments:
|
|
271
|
+
|
|
272
|
+
```python
|
|
273
|
+
# Before (asyncio.Queue)
|
|
274
|
+
q = asyncio.Queue(maxsize=100)
|
|
275
|
+
|
|
276
|
+
# After (ringq — same behavior, faster)
|
|
277
|
+
q = Queue(maxsize=100)
|
|
278
|
+
|
|
279
|
+
# After (ringq — with features)
|
|
280
|
+
q = Queue(maxsize=100, eviction=True, dedup="replace", key=lambda x: x["id"])
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
**Behavioral difference:** `put_nowait()` returns `bool` (always `True` for standard FIFO usage) instead of `None`.
|
|
284
|
+
|
|
285
|
+
## Benchmarks
|
|
286
|
+
|
|
287
|
+
1,000,000 `put_nowait` + 1,000,000 `get_nowait` operations, Python 3.14, Linux x86_64:
|
|
288
|
+
|
|
289
|
+
| Configuration | asyncio.Queue | ringq | Speedup |
|
|
290
|
+
|---|--:|--:|--:|
|
|
291
|
+
| Unbounded | 5.2M ops/s | 8.3M ops/s | **1.6x** |
|
|
292
|
+
| Bounded (maxsize=1000) | 4.7M ops/s | 9.0M ops/s | **1.9x** |
|
|
293
|
+
| Eviction old (maxsize=1000) | — | 7.6M ops/s | — |
|
|
294
|
+
| Eviction new (maxsize=1000) | — | 9.5M ops/s | — |
|
|
295
|
+
| Dedup drop (maxsize=1000) | — | 7.0M ops/s | — |
|
|
296
|
+
| Dedup replace (maxsize=1000) | — | 5.0M ops/s | — |
|
|
297
|
+
| Validate fast | — | 3.7M ops/s | — |
|
|
298
|
+
| Validate full | — | 2.4M ops/s | — |
|
|
299
|
+
|
|
300
|
+
Run benchmarks yourself:
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
uv run python benchmarks/run.py
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
## Development
|
|
307
|
+
|
|
308
|
+
### Setup
|
|
309
|
+
|
|
310
|
+
```bash
|
|
311
|
+
git clone https://github.com/cutient/ringq.git
|
|
312
|
+
cd ringq
|
|
313
|
+
uv sync --extra dev
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
### Test
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
uv run pytest tests/ -v
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
### Benchmark
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
uv run python benchmarks/run.py
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
## License
|
|
329
|
+
|
|
330
|
+
[MIT](LICENSE)
|
ringq-0.1.0/README.md
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
# ringq
|
|
2
|
+
|
|
3
|
+
[](https://github.com/cutient/ringq/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/ringq/)
|
|
5
|
+
[](https://pypi.org/project/ringq/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
High-performance async queue backed by a Cython ring buffer. Drop-in replacement for `asyncio.Queue` with optional eviction, deduplication, and JSON validation.
|
|
9
|
+
|
|
10
|
+
## Why ringq?
|
|
11
|
+
|
|
12
|
+
- **Faster than `asyncio.Queue`** — Cython ring buffer with power-of-2 bitmask indexing, zero Python overhead in the hot path. 10–50% faster depending on configuration.
|
|
13
|
+
- **Eviction policies** — bounded queues that never block: discard the oldest item or silently reject the newest.
|
|
14
|
+
- **Built-in deduplication** — drop or replace duplicates by value or custom key, without maintaining a separate set.
|
|
15
|
+
- **JSON validation** — catch non-serializable data at enqueue time, not when you try to send it downstream.
|
|
16
|
+
- **Drop-in compatible** — same interface as `asyncio.Queue`, including `shutdown()` (Python 3.13+), `task_done()`, and `join()`.
|
|
17
|
+
- **Zero runtime dependencies** — optional `orjson` for faster full validation.
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install ringq
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Optional: install `orjson` for faster `validate="full"` mode:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install ringq orjson
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Quick start
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
import asyncio
|
|
35
|
+
from ringq import Queue
|
|
36
|
+
|
|
37
|
+
async def main():
|
|
38
|
+
# Basic FIFO (same as asyncio.Queue)
|
|
39
|
+
q = Queue()
|
|
40
|
+
await q.put("hello")
|
|
41
|
+
print(await q.get()) # "hello"
|
|
42
|
+
|
|
43
|
+
# Bounded with eviction (discard oldest when full)
|
|
44
|
+
q = Queue(maxsize=100, eviction=True)
|
|
45
|
+
|
|
46
|
+
# Deduplication — drop new duplicates
|
|
47
|
+
q = Queue(maxsize=100, dedup=True)
|
|
48
|
+
|
|
49
|
+
# Deduplication — replace existing with new value
|
|
50
|
+
q = Queue(maxsize=100, dedup="replace", key=lambda x: x["id"])
|
|
51
|
+
|
|
52
|
+
# JSON validation
|
|
53
|
+
q = Queue(validate=True)
|
|
54
|
+
q.put_nowait({"key": "value"}) # OK
|
|
55
|
+
# q.put_nowait(set()) # raises TypeError
|
|
56
|
+
|
|
57
|
+
asyncio.run(main())
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Features
|
|
61
|
+
|
|
62
|
+
### Eviction policies
|
|
63
|
+
|
|
64
|
+
Control what happens when a bounded queue is full.
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
# Default: raise QueueFull (same as asyncio.Queue)
|
|
68
|
+
q = Queue(maxsize=100)
|
|
69
|
+
|
|
70
|
+
# Discard oldest item to make room for the new one
|
|
71
|
+
q = Queue(maxsize=100, eviction=True) # or eviction="old"
|
|
72
|
+
|
|
73
|
+
# Silently reject the new item, never blocks
|
|
74
|
+
q = Queue(maxsize=100, eviction="new")
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
With `eviction="old"`, `put()` and `put_nowait()` never block or raise `QueueFull` — the oldest item is evicted automatically. With `eviction="new"`, the new item is silently dropped and `put_nowait()` returns `False`.
|
|
78
|
+
|
|
79
|
+
### Deduplication
|
|
80
|
+
|
|
81
|
+
Prevent duplicate items from accumulating in the queue.
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
# Drop duplicates — keep the original, reject the new one
|
|
85
|
+
q = Queue(dedup=True) # or dedup="drop"
|
|
86
|
+
q.put_nowait("a") # True
|
|
87
|
+
q.put_nowait("a") # False (duplicate dropped)
|
|
88
|
+
|
|
89
|
+
# Replace duplicates — update the value in-place
|
|
90
|
+
q = Queue(dedup="replace")
|
|
91
|
+
q.put_nowait("old_value") # True
|
|
92
|
+
q.put_nowait("old_value") # True (original replaced)
|
|
93
|
+
|
|
94
|
+
# Custom key function — deduplicate by a specific field
|
|
95
|
+
q = Queue(dedup="replace", key=lambda x: x["id"])
|
|
96
|
+
q.put_nowait({"id": 1, "status": "pending"})
|
|
97
|
+
q.put_nowait({"id": 1, "status": "done"}) # replaces previous
|
|
98
|
+
print(q.get_nowait()) # {"id": 1, "status": "done"}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`put_nowait()` returns `True` if the item was inserted, `False` if it was dropped as a duplicate.
|
|
102
|
+
|
|
103
|
+
### JSON validation
|
|
104
|
+
|
|
105
|
+
Catch non-JSON-serializable data at enqueue time.
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
# Fast mode (Cython, type checks only — no serialization)
|
|
109
|
+
q = Queue(validate=True) # or validate="fast"
|
|
110
|
+
q.put_nowait({"key": [1, 2]}) # OK
|
|
111
|
+
q.put_nowait({1: "value"}) # TypeError — dict keys must be strings
|
|
112
|
+
|
|
113
|
+
# Full mode (actual JSON serialization via orjson or stdlib json)
|
|
114
|
+
q = Queue(validate="full")
|
|
115
|
+
q.put_nowait({"key": "value"}) # OK
|
|
116
|
+
q.put_nowait(set()) # TypeError
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Fast mode checks basic types recursively via Cython (None, bool, int, float, str, list, tuple, dict with string keys). Full mode performs an actual serialization round-trip and accepts anything that `json.dumps` (or `orjson.dumps`) accepts.
|
|
120
|
+
|
|
121
|
+
### Combining features
|
|
122
|
+
|
|
123
|
+
All features compose freely:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
# Bounded queue with eviction, dedup by key, and JSON validation
|
|
127
|
+
q = Queue(
|
|
128
|
+
maxsize=1000,
|
|
129
|
+
eviction=True,
|
|
130
|
+
dedup="replace",
|
|
131
|
+
key=lambda x: x["id"],
|
|
132
|
+
validate=True,
|
|
133
|
+
)
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### Shutdown
|
|
137
|
+
|
|
138
|
+
Gracefully shut down a queue, compatible with Python 3.13+ `asyncio.Queue.shutdown()`.
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
# Graceful — allow consumers to drain remaining items
|
|
142
|
+
q.shutdown()
|
|
143
|
+
# q.put_nowait(item) # raises QueueShutDown
|
|
144
|
+
await q.get() # returns remaining items, then raises QueueShutDown
|
|
145
|
+
|
|
146
|
+
# Immediate — discard all items, cancel all waiters
|
|
147
|
+
q.shutdown(immediate=True)
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Statistics
|
|
151
|
+
|
|
152
|
+
Track eviction and deduplication counters:
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
q = Queue(maxsize=2, eviction=True, dedup=True)
|
|
156
|
+
q.put_nowait("a")
|
|
157
|
+
q.put_nowait("b")
|
|
158
|
+
q.put_nowait("c") # evicts "a"
|
|
159
|
+
q.put_nowait("c") # duplicate dropped
|
|
160
|
+
|
|
161
|
+
print(q.stats())
|
|
162
|
+
# {
|
|
163
|
+
# "evictions": 1,
|
|
164
|
+
# "dedup_drops": 1,
|
|
165
|
+
# "dedup_replacements": 0,
|
|
166
|
+
# "invalidated_skips": 0,
|
|
167
|
+
# "maxsize": 2,
|
|
168
|
+
# }
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## API reference
|
|
172
|
+
|
|
173
|
+
### Constructor
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
Queue(
|
|
177
|
+
maxsize=0,
|
|
178
|
+
*,
|
|
179
|
+
eviction=False, # False | True | "old" | "new"
|
|
180
|
+
dedup=False, # False | True | "drop" | "replace"
|
|
181
|
+
key=None, # callable(item) -> hashable key
|
|
182
|
+
validate=False, # False | True | "fast" | "full"
|
|
183
|
+
)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
| Parameter | Type | Default | Description |
|
|
187
|
+
|-----------|------|---------|-------------|
|
|
188
|
+
| `maxsize` | `int` | `0` | Maximum number of items. `0` = unbounded. |
|
|
189
|
+
| `eviction` | `bool \| str` | `False` | `True`/`"old"`: evict oldest. `"new"`: reject newest. |
|
|
190
|
+
| `dedup` | `bool \| str` | `False` | `True`/`"drop"`: drop duplicates. `"replace"`: update in-place. |
|
|
191
|
+
| `key` | `callable` | `None` | Extract dedup key from items. Requires `dedup` to be enabled. |
|
|
192
|
+
| `validate` | `bool \| str` | `False` | `True`/`"fast"`: Cython basic type check. `"full"`: actual JSON serialization round-trip. |
|
|
193
|
+
|
|
194
|
+
### Methods
|
|
195
|
+
|
|
196
|
+
| Method | Returns | Raises | Description |
|
|
197
|
+
|--------|---------|--------|-------------|
|
|
198
|
+
| `put_nowait(item)` | `bool` | `QueueFull`, `QueueShutDown` | Insert item. Returns `True` if inserted, `False` if dropped. |
|
|
199
|
+
| `get_nowait()` | item | `QueueEmpty`, `QueueShutDown` | Remove and return next item. |
|
|
200
|
+
| `await put(item)` | `bool` | `QueueShutDown` | Async put. Waits if bounded and full (unless eviction enabled). |
|
|
201
|
+
| `await get()` | item | `QueueShutDown` | Async get. Waits if empty. |
|
|
202
|
+
| `peek_nowait()` | item | `QueueEmpty` | Return next item without removing it. |
|
|
203
|
+
| `task_done()` | `None` | `ValueError` | Mark a retrieved item as processed. |
|
|
204
|
+
| `await join()` | `None` | — | Wait until all items have been processed (`task_done()` called for each). |
|
|
205
|
+
| `clear()` | `None` | — | Remove all items and reset unfinished task counter. |
|
|
206
|
+
| `shutdown(immediate=False)` | `None` | — | Shut down the queue. Idempotent. |
|
|
207
|
+
| `stats()` | `dict` | — | Return `{"evictions", "dedup_drops", "dedup_replacements", "invalidated_skips", "maxsize"}`. |
|
|
208
|
+
| `qsize()` / `len(q)` | `int` | — | Number of items currently in the queue. |
|
|
209
|
+
| `empty()` | `bool` | — | `True` if the queue is empty. |
|
|
210
|
+
| `full()` | `bool` | — | `True` if bounded and at capacity. Always `False` for unbounded queues. |
|
|
211
|
+
| `maxsize` (property) | `int` | — | The queue's capacity (from constructor). |
|
|
212
|
+
|
|
213
|
+
### Exceptions
|
|
214
|
+
|
|
215
|
+
| Exception | When raised |
|
|
216
|
+
|-----------|-------------|
|
|
217
|
+
| `QueueEmpty` | `get_nowait()` on an empty queue |
|
|
218
|
+
| `QueueFull` | `put_nowait()` on a full bounded queue (without eviction) |
|
|
219
|
+
| `QueueShutDown` | `put()`/`get()` after `shutdown()` |
|
|
220
|
+
|
|
221
|
+
All exceptions are importable from `ringq`:
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
from ringq import Queue, QueueEmpty, QueueFull, QueueShutDown
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Migrating from asyncio.Queue
|
|
228
|
+
|
|
229
|
+
ringq is a drop-in replacement. Change one import:
|
|
230
|
+
|
|
231
|
+
```diff
|
|
232
|
+
-from asyncio import Queue
|
|
233
|
+
+from ringq import Queue
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Existing code continues to work. To take advantage of new features, add keyword arguments:
|
|
237
|
+
|
|
238
|
+
```python
|
|
239
|
+
# Before (asyncio.Queue)
|
|
240
|
+
q = asyncio.Queue(maxsize=100)
|
|
241
|
+
|
|
242
|
+
# After (ringq — same behavior, faster)
|
|
243
|
+
q = Queue(maxsize=100)
|
|
244
|
+
|
|
245
|
+
# After (ringq — with features)
|
|
246
|
+
q = Queue(maxsize=100, eviction=True, dedup="replace", key=lambda x: x["id"])
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
**Behavioral difference:** `put_nowait()` returns `bool` (always `True` for standard FIFO usage) instead of `None`.
|
|
250
|
+
|
|
251
|
+
## Benchmarks
|
|
252
|
+
|
|
253
|
+
1,000,000 `put_nowait` + 1,000,000 `get_nowait` operations, Python 3.14, Linux x86_64:
|
|
254
|
+
|
|
255
|
+
| Configuration | asyncio.Queue | ringq | Speedup |
|
|
256
|
+
|---|--:|--:|--:|
|
|
257
|
+
| Unbounded | 5.2M ops/s | 8.3M ops/s | **1.6x** |
|
|
258
|
+
| Bounded (maxsize=1000) | 4.7M ops/s | 9.0M ops/s | **1.9x** |
|
|
259
|
+
| Eviction old (maxsize=1000) | — | 7.6M ops/s | — |
|
|
260
|
+
| Eviction new (maxsize=1000) | — | 9.5M ops/s | — |
|
|
261
|
+
| Dedup drop (maxsize=1000) | — | 7.0M ops/s | — |
|
|
262
|
+
| Dedup replace (maxsize=1000) | — | 5.0M ops/s | — |
|
|
263
|
+
| Validate fast | — | 3.7M ops/s | — |
|
|
264
|
+
| Validate full | — | 2.4M ops/s | — |
|
|
265
|
+
|
|
266
|
+
Run benchmarks yourself:
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
uv run python benchmarks/run.py
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## Development
|
|
273
|
+
|
|
274
|
+
### Setup
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
git clone https://github.com/cutient/ringq.git
|
|
278
|
+
cd ringq
|
|
279
|
+
uv sync --extra dev
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
### Test
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
uv run pytest tests/ -v
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
### Benchmark
|
|
289
|
+
|
|
290
|
+
```bash
|
|
291
|
+
uv run python benchmarks/run.py
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
## License
|
|
295
|
+
|
|
296
|
+
[MIT](LICENSE)
|