slick-queue-py 2.1.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/API_DIFFERENCES.md +61 -8
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/CHANGELOG.md +128 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/PKG-INFO +126 -5
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/README.md +125 -4
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/slick_queue_py.egg-info/SOURCES.txt +4 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/slick_queue_py.py +667 -151
- slick_queue_py-2.4.0/tests/cpp_spsc_tester.cpp +136 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_items_per_slot.py +3 -4
- slick_queue_py-2.4.0/tests/test_segment_lifetime.py +410 -0
- slick_queue_py-2.4.0/tests/test_spsc.py +715 -0
- slick_queue_py-2.4.0/tests/test_spsc_interop.py +138 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_traits.py +8 -12
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/FUNDING.yml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/ci.yml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/publish-conda.yml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/publish-pypi.yml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/publish-release.yml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/release.yml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.gitignore +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.pypirc.template +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/BUILDING.md +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/CMakeLists.txt +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/LICENSE +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/MANIFEST.in +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/atomic_ops.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/atomic_ops_ext.cpp +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/conda.recipe/meta.yaml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/pyproject.toml +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/requirements.txt +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/setup.cfg +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/setup.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cleanup_shm.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_consumer.cpp +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_multi_producer.cpp +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_producer.cpp +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_read_last_tester.cpp +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_work_stealing_consumer.cpp +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/run_test.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_atomic_cursor.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_atomic_ops.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_interop.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_local_mode.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_loss_count.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_modern_format.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_multi_producer.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_queue.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_read_last.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_read_last_interop.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_reset_detection.py +0 -0
- {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_wrap_invariants.py +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# C++ / Python API Differences
|
|
2
2
|
|
|
3
|
-
This document explains the intentional API differences between the C++ `slick::queue<T>` and Python `SlickQueue` implementations.
|
|
3
|
+
This document explains the intentional API differences between the C++ `slick::queue<T>` and Python `SlickQueue` implementations, and between C++ `slick::spsc_queue<T>` and Python `SpscQueue` (section 8).
|
|
4
4
|
|
|
5
|
-
Both track the same version line; this document describes slick-queue v2.
|
|
5
|
+
Both track the same version line; this document describes slick-queue v2.4.0.
|
|
6
6
|
|
|
7
7
|
## Core Compatibility
|
|
8
8
|
|
|
@@ -227,27 +227,77 @@ q[idx][:4] = struct.pack("<i", 42) # Manual packing
|
|
|
227
227
|
{
|
|
228
228
|
slick::queue<T> q(size, "name");
|
|
229
229
|
// ... use queue ...
|
|
230
|
-
|
|
230
|
+
q.remove_shm(); // Coordinator, once no peer will attach again; q's mapping stays valid
|
|
231
|
+
} // Destructor unmaps; it never removes the segment
|
|
232
|
+
|
|
233
|
+
slick::shm::shared_memory::remove("name"); // By name, without a queue - e.g. at startup
|
|
231
234
|
```
|
|
232
235
|
|
|
233
236
|
**Python:**
|
|
234
237
|
```python
|
|
235
238
|
# Option 1: Manual cleanup
|
|
236
|
-
q = SlickQueue(name='name', size=size, element_size=elem_size
|
|
239
|
+
q = SlickQueue(name='name', size=size, element_size=elem_size)
|
|
237
240
|
try:
|
|
238
241
|
# ... use queue ...
|
|
239
242
|
finally:
|
|
240
243
|
q.close()
|
|
241
|
-
q.unlink() # Only
|
|
244
|
+
q.unlink() # Only from the process that coordinates the queue's lifetime
|
|
242
245
|
|
|
243
|
-
# Option 2: Context manager
|
|
244
|
-
with SlickQueue(name='name', size=size, element_size=elem_size
|
|
246
|
+
# Option 2: Context manager - closes, never unlinks
|
|
247
|
+
with SlickQueue(name='name', size=size, element_size=elem_size) as q:
|
|
245
248
|
# ... use queue ...
|
|
246
|
-
pass
|
|
249
|
+
pass
|
|
250
|
+
|
|
251
|
+
SlickQueue.remove('name') # By name, without attaching - e.g. at startup
|
|
247
252
|
```
|
|
248
253
|
|
|
249
254
|
**Rationale:** Python's garbage collection is non-deterministic, so explicit cleanup or context managers are needed.
|
|
250
255
|
|
|
256
|
+
Neither side deletes a segment on its own, matching C++ v2.3.0: not the C++ destructor, and not
|
|
257
|
+
Python's `close()`, context manager or process exit. Python keeps its segments out of
|
|
258
|
+
`multiprocessing`'s resource tracker for this, since the tracker would otherwise unlink them when
|
|
259
|
+
the process exits.
|
|
260
|
+
|
|
261
|
+
| C++ | Python |
|
|
262
|
+
|-----|--------|
|
|
263
|
+
| `remove_shm()` | `remove_shm()` - same contract, unlinks by name on every call |
|
|
264
|
+
| - | `unlink()` - unlinks once, valid after `close()`; a repeat is a no-op |
|
|
265
|
+
| `slick::shm::shared_memory::remove(name)` | `SlickQueue.remove(name)` - also accepts a `/`-prefixed name |
|
|
266
|
+
| `shm_name()` - `""` for a local queue | `get_shm_name()` - `None` for a local queue, `/`-prefixed on POSIX |
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
### 8. SpscQueue: Held Slot vs. Copy
|
|
271
|
+
|
|
272
|
+
**C++:**
|
|
273
|
+
```cpp
|
|
274
|
+
slick::spsc_queue<uint64_t> q("ticks");
|
|
275
|
+
if (uint64_t* v = q.read()) { /* slot held until the next read(), read_last() or release() */ }
|
|
276
|
+
uint64_t dropped = 0;
|
|
277
|
+
uint64_t* last = q.read_last(dropped);
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
**Python:**
|
|
281
|
+
```python
|
|
282
|
+
q = SpscQueue(name='ticks', element_size=8)
|
|
283
|
+
data = q.read() # bytes copy; the slot is already back with the producer
|
|
284
|
+
data, dropped = q.read_last()
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
**Rationale:** Python copies the element anyway, so once `read()` returns there is nothing
|
|
288
|
+
left to hold, and it hands the slot back at once. A Python consumer therefore gets the full
|
|
289
|
+
`size` as capacity, even at size 1, and has no `release()`. A C++ consumer that exits while
|
|
290
|
+
holding a slot still has that element delivered again to whoever attaches next, Python
|
|
291
|
+
included.
|
|
292
|
+
|
|
293
|
+
| C++ | Python |
|
|
294
|
+
|-----|--------|
|
|
295
|
+
| `try_reserve()` returns `invalid_index` | Same, as `SpscQueue.invalid_index` |
|
|
296
|
+
| `read()` returns `T*`, `nullptr` when empty | `bytes`, `None` when empty |
|
|
297
|
+
| `read_last(dropped)` with an out-parameter | `read_last()` returns `(data, dropped)` |
|
|
298
|
+
| `release()` | Not needed: nothing is held |
|
|
299
|
+
| `spsc_queue(name)` adopts `size`, and `T` gives the element size | `SpscQueue(name=..., element_size=...)` adopts `size`; `element_size` is always required |
|
|
300
|
+
|
|
251
301
|
---
|
|
252
302
|
|
|
253
303
|
## Interoperability Examples
|
|
@@ -339,11 +389,14 @@ int main() {
|
|
|
339
389
|
| Element access | Pointer | `memoryview` | Python memory safety |
|
|
340
390
|
| Type safety | Templates | `struct` module | Language difference |
|
|
341
391
|
| Cleanup | RAII (destructor) | Explicit or context manager | Python GC is non-deterministic |
|
|
392
|
+
| Segment removal | `remove_shm()` | `remove_shm()`, `unlink()`, `SlickQueue.remove()` | Neither side unlinks on its own |
|
|
342
393
|
| `read()` data | `T*` into the ring | `bytes` copy | No safe raw pointers in Python |
|
|
343
394
|
| Overwritten-in-flight data | Caller sees a volatile pointer | Copy may hold a newer record's bytes | Detect with `loss_count()` |
|
|
344
395
|
| Feature configuration | `Traits` template parameter | `traits=` argument | Python has no templates |
|
|
345
396
|
| Misconfigured traits | Compile error (concept / `static_assert`) | `TypeError` / `RuntimeError` at runtime | No compile step |
|
|
346
397
|
| Default loss detection | Off in Release | Always on | +0.2% here, and the only overrun signal a caller has |
|
|
347
398
|
| Layout marker | `'SLQ1'` / `'SLQ0'` | Identical, and validated against C++ peers | Shared contract |
|
|
399
|
+
| SPSC `read()` | `T*` held until the next read | `bytes` copy, slot freed at once | Python copies anyway |
|
|
400
|
+
| SPSC `read_last()` dropped count | Out-parameter | Second item of the returned tuple | Python has no pass-by-reference |
|
|
348
401
|
|
|
349
402
|
**Bottom Line:** The Python API is designed to be Pythonic while maintaining **100% binary compatibility** with C++. All atomic operations and memory layouts are identical, enabling seamless C++/Python interoperability.
|
|
@@ -1,5 +1,133 @@
|
|
|
1
1
|
# Changelogs
|
|
2
2
|
|
|
3
|
+
## [v2.4.0] - 2026-11-09
|
|
4
|
+
|
|
5
|
+
Tracks C++ slick-queue v2.4.0, which added `slick::spsc_queue`. The `SlickQueue`
|
|
6
|
+
shared-memory layout is unchanged, so 2.3.0 peers in either language interoperate
|
|
7
|
+
with it as before.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `SpscQueue`, a port of C++ `slick::spsc_queue<T>`: a lossless single-producer
|
|
12
|
+
single-consumer ring in local or shared memory. `try_reserve(n)` returns
|
|
13
|
+
`SpscQueue.invalid_index` when the ring is full and `reserve(n)` waits until the
|
|
14
|
+
consumer frees room. `publish()` never moves the tail backwards, so publishing an
|
|
15
|
+
earlier reservation after a later one is a no-op.
|
|
16
|
+
- `read()` returns the next element as `bytes`. `read_last()` returns
|
|
17
|
+
`(data, dropped)`, the newest element and how many unread ones it skipped, from
|
|
18
|
+
one load of the producer's tail however large the backlog.
|
|
19
|
+
- Both reads copy the element and hand its slot back before they return. C++ holds
|
|
20
|
+
the slot until the consumer's next call, so a Python consumer gets the full `size`
|
|
21
|
+
as capacity, nothing it read is delivered again after it reattaches, and there is
|
|
22
|
+
no `release()`.
|
|
23
|
+
- `SpscQueueTraits` with one setting, `enable_cpu_relax`, which yields the GIL while
|
|
24
|
+
`reserve()` waits. `q.traits` is a read-only snapshot, as for `SlickQueue`.
|
|
25
|
+
- The C++ shared-memory layout: a 384-byte header with the tail at offset 128 and the
|
|
26
|
+
head at 256, then the data array. Size, element size, the `'SLS0'` marker and init
|
|
27
|
+
state keep the `SlickQueue` offsets. Both cursors live in the segment, so a
|
|
28
|
+
reattached peer resumes where the previous one stopped.
|
|
29
|
+
- A creator that claims a segment left uninitialized, whose own creator died before
|
|
30
|
+
claiming it, checks the mapping holds its geometry first. If it doesn't, it raises
|
|
31
|
+
and hands the claim back, instead of writing a header for a ring the mapping can't
|
|
32
|
+
hold. Every attacher checks the mapping size too.
|
|
33
|
+
- Using an `SpscQueue` after `close()` raises `RuntimeError("queue is closed")`.
|
|
34
|
+
|
|
35
|
+
### Changed
|
|
36
|
+
|
|
37
|
+
- `SlickQueue` attaching to an `SpscQueue` segment now raises "holds a
|
|
38
|
+
slick::spsc_queue" instead of the generic missing-marker error, matching C++, and
|
|
39
|
+
`SpscQueue` names a `SlickQueue` segment the same way.
|
|
40
|
+
- Name handling, opening and closing the segment, `unlink()`, `remove_shm()`,
|
|
41
|
+
`remove()` and the context manager moved into a base class both queues share.
|
|
42
|
+
`SlickQueue` behaves as before.
|
|
43
|
+
- `close()` still never raises, but no longer prints the exception it swallows.
|
|
44
|
+
|
|
45
|
+
### Tests
|
|
46
|
+
|
|
47
|
+
- `tests/test_spsc.py` covers the local and shared-memory API: wraps, a full ring,
|
|
48
|
+
out-of-order publish, element sizes other than 8, `read_last()` accounting under a
|
|
49
|
+
concurrent producer, reset, reattach, a too-small leftover segment, use after
|
|
50
|
+
`close()` and the cross-type marker checks.
|
|
51
|
+
- `tests/test_spsc_interop.py` runs against C++ through the new `cpp_spsc_tester`:
|
|
52
|
+
data in both directions, attaching to a C++-created segment, `read_last()` from
|
|
53
|
+
each side, and each language rejecting the other queue type.
|
|
54
|
+
|
|
55
|
+
Checked against C++ slick-queue v2.4.0 with the interop tests on Windows.
|
|
56
|
+
|
|
57
|
+
## [v2.3.0] - 2026-10-17
|
|
58
|
+
|
|
59
|
+
Tracks C++ slick-queue v2.3.0: no queue removes its shared-memory segment on its
|
|
60
|
+
own any more, and `remove_shm()` is how the coordinating process does. The
|
|
61
|
+
version moves from 2.1.0 to 2.3.0 to stay on the C++ number. C++ v2.2.0 added
|
|
62
|
+
only `shm_name()`, which Python already had as `get_shm_name()`. The
|
|
63
|
+
shared-memory layout is unchanged, so 2.1.0 peers in either language
|
|
64
|
+
interoperate as before.
|
|
65
|
+
|
|
66
|
+
### Segment lifetime (behavior change)
|
|
67
|
+
|
|
68
|
+
- On POSIX, a segment is no longer unlinked when the process that created it
|
|
69
|
+
exits. `SharedMemory` registered every segment with `multiprocessing`'s
|
|
70
|
+
resource tracker, which unlinks what is registered when the process exits, so
|
|
71
|
+
a creator that exited first stranded every peer still attached on an orphaned
|
|
72
|
+
mapping, while its restart silently created a fresh segment under the same
|
|
73
|
+
name that those peers never saw. C++ v2.3.0 stopped its destructor doing the
|
|
74
|
+
same. A name now lives until `unlink()`, `remove_shm()` or
|
|
75
|
+
`SlickQueue.remove(name)`, and a restarted process reattaches to the existing
|
|
76
|
+
segment and keeps its contents. Windows is unaffected: a section dies with its
|
|
77
|
+
last handle.
|
|
78
|
+
- The queue sends the resource tracker nothing at all: `track=False` on Python
|
|
79
|
+
3.13+, and on older POSIX Pythons a `SharedMemory` subclass that maps the
|
|
80
|
+
segment without registering it. With no register/unregister pair there is
|
|
81
|
+
nothing to race between threads, or between `multiprocessing` children, which
|
|
82
|
+
share their parent's tracker.
|
|
83
|
+
- Migration: code that relied on the cleanup at exit now leaves segments in
|
|
84
|
+
`/dev/shm`, and its next run attaches to them - a creator joins the existing
|
|
85
|
+
segment instead of initializing it, the contents are the old run's, and a
|
|
86
|
+
creator with a different geometry raises `ValueError`. Call `SlickQueue.remove(name)` at startup for a clean start, or
|
|
87
|
+
`unlink()` / `remove_shm()` at shutdown once no peer will attach again.
|
|
88
|
+
|
|
89
|
+
### Added
|
|
90
|
+
|
|
91
|
+
- `remove_shm() -> bool`, matching C++: unlinks the segment's name so the next
|
|
92
|
+
queue created with it starts fresh, leaving open mappings - this queue's own
|
|
93
|
+
included - valid. It unlinks by name on every call, and leaves `unlink()`
|
|
94
|
+
nothing to do afterwards, so a later `unlink()` cannot delete a replacement
|
|
95
|
+
created under the name. Returns `False` for a local-memory queue or when the
|
|
96
|
+
name is already gone, and always `True` for a shared queue on Windows.
|
|
97
|
+
- `SlickQueue.remove(name) -> bool`, matching C++
|
|
98
|
+
`slick::shm::shared_memory::remove()`: unlinks a segment by name without
|
|
99
|
+
attaching to it. Accepts the `/`-prefixed value of `get_shm_name()`, and is
|
|
100
|
+
always `True` on Windows.
|
|
101
|
+
|
|
102
|
+
### Fixed
|
|
103
|
+
|
|
104
|
+
- `unlink()` was a silent no-op once `close()` had run, because `close()` drops
|
|
105
|
+
the `SharedMemory` object it went through. The documented `shared_memory`
|
|
106
|
+
idiom - close every handle, then unlink the name once - therefore leaked the
|
|
107
|
+
segment on POSIX. The name is now kept alongside the mapping, so `unlink()`
|
|
108
|
+
works after `close()`, and repeating it is a no-op, so it cannot delete a
|
|
109
|
+
segment a later run has created under the same name.
|
|
110
|
+
- Before Python 3.13 on POSIX, a process that only attached to a queue unlinked
|
|
111
|
+
its segment when it exited, because attaching registered the name with that
|
|
112
|
+
process's resource tracker too.
|
|
113
|
+
|
|
114
|
+
### Tests
|
|
115
|
+
|
|
116
|
+
- New `tests/test_segment_lifetime.py` with 12 tests, registered with CTest:
|
|
117
|
+
`unlink()` after `close()` and repeated, `close()` and the context manager
|
|
118
|
+
keeping the segment, `remove_shm()` on local and shared queues, `unlink()`
|
|
119
|
+
after `remove_shm()` sparing a replacement segment, `remove()` by name, a
|
|
120
|
+
creator's and an attacher's exit in a fresh interpreter, no resource-tracker
|
|
121
|
+
traffic at all, a child forked while another thread is constructing a queue,
|
|
122
|
+
and spawned children attaching at once.
|
|
123
|
+
- `test_traits.py` and `test_items_per_slot.py` open their hand-built raw
|
|
124
|
+
segments through the library's opener, so those handles and the queue no
|
|
125
|
+
longer both claim a name in the resource tracker.
|
|
126
|
+
|
|
127
|
+
Checked against C++ slick-queue v2.3.0 with the interop tests on Windows and on
|
|
128
|
+
Linux, run twice on Linux so a segment left behind would fail the second run,
|
|
129
|
+
and on Linux under Python 3.12 (the subclass) and 3.14 (`track=False`).
|
|
130
|
+
|
|
3
131
|
## [v2.1.0] - 2026-09-24
|
|
4
132
|
|
|
5
133
|
Tracks C++ slick-queue v2.1.0. Adds `items_per_slot`, so one control slot can
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: slick-queue-py
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: Lock-free MPMC queue with C++ interoperability via shared memory
|
|
5
5
|
Home-page: https://github.com/SlickQuant/slick-queue-py
|
|
6
6
|
Author: Slick Quant
|
|
@@ -36,7 +36,7 @@ Dynamic: requires-python
|
|
|
36
36
|
|
|
37
37
|
# slick-queue-py
|
|
38
38
|
|
|
39
|
-
Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue with C++ interoperability through shared memory.
|
|
39
|
+
Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue, plus a lossless single-producer single-consumer (SPSC) queue, with C++ interoperability through shared memory.
|
|
40
40
|
|
|
41
41
|
This is the Python binding for the [SlickQueue C++ library](https://github.com/SlickQuant/slick-queue). The Python implementation maintains exact binary compatibility with the C++ version, enabling seamless interprocess communication between Python and C++ applications.
|
|
42
42
|
|
|
@@ -50,6 +50,8 @@ This is the Python binding for the [SlickQueue C++ library](https://github.com/S
|
|
|
50
50
|
- **Local Memory Mode**: In-process queue using local memory (no shared memory overhead)
|
|
51
51
|
- **Shared Memory Mode**: Inter-process queue for interprocess communication
|
|
52
52
|
- **Lock-Free Multi-Producer Multi-Consumer**: True MPMC support using atomic operations
|
|
53
|
+
- **Lossless SPSC Queue**: `SpscQueue` ports C++ `slick::spsc_queue`. It never overwrites an
|
|
54
|
+
unread element and reports a full ring to the producer instead. See [SpscQueue](#spscqueue)
|
|
53
55
|
- **C++/Python Interoperability**: Python and C++ processes can share the same queue
|
|
54
56
|
- **Cross-Platform**: Windows and Linux/macOS support (x86-64)
|
|
55
57
|
- **Memory Layout Compatible**: Exact binary compatibility with C++ `slick::queue<T>`
|
|
@@ -600,11 +602,92 @@ Get memoryview for writing to reserved slot.
|
|
|
600
602
|
|
|
601
603
|
#### `close()`
|
|
602
604
|
|
|
603
|
-
Close the shared memory connection.
|
|
605
|
+
Close the shared memory connection. Never deletes the segment.
|
|
604
606
|
|
|
605
607
|
#### `unlink()`
|
|
606
608
|
|
|
607
|
-
Delete the shared memory segment
|
|
609
|
+
Delete the shared memory segment this queue is attached to. Valid before or after `close()`
|
|
610
|
+
(close every handle, then unlink once), and a no-op when repeated, so it cannot delete a segment
|
|
611
|
+
a later run has since created under the same name. A no-op for local memory mode.
|
|
612
|
+
|
|
613
|
+
#### `remove_shm() -> bool`
|
|
614
|
+
|
|
615
|
+
Unlink the segment's name so the next queue created with it starts fresh, matching C++
|
|
616
|
+
`remove_shm()`. Mappings already open, including this queue's own, stay valid. Unlike `unlink()`
|
|
617
|
+
it unlinks by name on every call. Returns `False` for a local-memory queue or if the name is
|
|
618
|
+
already gone, and always `True` for a shared queue on Windows.
|
|
619
|
+
|
|
620
|
+
#### `SlickQueue.remove(name) -> bool`
|
|
621
|
+
|
|
622
|
+
Unlink a segment by name without attaching to it, matching C++
|
|
623
|
+
`slick::shm::shared_memory::remove()`. Typically called at startup to clear a segment a previous
|
|
624
|
+
run left behind. Accepts the `/`-prefixed value of `get_shm_name()`. Always `True` on Windows.
|
|
625
|
+
|
|
626
|
+
#### `get_shm_name() -> Optional[str]`
|
|
627
|
+
|
|
628
|
+
The segment name to pass to C++ (`/`-prefixed on POSIX), or `None` for a local-memory queue.
|
|
629
|
+
|
|
630
|
+
#### Segment lifetime
|
|
631
|
+
|
|
632
|
+
No queue deletes its segment on its own: not `close()`, not the context manager, and not process
|
|
633
|
+
exit, matching C++ slick-queue v2.3.0. On POSIX a name outlives every mapping until it is
|
|
634
|
+
unlinked, so a departing process that unlinked it while a peer was still attached would leave that
|
|
635
|
+
peer reading an orphaned mapping while a restarted process silently created a fresh segment under
|
|
636
|
+
the same name. The process that coordinates the queue's lifetime calls `unlink()`, `remove_shm()`
|
|
637
|
+
or `SlickQueue.remove(name)` once no peer will attach again; until then a restarted process
|
|
638
|
+
reattaches to the existing segment and keeps its contents. On Windows a segment is freed when its
|
|
639
|
+
last handle closes.
|
|
640
|
+
|
|
641
|
+
The queue keeps its segments out of `multiprocessing`'s resource tracker, which would otherwise
|
|
642
|
+
unlink them when the process exits - before Python 3.13, even in a process that only attached. It
|
|
643
|
+
sends the tracker nothing at all: `track=False` on Python 3.13+, and on older POSIX Pythons a
|
|
644
|
+
`SharedMemory` subclass that maps the segment without registering it, so there is no
|
|
645
|
+
register/unregister pair to race between threads or between processes sharing one tracker.
|
|
646
|
+
|
|
647
|
+
### SpscQueue
|
|
648
|
+
|
|
649
|
+
A lossless single-producer single-consumer ring, compatible with C++ `slick::spsc_queue<T>`. One
|
|
650
|
+
thread or process produces and one consumes. Unlike `SlickQueue` it never overwrites an unread
|
|
651
|
+
element: `try_reserve()` reports a full ring instead.
|
|
652
|
+
|
|
653
|
+
```python
|
|
654
|
+
import struct
|
|
655
|
+
from slick_queue_py import SpscQueue
|
|
656
|
+
|
|
657
|
+
q = SpscQueue(name='ticks', size=1024, element_size=8) # create, or open with the same geometry
|
|
658
|
+
peer = SpscQueue(name='ticks', element_size=8) # attach, adopting the segment's size
|
|
659
|
+
local = SpscQueue(size=1024, element_size=8) # local memory
|
|
660
|
+
|
|
661
|
+
# Producer
|
|
662
|
+
i = q.try_reserve() # SpscQueue.invalid_index when full; reserve() waits instead
|
|
663
|
+
if i != SpscQueue.invalid_index:
|
|
664
|
+
q[i][:8] = struct.pack('<Q', 42)
|
|
665
|
+
q.publish(i)
|
|
666
|
+
|
|
667
|
+
# Consumer
|
|
668
|
+
data = peer.read() # next element as bytes, or None
|
|
669
|
+
data, dropped = peer.read_last() # newest element, and how many unread ones it skipped
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
- `reserve(n)` and `try_reserve(n)` reserve `n` separate elements. Write element `k` through
|
|
673
|
+
`q[i + k]`, because a batch may wrap around the end of the ring. Publish in reservation order.
|
|
674
|
+
Publishing a later batch first also exposes the earlier one, and publishing the earlier one
|
|
675
|
+
afterwards is a no-op.
|
|
676
|
+
- `read()` and `read_last()` copy the element and hand its slot back before they return. C++ holds
|
|
677
|
+
the slot until the consumer's next call, so here the full `size` is always usable and there is no
|
|
678
|
+
`release()`.
|
|
679
|
+
- `read_last()` loads the producer's tail and moves the read cursor there, so its cost doesn't grow
|
|
680
|
+
with the backlog. It returns `(None, 0)` if nothing was published since the last read. Note
|
|
681
|
+
that the second item counts skipped elements, where `SlickQueue.read_last()` returns a size.
|
|
682
|
+
- Both cursors live in the segment, and a peer that reattaches resumes where the previous one
|
|
683
|
+
stopped. Each object caches its own side's cursor, so keep one live object per side. After
|
|
684
|
+
another process has produced or consumed, attach a fresh object rather than reusing an old one.
|
|
685
|
+
- `reset()` empties the queue. It isn't thread-safe, and a peer in another process must reattach.
|
|
686
|
+
- `close()`, `unlink()`, `remove_shm()`, `SpscQueue.remove(name)`, `get_shm_name()` and the
|
|
687
|
+
context manager behave as they do for `SlickQueue`. Using the queue after `close()` raises
|
|
688
|
+
`RuntimeError("queue is closed")`.
|
|
689
|
+
- `traits=` takes an `SpscQueueTraits` subclass. Its one setting is `enable_cpu_relax` (default
|
|
690
|
+
`True`), which yields the GIL while `reserve()` waits. `q.traits` is a read-only snapshot.
|
|
608
691
|
|
|
609
692
|
### AtomicCursor
|
|
610
693
|
|
|
@@ -781,6 +864,34 @@ Bits 2-3 of the nibble are reserved and must be 0; a marker that sets one is rej
|
|
|
781
864
|
newer than this build understands. Segments created before slick-queue v1.4.0 carry no
|
|
782
865
|
marker at all and are rejected at attach time.
|
|
783
866
|
|
|
867
|
+
### SpscQueue layout
|
|
868
|
+
|
|
869
|
+
`SpscQueue` matches C++ `slick::spsc_queue<T>`. Each cursor sits on its own 128-byte line on every
|
|
870
|
+
platform:
|
|
871
|
+
|
|
872
|
+
```
|
|
873
|
+
Offset | Size | Content
|
|
874
|
+
-------|---------------|------------------
|
|
875
|
+
0 | 8 bytes | reserved (0)
|
|
876
|
+
8 | 4 bytes | uint32_t size (queue capacity)
|
|
877
|
+
12 | 4 bytes | uint32_t element_size
|
|
878
|
+
16 | 8 bytes | reserved (0)
|
|
879
|
+
24 | 4 bytes | uint32_t header_magic - 'SLS0' (0x534C5330)
|
|
880
|
+
28 | 20 bytes | reserved (0)
|
|
881
|
+
48 | 4 bytes | uint32_t init_state (atomic)
|
|
882
|
+
52 | 76 bytes | padding (to 128)
|
|
883
|
+
128 | 8 bytes | uint64_t tail (atomic), written by the producer
|
|
884
|
+
136 | 120 bytes | padding
|
|
885
|
+
256 | 8 bytes | uint64_t head (atomic), written by the consumer
|
|
886
|
+
264 | 120 bytes | padding (to 384)
|
|
887
|
+
384 | elem*size | data array
|
|
888
|
+
```
|
|
889
|
+
|
|
890
|
+
`size`, `element_size`, the marker and `init_state` sit at the same offsets as in a `SlickQueue`
|
|
891
|
+
segment. Each queue type therefore turns the other's segment away at attach time, with *"holds a
|
|
892
|
+
slick::spsc_queue"* or *"holds a slick::queue"*, instead of timing out. The marker's low nibble is
|
|
893
|
+
reserved for future layout features and must be 0. The C++ side requires `alignof(T) <= 128`.
|
|
894
|
+
|
|
784
895
|
## Platform Support
|
|
785
896
|
|
|
786
897
|
### Fully Supported (Lock-Free)
|
|
@@ -920,6 +1031,12 @@ python tests/test_reset_detection.py
|
|
|
920
1031
|
|
|
921
1032
|
# Wrapping-producer record/size invariants
|
|
922
1033
|
python tests/test_wrap_invariants.py
|
|
1034
|
+
|
|
1035
|
+
# Segment lifetime: unlink(), remove_shm(), remove(), process exit
|
|
1036
|
+
python tests/test_segment_lifetime.py
|
|
1037
|
+
|
|
1038
|
+
# SpscQueue, local and shared memory
|
|
1039
|
+
python tests/test_spsc.py
|
|
923
1040
|
```
|
|
924
1041
|
|
|
925
1042
|
Or run everything with pytest:
|
|
@@ -948,6 +1065,9 @@ python tests/test_interop.py --test cpp_producer_python_consumer
|
|
|
948
1065
|
python tests/test_interop.py --test multi_producer_interop
|
|
949
1066
|
python tests/test_interop.py --test stress_interop
|
|
950
1067
|
python tests/test_interop.py --test cpp_shm_creation
|
|
1068
|
+
|
|
1069
|
+
# SpscQueue against C++ slick::spsc_queue (uses cpp_spsc_tester)
|
|
1070
|
+
python tests/test_spsc_interop.py
|
|
951
1071
|
```
|
|
952
1072
|
|
|
953
1073
|
The interop tests verify:
|
|
@@ -963,7 +1083,7 @@ The interop tests verify:
|
|
|
963
1083
|
|
|
964
1084
|
1. **Buffer Cleanup Warning**: You may see a `BufferError: cannot close exported pointers exist` warning during garbage collection. This is a **harmless warning** caused by Python's ctypes creating internal buffer references that persist beyond explicit cleanup. It occurs during program exit and **does not affect functionality, performance, or correctness**. The queue works perfectly despite this warning.
|
|
965
1085
|
|
|
966
|
-
2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`.
|
|
1086
|
+
2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`. `SlickQueue` keeps its own segments out of the resource tracker (see [Segment lifetime](#segment-lifetime)), so the warning comes from a `multiprocessing.shared_memory.SharedMemory` your code opened directly, such as one holding an `AtomicCursor`. The tracker unlinks those segments at exit; pass `track=False` (Python 3.13+) to keep them.
|
|
967
1087
|
|
|
968
1088
|
## Architecture
|
|
969
1089
|
|
|
@@ -1013,6 +1133,7 @@ queue so consumers keep up, and use `loss_count()` to detect when they have not.
|
|
|
1013
1133
|
| read(atomic cursor) multi-consumer | ✅ | ✅ |
|
|
1014
1134
|
| Feature traits | Template parameter | `traits=` argument |
|
|
1015
1135
|
| Shared layout marker | `'SLQ1'` / `'SLQ0'` | Same, and validated against C++ peers |
|
|
1136
|
+
| Lossless SPSC queue | `slick::spsc_queue<T>` | `SpscQueue`, same `'SLS0'` layout |
|
|
1016
1137
|
|
|
1017
1138
|
## Contributing
|
|
1018
1139
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# slick-queue-py
|
|
2
2
|
|
|
3
|
-
Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue with C++ interoperability through shared memory.
|
|
3
|
+
Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue, plus a lossless single-producer single-consumer (SPSC) queue, with C++ interoperability through shared memory.
|
|
4
4
|
|
|
5
5
|
This is the Python binding for the [SlickQueue C++ library](https://github.com/SlickQuant/slick-queue). The Python implementation maintains exact binary compatibility with the C++ version, enabling seamless interprocess communication between Python and C++ applications.
|
|
6
6
|
|
|
@@ -14,6 +14,8 @@ This is the Python binding for the [SlickQueue C++ library](https://github.com/S
|
|
|
14
14
|
- **Local Memory Mode**: In-process queue using local memory (no shared memory overhead)
|
|
15
15
|
- **Shared Memory Mode**: Inter-process queue for interprocess communication
|
|
16
16
|
- **Lock-Free Multi-Producer Multi-Consumer**: True MPMC support using atomic operations
|
|
17
|
+
- **Lossless SPSC Queue**: `SpscQueue` ports C++ `slick::spsc_queue`. It never overwrites an
|
|
18
|
+
unread element and reports a full ring to the producer instead. See [SpscQueue](#spscqueue)
|
|
17
19
|
- **C++/Python Interoperability**: Python and C++ processes can share the same queue
|
|
18
20
|
- **Cross-Platform**: Windows and Linux/macOS support (x86-64)
|
|
19
21
|
- **Memory Layout Compatible**: Exact binary compatibility with C++ `slick::queue<T>`
|
|
@@ -564,11 +566,92 @@ Get memoryview for writing to reserved slot.
|
|
|
564
566
|
|
|
565
567
|
#### `close()`
|
|
566
568
|
|
|
567
|
-
Close the shared memory connection.
|
|
569
|
+
Close the shared memory connection. Never deletes the segment.
|
|
568
570
|
|
|
569
571
|
#### `unlink()`
|
|
570
572
|
|
|
571
|
-
Delete the shared memory segment
|
|
573
|
+
Delete the shared memory segment this queue is attached to. Valid before or after `close()`
|
|
574
|
+
(close every handle, then unlink once), and a no-op when repeated, so it cannot delete a segment
|
|
575
|
+
a later run has since created under the same name. A no-op for local memory mode.
|
|
576
|
+
|
|
577
|
+
#### `remove_shm() -> bool`
|
|
578
|
+
|
|
579
|
+
Unlink the segment's name so the next queue created with it starts fresh, matching C++
|
|
580
|
+
`remove_shm()`. Mappings already open, including this queue's own, stay valid. Unlike `unlink()`
|
|
581
|
+
it unlinks by name on every call. Returns `False` for a local-memory queue or if the name is
|
|
582
|
+
already gone, and always `True` for a shared queue on Windows.
|
|
583
|
+
|
|
584
|
+
#### `SlickQueue.remove(name) -> bool`
|
|
585
|
+
|
|
586
|
+
Unlink a segment by name without attaching to it, matching C++
|
|
587
|
+
`slick::shm::shared_memory::remove()`. Typically called at startup to clear a segment a previous
|
|
588
|
+
run left behind. Accepts the `/`-prefixed value of `get_shm_name()`. Always `True` on Windows.
|
|
589
|
+
|
|
590
|
+
#### `get_shm_name() -> Optional[str]`
|
|
591
|
+
|
|
592
|
+
The segment name to pass to C++ (`/`-prefixed on POSIX), or `None` for a local-memory queue.
|
|
593
|
+
|
|
594
|
+
#### Segment lifetime
|
|
595
|
+
|
|
596
|
+
No queue deletes its segment on its own: not `close()`, not the context manager, and not process
|
|
597
|
+
exit, matching C++ slick-queue v2.3.0. On POSIX a name outlives every mapping until it is
|
|
598
|
+
unlinked, so a departing process that unlinked it while a peer was still attached would leave that
|
|
599
|
+
peer reading an orphaned mapping while a restarted process silently created a fresh segment under
|
|
600
|
+
the same name. The process that coordinates the queue's lifetime calls `unlink()`, `remove_shm()`
|
|
601
|
+
or `SlickQueue.remove(name)` once no peer will attach again; until then a restarted process
|
|
602
|
+
reattaches to the existing segment and keeps its contents. On Windows a segment is freed when its
|
|
603
|
+
last handle closes.
|
|
604
|
+
|
|
605
|
+
The queue keeps its segments out of `multiprocessing`'s resource tracker, which would otherwise
|
|
606
|
+
unlink them when the process exits - before Python 3.13, even in a process that only attached. It
|
|
607
|
+
sends the tracker nothing at all: `track=False` on Python 3.13+, and on older POSIX Pythons a
|
|
608
|
+
`SharedMemory` subclass that maps the segment without registering it, so there is no
|
|
609
|
+
register/unregister pair to race between threads or between processes sharing one tracker.
|
|
610
|
+
|
|
611
|
+
### SpscQueue
|
|
612
|
+
|
|
613
|
+
A lossless single-producer single-consumer ring, compatible with C++ `slick::spsc_queue<T>`. One
|
|
614
|
+
thread or process produces and one consumes. Unlike `SlickQueue` it never overwrites an unread
|
|
615
|
+
element: `try_reserve()` reports a full ring instead.
|
|
616
|
+
|
|
617
|
+
```python
|
|
618
|
+
import struct
|
|
619
|
+
from slick_queue_py import SpscQueue
|
|
620
|
+
|
|
621
|
+
q = SpscQueue(name='ticks', size=1024, element_size=8) # create, or open with the same geometry
|
|
622
|
+
peer = SpscQueue(name='ticks', element_size=8) # attach, adopting the segment's size
|
|
623
|
+
local = SpscQueue(size=1024, element_size=8) # local memory
|
|
624
|
+
|
|
625
|
+
# Producer
|
|
626
|
+
i = q.try_reserve() # SpscQueue.invalid_index when full; reserve() waits instead
|
|
627
|
+
if i != SpscQueue.invalid_index:
|
|
628
|
+
q[i][:8] = struct.pack('<Q', 42)
|
|
629
|
+
q.publish(i)
|
|
630
|
+
|
|
631
|
+
# Consumer
|
|
632
|
+
data = peer.read() # next element as bytes, or None
|
|
633
|
+
data, dropped = peer.read_last() # newest element, and how many unread ones it skipped
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
- `reserve(n)` and `try_reserve(n)` reserve `n` separate elements. Write element `k` through
|
|
637
|
+
`q[i + k]`, because a batch may wrap around the end of the ring. Publish in reservation order.
|
|
638
|
+
Publishing a later batch first also exposes the earlier one, and publishing the earlier one
|
|
639
|
+
afterwards is a no-op.
|
|
640
|
+
- `read()` and `read_last()` copy the element and hand its slot back before they return. C++ holds
|
|
641
|
+
the slot until the consumer's next call, so here the full `size` is always usable and there is no
|
|
642
|
+
`release()`.
|
|
643
|
+
- `read_last()` loads the producer's tail and moves the read cursor there, so its cost doesn't grow
|
|
644
|
+
with the backlog. It returns `(None, 0)` if nothing was published since the last read. Note
|
|
645
|
+
that the second item counts skipped elements, where `SlickQueue.read_last()` returns a size.
|
|
646
|
+
- Both cursors live in the segment, and a peer that reattaches resumes where the previous one
|
|
647
|
+
stopped. Each object caches its own side's cursor, so keep one live object per side. After
|
|
648
|
+
another process has produced or consumed, attach a fresh object rather than reusing an old one.
|
|
649
|
+
- `reset()` empties the queue. It isn't thread-safe, and a peer in another process must reattach.
|
|
650
|
+
- `close()`, `unlink()`, `remove_shm()`, `SpscQueue.remove(name)`, `get_shm_name()` and the
|
|
651
|
+
context manager behave as they do for `SlickQueue`. Using the queue after `close()` raises
|
|
652
|
+
`RuntimeError("queue is closed")`.
|
|
653
|
+
- `traits=` takes an `SpscQueueTraits` subclass. Its one setting is `enable_cpu_relax` (default
|
|
654
|
+
`True`), which yields the GIL while `reserve()` waits. `q.traits` is a read-only snapshot.
|
|
572
655
|
|
|
573
656
|
### AtomicCursor
|
|
574
657
|
|
|
@@ -745,6 +828,34 @@ Bits 2-3 of the nibble are reserved and must be 0; a marker that sets one is rej
|
|
|
745
828
|
newer than this build understands. Segments created before slick-queue v1.4.0 carry no
|
|
746
829
|
marker at all and are rejected at attach time.
|
|
747
830
|
|
|
831
|
+
### SpscQueue layout
|
|
832
|
+
|
|
833
|
+
`SpscQueue` matches C++ `slick::spsc_queue<T>`. Each cursor sits on its own 128-byte line on every
|
|
834
|
+
platform:
|
|
835
|
+
|
|
836
|
+
```
|
|
837
|
+
Offset | Size | Content
|
|
838
|
+
-------|---------------|------------------
|
|
839
|
+
0 | 8 bytes | reserved (0)
|
|
840
|
+
8 | 4 bytes | uint32_t size (queue capacity)
|
|
841
|
+
12 | 4 bytes | uint32_t element_size
|
|
842
|
+
16 | 8 bytes | reserved (0)
|
|
843
|
+
24 | 4 bytes | uint32_t header_magic - 'SLS0' (0x534C5330)
|
|
844
|
+
28 | 20 bytes | reserved (0)
|
|
845
|
+
48 | 4 bytes | uint32_t init_state (atomic)
|
|
846
|
+
52 | 76 bytes | padding (to 128)
|
|
847
|
+
128 | 8 bytes | uint64_t tail (atomic), written by the producer
|
|
848
|
+
136 | 120 bytes | padding
|
|
849
|
+
256 | 8 bytes | uint64_t head (atomic), written by the consumer
|
|
850
|
+
264 | 120 bytes | padding (to 384)
|
|
851
|
+
384 | elem*size | data array
|
|
852
|
+
```
|
|
853
|
+
|
|
854
|
+
`size`, `element_size`, the marker and `init_state` sit at the same offsets as in a `SlickQueue`
|
|
855
|
+
segment. Each queue type therefore turns the other's segment away at attach time, with *"holds a
|
|
856
|
+
slick::spsc_queue"* or *"holds a slick::queue"*, instead of timing out. The marker's low nibble is
|
|
857
|
+
reserved for future layout features and must be 0. The C++ side requires `alignof(T) <= 128`.
|
|
858
|
+
|
|
748
859
|
## Platform Support
|
|
749
860
|
|
|
750
861
|
### Fully Supported (Lock-Free)
|
|
@@ -884,6 +995,12 @@ python tests/test_reset_detection.py
|
|
|
884
995
|
|
|
885
996
|
# Wrapping-producer record/size invariants
|
|
886
997
|
python tests/test_wrap_invariants.py
|
|
998
|
+
|
|
999
|
+
# Segment lifetime: unlink(), remove_shm(), remove(), process exit
|
|
1000
|
+
python tests/test_segment_lifetime.py
|
|
1001
|
+
|
|
1002
|
+
# SpscQueue, local and shared memory
|
|
1003
|
+
python tests/test_spsc.py
|
|
887
1004
|
```
|
|
888
1005
|
|
|
889
1006
|
Or run everything with pytest:
|
|
@@ -912,6 +1029,9 @@ python tests/test_interop.py --test cpp_producer_python_consumer
|
|
|
912
1029
|
python tests/test_interop.py --test multi_producer_interop
|
|
913
1030
|
python tests/test_interop.py --test stress_interop
|
|
914
1031
|
python tests/test_interop.py --test cpp_shm_creation
|
|
1032
|
+
|
|
1033
|
+
# SpscQueue against C++ slick::spsc_queue (uses cpp_spsc_tester)
|
|
1034
|
+
python tests/test_spsc_interop.py
|
|
915
1035
|
```
|
|
916
1036
|
|
|
917
1037
|
The interop tests verify:
|
|
@@ -927,7 +1047,7 @@ The interop tests verify:
|
|
|
927
1047
|
|
|
928
1048
|
1. **Buffer Cleanup Warning**: You may see a `BufferError: cannot close exported pointers exist` warning during garbage collection. This is a **harmless warning** caused by Python's ctypes creating internal buffer references that persist beyond explicit cleanup. It occurs during program exit and **does not affect functionality, performance, or correctness**. The queue works perfectly despite this warning.
|
|
929
1049
|
|
|
930
|
-
2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`.
|
|
1050
|
+
2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`. `SlickQueue` keeps its own segments out of the resource tracker (see [Segment lifetime](#segment-lifetime)), so the warning comes from a `multiprocessing.shared_memory.SharedMemory` your code opened directly, such as one holding an `AtomicCursor`. The tracker unlinks those segments at exit; pass `track=False` (Python 3.13+) to keep them.
|
|
931
1051
|
|
|
932
1052
|
## Architecture
|
|
933
1053
|
|
|
@@ -977,6 +1097,7 @@ queue so consumers keep up, and use `loss_count()` to detect when they have not.
|
|
|
977
1097
|
| read(atomic cursor) multi-consumer | ✅ | ✅ |
|
|
978
1098
|
| Feature traits | Template parameter | `traits=` argument |
|
|
979
1099
|
| Shared layout marker | `'SLQ1'` / `'SLQ0'` | Same, and validated against C++ peers |
|
|
1100
|
+
| Lossless SPSC queue | `slick::spsc_queue<T>` | `SpscQueue`, same `'SLS0'` layout |
|
|
980
1101
|
|
|
981
1102
|
## Contributing
|
|
982
1103
|
|
|
@@ -25,6 +25,7 @@ tests/cpp_consumer.cpp
|
|
|
25
25
|
tests/cpp_multi_producer.cpp
|
|
26
26
|
tests/cpp_producer.cpp
|
|
27
27
|
tests/cpp_read_last_tester.cpp
|
|
28
|
+
tests/cpp_spsc_tester.cpp
|
|
28
29
|
tests/cpp_work_stealing_consumer.cpp
|
|
29
30
|
tests/run_test.py
|
|
30
31
|
tests/test_atomic_cursor.py
|
|
@@ -39,5 +40,8 @@ tests/test_queue.py
|
|
|
39
40
|
tests/test_read_last.py
|
|
40
41
|
tests/test_read_last_interop.py
|
|
41
42
|
tests/test_reset_detection.py
|
|
43
|
+
tests/test_segment_lifetime.py
|
|
44
|
+
tests/test_spsc.py
|
|
45
|
+
tests/test_spsc_interop.py
|
|
42
46
|
tests/test_traits.py
|
|
43
47
|
tests/test_wrap_invariants.py
|