slick-queue-py 2.1.0__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/API_DIFFERENCES.md +61 -8
  2. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/CHANGELOG.md +128 -0
  3. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/PKG-INFO +126 -5
  4. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/README.md +125 -4
  5. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/slick_queue_py.egg-info/SOURCES.txt +4 -0
  6. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/slick_queue_py.py +667 -151
  7. slick_queue_py-2.4.0/tests/cpp_spsc_tester.cpp +136 -0
  8. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_items_per_slot.py +3 -4
  9. slick_queue_py-2.4.0/tests/test_segment_lifetime.py +410 -0
  10. slick_queue_py-2.4.0/tests/test_spsc.py +715 -0
  11. slick_queue_py-2.4.0/tests/test_spsc_interop.py +138 -0
  12. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_traits.py +8 -12
  13. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/FUNDING.yml +0 -0
  14. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/ci.yml +0 -0
  15. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/publish-conda.yml +0 -0
  16. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/publish-pypi.yml +0 -0
  17. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/publish-release.yml +0 -0
  18. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.github/workflows/release.yml +0 -0
  19. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.gitignore +0 -0
  20. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/.pypirc.template +0 -0
  21. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/BUILDING.md +0 -0
  22. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/CMakeLists.txt +0 -0
  23. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/LICENSE +0 -0
  24. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/MANIFEST.in +0 -0
  25. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/atomic_ops.py +0 -0
  26. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/atomic_ops_ext.cpp +0 -0
  27. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/conda.recipe/meta.yaml +0 -0
  28. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/pyproject.toml +0 -0
  29. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/requirements.txt +0 -0
  30. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/setup.cfg +0 -0
  31. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/setup.py +0 -0
  32. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cleanup_shm.py +0 -0
  33. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_consumer.cpp +0 -0
  34. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_multi_producer.cpp +0 -0
  35. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_producer.cpp +0 -0
  36. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_read_last_tester.cpp +0 -0
  37. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/cpp_work_stealing_consumer.cpp +0 -0
  38. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/run_test.py +0 -0
  39. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_atomic_cursor.py +0 -0
  40. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_atomic_ops.py +0 -0
  41. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_interop.py +0 -0
  42. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_local_mode.py +0 -0
  43. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_loss_count.py +0 -0
  44. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_modern_format.py +0 -0
  45. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_multi_producer.py +0 -0
  46. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_queue.py +0 -0
  47. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_read_last.py +0 -0
  48. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_read_last_interop.py +0 -0
  49. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_reset_detection.py +0 -0
  50. {slick_queue_py-2.1.0 → slick_queue_py-2.4.0}/tests/test_wrap_invariants.py +0 -0
@@ -1,8 +1,8 @@
1
1
  # C++ / Python API Differences
2
2
 
3
- This document explains the intentional API differences between the C++ `slick::queue<T>` and Python `SlickQueue` implementations.
3
+ This document explains the intentional API differences between the C++ `slick::queue<T>` and Python `SlickQueue` implementations, and between C++ `slick::spsc_queue<T>` and Python `SpscQueue` (section 8).
4
4
 
5
- Both track the same version line; this document describes slick-queue v2.1.0.
5
+ Both track the same version line; this document describes slick-queue v2.4.0.
6
6
 
7
7
  ## Core Compatibility
8
8
 
@@ -227,27 +227,77 @@ q[idx][:4] = struct.pack("<i", 42) # Manual packing
227
227
  {
228
228
  slick::queue<T> q(size, "name");
229
229
  // ... use queue ...
230
- } // Destructor automatically cleans up
230
+ q.remove_shm(); // Coordinator, once no peer will attach again; q's mapping stays valid
231
+ } // Destructor unmaps; it never removes the segment
232
+
233
+ slick::shm::shared_memory::remove("name"); // By name, without a queue - e.g. at startup
231
234
  ```
232
235
 
233
236
  **Python:**
234
237
  ```python
235
238
  # Option 1: Manual cleanup
236
- q = SlickQueue(name='name', size=size, element_size=elem_size, create=True)
239
+ q = SlickQueue(name='name', size=size, element_size=elem_size)
237
240
  try:
238
241
  # ... use queue ...
239
242
  finally:
240
243
  q.close()
241
- q.unlink() # Only call from creator process
244
+ q.unlink() # Only from the process that coordinates the queue's lifetime
242
245
 
243
- # Option 2: Context manager (recommended)
244
- with SlickQueue(name='name', size=size, element_size=elem_size, create=True) as q:
246
+ # Option 2: Context manager - closes, never unlinks
247
+ with SlickQueue(name='name', size=size, element_size=elem_size) as q:
245
248
  # ... use queue ...
246
- pass # Automatic cleanup
249
+ pass
250
+
251
+ SlickQueue.remove('name') # By name, without attaching - e.g. at startup
247
252
  ```
248
253
 
249
254
  **Rationale:** Python's garbage collection is non-deterministic, so explicit cleanup or context managers are needed.
250
255
 
256
+ Neither side deletes a segment on its own, matching C++ v2.3.0: not the C++ destructor, and not
257
+ Python's `close()`, context manager or process exit. Python keeps its segments out of
258
+ `multiprocessing`'s resource tracker for this, since the tracker would otherwise unlink them when
259
+ the process exits.
260
+
261
+ | C++ | Python |
262
+ |-----|--------|
263
+ | `remove_shm()` | `remove_shm()` - same contract, unlinks by name on every call |
264
+ | - | `unlink()` - unlinks once, valid after `close()`; a repeat is a no-op |
265
+ | `slick::shm::shared_memory::remove(name)` | `SlickQueue.remove(name)` - also accepts a `/`-prefixed name |
266
+ | `shm_name()` - `""` for a local queue | `get_shm_name()` - `None` for a local queue, `/`-prefixed on POSIX |
267
+
268
+ ---
269
+
270
+ ### 8. SpscQueue: Held Slot vs. Copy
271
+
272
+ **C++:**
273
+ ```cpp
274
+ slick::spsc_queue<uint64_t> q("ticks");
275
+ if (uint64_t* v = q.read()) { /* slot held until the next read(), read_last() or release() */ }
276
+ uint64_t dropped = 0;
277
+ uint64_t* last = q.read_last(dropped);
278
+ ```
279
+
280
+ **Python:**
281
+ ```python
282
+ q = SpscQueue(name='ticks', element_size=8)
283
+ data = q.read() # bytes copy; the slot is already back with the producer
284
+ data, dropped = q.read_last()
285
+ ```
286
+
287
+ **Rationale:** Python copies the element anyway, so once `read()` returns there is nothing
288
+ left to hold, and it hands the slot back at once. A Python consumer therefore gets the full
289
+ `size` as capacity, even at size 1, and has no `release()`. A C++ consumer that exits while
290
+ holding a slot still has that element delivered again to whoever attaches next, Python
291
+ included.
292
+
293
+ | C++ | Python |
294
+ |-----|--------|
295
+ | `try_reserve()` returns `invalid_index` | Same, as `SpscQueue.invalid_index` |
296
+ | `read()` returns `T*`, `nullptr` when empty | `bytes`, `None` when empty |
297
+ | `read_last(dropped)` with an out-parameter | `read_last()` returns `(data, dropped)` |
298
+ | `release()` | Not needed: nothing is held |
299
+ | `spsc_queue(name)` adopts `size`, and `T` gives the element size | `SpscQueue(name=..., element_size=...)` adopts `size`; `element_size` is always required |
300
+
251
301
  ---
252
302
 
253
303
  ## Interoperability Examples
@@ -339,11 +389,14 @@ int main() {
339
389
  | Element access | Pointer | `memoryview` | Python memory safety |
340
390
  | Type safety | Templates | `struct` module | Language difference |
341
391
  | Cleanup | RAII (destructor) | Explicit or context manager | Python GC is non-deterministic |
392
+ | Segment removal | `remove_shm()` | `remove_shm()`, `unlink()`, `SlickQueue.remove()` | Neither side unlinks on its own |
342
393
  | `read()` data | `T*` into the ring | `bytes` copy | No safe raw pointers in Python |
343
394
  | Overwritten-in-flight data | Caller sees a volatile pointer | Copy may hold a newer record's bytes | Detect with `loss_count()` |
344
395
  | Feature configuration | `Traits` template parameter | `traits=` argument | Python has no templates |
345
396
  | Misconfigured traits | Compile error (concept / `static_assert`) | `TypeError` / `RuntimeError` at runtime | No compile step |
346
397
  | Default loss detection | Off in Release | Always on | +0.2% here, and the only overrun signal a caller has |
347
398
  | Layout marker | `'SLQ1'` / `'SLQ0'` | Identical, and validated against C++ peers | Shared contract |
399
+ | SPSC `read()` | `T*` held until the next read | `bytes` copy, slot freed at once | Python copies anyway |
400
+ | SPSC `read_last()` dropped count | Out-parameter | Second item of the returned tuple | Python has no pass-by-reference |
348
401
 
349
402
  **Bottom Line:** The Python API is designed to be Pythonic while maintaining **100% binary compatibility** with C++. All atomic operations and memory layouts are identical, enabling seamless C++/Python interoperability.
@@ -1,5 +1,133 @@
1
1
  # Changelogs
2
2
 
3
+ ## [v2.4.0] - 2026-11-09
4
+
5
+ Tracks C++ slick-queue v2.4.0, which added `slick::spsc_queue`. The `SlickQueue`
6
+ shared-memory layout is unchanged, so 2.3.0 peers in either language interoperate
7
+ with it as before.
8
+
9
+ ### Added
10
+
11
+ - `SpscQueue`, a port of C++ `slick::spsc_queue<T>`: a lossless single-producer
12
+ single-consumer ring in local or shared memory. `try_reserve(n)` returns
13
+ `SpscQueue.invalid_index` when the ring is full and `reserve(n)` waits until the
14
+ consumer frees room. `publish()` never moves the tail backwards, so publishing an
15
+ earlier reservation after a later one is a no-op.
16
+ - `read()` returns the next element as `bytes`. `read_last()` returns
17
+ `(data, dropped)`, the newest element and how many unread ones it skipped, from
18
+ one load of the producer's tail however large the backlog.
19
+ - Both reads copy the element and hand its slot back before they return. C++ holds
20
+ the slot until the consumer's next call, so a Python consumer gets the full `size`
21
+ as capacity, nothing it read is delivered again after it reattaches, and there is
22
+ no `release()`.
23
+ - `SpscQueueTraits` with one setting, `enable_cpu_relax`, which yields the GIL while
24
+ `reserve()` waits. `q.traits` is a read-only snapshot, as for `SlickQueue`.
25
+ - The C++ shared-memory layout: a 384-byte header with the tail at offset 128 and the
26
+ head at 256, then the data array. Size, element size, the `'SLS0'` marker and init
27
+ state keep the `SlickQueue` offsets. Both cursors live in the segment, so a
28
+ reattached peer resumes where the previous one stopped.
29
+ - A creator that claims a segment left uninitialized, whose own creator died before
30
+ claiming it, checks the mapping holds its geometry first. If it doesn't, it raises
31
+ and hands the claim back, instead of writing a header for a ring the mapping can't
32
+ hold. Every attacher checks the mapping size too.
33
+ - Using an `SpscQueue` after `close()` raises `RuntimeError("queue is closed")`.
34
+
35
+ ### Changed
36
+
37
+ - `SlickQueue` attaching to an `SpscQueue` segment now raises "holds a
38
+ slick::spsc_queue" instead of the generic missing-marker error, matching C++, and
39
+ `SpscQueue` names a `SlickQueue` segment the same way.
40
+ - Name handling, opening and closing the segment, `unlink()`, `remove_shm()`,
41
+ `remove()` and the context manager moved into a base class both queues share.
42
+ `SlickQueue` behaves as before.
43
+ - `close()` still never raises, but no longer prints the exception it swallows.
44
+
45
+ ### Tests
46
+
47
+ - `tests/test_spsc.py` covers the local and shared-memory API: wraps, a full ring,
48
+ out-of-order publish, element sizes other than 8, `read_last()` accounting under a
49
+ concurrent producer, reset, reattach, a too-small leftover segment, use after
50
+ `close()` and the cross-type marker checks.
51
+ - `tests/test_spsc_interop.py` runs against C++ through the new `cpp_spsc_tester`:
52
+ data in both directions, attaching to a C++-created segment, `read_last()` from
53
+ each side, and each language rejecting the other queue type.
54
+
55
+ Checked against C++ slick-queue v2.4.0 with the interop tests on Windows.
56
+
57
+ ## [v2.3.0] - 2026-10-17
58
+
59
+ Tracks C++ slick-queue v2.3.0: no queue removes its shared-memory segment on its
60
+ own any more, and `remove_shm()` is how the coordinating process does. The
61
+ version moves from 2.1.0 to 2.3.0 to stay on the C++ number. C++ v2.2.0 added
62
+ only `shm_name()`, which Python already had as `get_shm_name()`. The
63
+ shared-memory layout is unchanged, so 2.1.0 peers in either language
64
+ interoperate as before.
65
+
66
+ ### Segment lifetime (behavior change)
67
+
68
+ - On POSIX, a segment is no longer unlinked when the process that created it
69
+ exits. `SharedMemory` registered every segment with `multiprocessing`'s
70
+ resource tracker, which unlinks what is registered when the process exits, so
71
+ a creator that exited first stranded every peer still attached on an orphaned
72
+ mapping, while its restart silently created a fresh segment under the same
73
+ name that those peers never saw. C++ v2.3.0 stopped its destructor doing the
74
+ same. A name now lives until `unlink()`, `remove_shm()` or
75
+ `SlickQueue.remove(name)`, and a restarted process reattaches to the existing
76
+ segment and keeps its contents. Windows is unaffected: a section dies with its
77
+ last handle.
78
+ - The queue sends the resource tracker nothing at all: `track=False` on Python
79
+ 3.13+, and on older POSIX Pythons a `SharedMemory` subclass that maps the
80
+ segment without registering it. With no register/unregister pair there is
81
+ nothing to race between threads, or between `multiprocessing` children, which
82
+ share their parent's tracker.
83
+ - Migration: code that relied on the cleanup at exit now leaves segments in
84
+ `/dev/shm`, and its next run attaches to them - a creator joins the existing
85
+ segment instead of initializing it, the contents are the old run's, and a
86
+ creator with a different geometry raises `ValueError`. Call `SlickQueue.remove(name)` at startup for a clean start, or
87
+ `unlink()` / `remove_shm()` at shutdown once no peer will attach again.
88
+
89
+ ### Added
90
+
91
+ - `remove_shm() -> bool`, matching C++: unlinks the segment's name so the next
92
+ queue created with it starts fresh, leaving open mappings - this queue's own
93
+ included - valid. It unlinks by name on every call, and leaves `unlink()`
94
+ nothing to do afterwards, so a later `unlink()` cannot delete a replacement
95
+ created under the name. Returns `False` for a local-memory queue or when the
96
+ name is already gone, and always `True` for a shared queue on Windows.
97
+ - `SlickQueue.remove(name) -> bool`, matching C++
98
+ `slick::shm::shared_memory::remove()`: unlinks a segment by name without
99
+ attaching to it. Accepts the `/`-prefixed value of `get_shm_name()`, and is
100
+ always `True` on Windows.
101
+
102
+ ### Fixed
103
+
104
+ - `unlink()` was a silent no-op once `close()` had run, because `close()` drops
105
+ the `SharedMemory` object it went through. The documented `shared_memory`
106
+ idiom - close every handle, then unlink the name once - therefore leaked the
107
+ segment on POSIX. The name is now kept alongside the mapping, so `unlink()`
108
+ works after `close()`, and repeating it is a no-op, so it cannot delete a
109
+ segment a later run has created under the same name.
110
+ - Before Python 3.13 on POSIX, a process that only attached to a queue unlinked
111
+ its segment when it exited, because attaching registered the name with that
112
+ process's resource tracker too.
113
+
114
+ ### Tests
115
+
116
+ - New `tests/test_segment_lifetime.py` with 12 tests, registered with CTest:
117
+ `unlink()` after `close()` and repeated, `close()` and the context manager
118
+ keeping the segment, `remove_shm()` on local and shared queues, `unlink()`
119
+ after `remove_shm()` sparing a replacement segment, `remove()` by name, a
120
+ creator's and an attacher's exit in a fresh interpreter, no resource-tracker
121
+ traffic at all, a child forked while another thread is constructing a queue,
122
+ and spawned children attaching at once.
123
+ - `test_traits.py` and `test_items_per_slot.py` open their hand-built raw
124
+ segments through the library's opener, so those handles and the queue no
125
+ longer both claim a name in the resource tracker.
126
+
127
+ Checked against C++ slick-queue v2.3.0 with the interop tests on Windows and on
128
+ Linux, run twice on Linux so a segment left behind would fail the second run,
129
+ and on Linux under Python 3.12 (the subclass) and 3.14 (`track=False`).
130
+
3
131
  ## [v2.1.0] - 2026-09-24
4
132
 
5
133
  Tracks C++ slick-queue v2.1.0. Adds `items_per_slot`, so one control slot can
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: slick-queue-py
3
- Version: 2.1.0
3
+ Version: 2.4.0
4
4
  Summary: Lock-free MPMC queue with C++ interoperability via shared memory
5
5
  Home-page: https://github.com/SlickQuant/slick-queue-py
6
6
  Author: Slick Quant
@@ -36,7 +36,7 @@ Dynamic: requires-python
36
36
 
37
37
  # slick-queue-py
38
38
 
39
- Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue with C++ interoperability through shared memory.
39
+ Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue, plus a lossless single-producer single-consumer (SPSC) queue, with C++ interoperability through shared memory.
40
40
 
41
41
  This is the Python binding for the [SlickQueue C++ library](https://github.com/SlickQuant/slick-queue). The Python implementation maintains exact binary compatibility with the C++ version, enabling seamless interprocess communication between Python and C++ applications.
42
42
 
@@ -50,6 +50,8 @@ This is the Python binding for the [SlickQueue C++ library](https://github.com/S
50
50
  - **Local Memory Mode**: In-process queue using local memory (no shared memory overhead)
51
51
  - **Shared Memory Mode**: Inter-process queue for interprocess communication
52
52
  - **Lock-Free Multi-Producer Multi-Consumer**: True MPMC support using atomic operations
53
+ - **Lossless SPSC Queue**: `SpscQueue` ports C++ `slick::spsc_queue`. It never overwrites an
54
+ unread element and reports a full ring to the producer instead. See [SpscQueue](#spscqueue)
53
55
  - **C++/Python Interoperability**: Python and C++ processes can share the same queue
54
56
  - **Cross-Platform**: Windows and Linux/macOS support (x86-64)
55
57
  - **Memory Layout Compatible**: Exact binary compatibility with C++ `slick::queue<T>`
@@ -600,11 +602,92 @@ Get memoryview for writing to reserved slot.
600
602
 
601
603
  #### `close()`
602
604
 
603
- Close the shared memory connection. Always call this before unlinking.
605
+ Close the shared memory connection. Never deletes the segment.
604
606
 
605
607
  #### `unlink()`
606
608
 
607
- Delete the shared memory segment. Only call from the process that created it.
609
+ Delete the shared memory segment this queue is attached to. Valid before or after `close()`
610
+ (close every handle, then unlink once), and a no-op when repeated, so it cannot delete a segment
611
+ a later run has since created under the same name. A no-op for local memory mode.
612
+
613
+ #### `remove_shm() -> bool`
614
+
615
+ Unlink the segment's name so the next queue created with it starts fresh, matching C++
616
+ `remove_shm()`. Mappings already open, including this queue's own, stay valid. Unlike `unlink()`
617
+ it unlinks by name on every call. Returns `False` for a local-memory queue or if the name is
618
+ already gone, and always `True` for a shared queue on Windows.
619
+
620
+ #### `SlickQueue.remove(name) -> bool`
621
+
622
+ Unlink a segment by name without attaching to it, matching C++
623
+ `slick::shm::shared_memory::remove()`. Typically called at startup to clear a segment a previous
624
+ run left behind. Accepts the `/`-prefixed value of `get_shm_name()`. Always `True` on Windows.
625
+
626
+ #### `get_shm_name() -> Optional[str]`
627
+
628
+ The segment name to pass to C++ (`/`-prefixed on POSIX), or `None` for a local-memory queue.
629
+
630
+ #### Segment lifetime
631
+
632
+ No queue deletes its segment on its own: not `close()`, not the context manager, and not process
633
+ exit, matching C++ slick-queue v2.3.0. On POSIX a name outlives every mapping until it is
634
+ unlinked, so a departing process that unlinked it while a peer was still attached would leave that
635
+ peer reading an orphaned mapping while a restarted process silently created a fresh segment under
636
+ the same name. The process that coordinates the queue's lifetime calls `unlink()`, `remove_shm()`
637
+ or `SlickQueue.remove(name)` once no peer will attach again; until then a restarted process
638
+ reattaches to the existing segment and keeps its contents. On Windows a segment is freed when its
639
+ last handle closes.
640
+
641
+ The queue keeps its segments out of `multiprocessing`'s resource tracker, which would otherwise
642
+ unlink them when the process exits - before Python 3.13, even in a process that only attached. It
643
+ sends the tracker nothing at all: `track=False` on Python 3.13+, and on older POSIX Pythons a
644
+ `SharedMemory` subclass that maps the segment without registering it, so there is no
645
+ register/unregister pair to race between threads or between processes sharing one tracker.
646
+
647
+ ### SpscQueue
648
+
649
+ A lossless single-producer single-consumer ring, compatible with C++ `slick::spsc_queue<T>`. One
650
+ thread or process produces and one consumes. Unlike `SlickQueue` it never overwrites an unread
651
+ element: `try_reserve()` reports a full ring instead.
652
+
653
+ ```python
654
+ import struct
655
+ from slick_queue_py import SpscQueue
656
+
657
+ q = SpscQueue(name='ticks', size=1024, element_size=8) # create, or open with the same geometry
658
+ peer = SpscQueue(name='ticks', element_size=8) # attach, adopting the segment's size
659
+ local = SpscQueue(size=1024, element_size=8) # local memory
660
+
661
+ # Producer
662
+ i = q.try_reserve() # SpscQueue.invalid_index when full; reserve() waits instead
663
+ if i != SpscQueue.invalid_index:
664
+ q[i][:8] = struct.pack('<Q', 42)
665
+ q.publish(i)
666
+
667
+ # Consumer
668
+ data = peer.read() # next element as bytes, or None
669
+ data, dropped = peer.read_last() # newest element, and how many unread ones it skipped
670
+ ```
671
+
672
+ - `reserve(n)` and `try_reserve(n)` reserve `n` separate elements. Write element `k` through
673
+ `q[i + k]`, because a batch may wrap around the end of the ring. Publish in reservation order.
674
+ Publishing a later batch first also exposes the earlier one, and publishing the earlier one
675
+ afterwards is a no-op.
676
+ - `read()` and `read_last()` copy the element and hand its slot back before they return. C++ holds
677
+ the slot until the consumer's next call, so here the full `size` is always usable and there is no
678
+ `release()`.
679
+ - `read_last()` loads the producer's tail and moves the read cursor there, so its cost doesn't grow
680
+ with the backlog. It returns `(None, 0)` if nothing was published since the last read. Note
681
+ that the second item counts skipped elements, where `SlickQueue.read_last()` returns a size.
682
+ - Both cursors live in the segment, and a peer that reattaches resumes where the previous one
683
+ stopped. Each object caches its own side's cursor, so keep one live object per side. After
684
+ another process has produced or consumed, attach a fresh object rather than reusing an old one.
685
+ - `reset()` empties the queue. It isn't thread-safe, and a peer in another process must reattach.
686
+ - `close()`, `unlink()`, `remove_shm()`, `SpscQueue.remove(name)`, `get_shm_name()` and the
687
+ context manager behave as they do for `SlickQueue`. Using the queue after `close()` raises
688
+ `RuntimeError("queue is closed")`.
689
+ - `traits=` takes an `SpscQueueTraits` subclass. Its one setting is `enable_cpu_relax` (default
690
+ `True`), which yields the GIL while `reserve()` waits. `q.traits` is a read-only snapshot.
608
691
 
609
692
  ### AtomicCursor
610
693
 
@@ -781,6 +864,34 @@ Bits 2-3 of the nibble are reserved and must be 0; a marker that sets one is rej
781
864
  newer than this build understands. Segments created before slick-queue v1.4.0 carry no
782
865
  marker at all and are rejected at attach time.
783
866
 
867
+ ### SpscQueue layout
868
+
869
+ `SpscQueue` matches C++ `slick::spsc_queue<T>`. Each cursor sits on its own 128-byte line on every
870
+ platform:
871
+
872
+ ```
873
+ Offset | Size | Content
874
+ -------|---------------|------------------
875
+ 0 | 8 bytes | reserved (0)
876
+ 8 | 4 bytes | uint32_t size (queue capacity)
877
+ 12 | 4 bytes | uint32_t element_size
878
+ 16 | 8 bytes | reserved (0)
879
+ 24 | 4 bytes | uint32_t header_magic - 'SLS0' (0x534C5330)
880
+ 28 | 20 bytes | reserved (0)
881
+ 48 | 4 bytes | uint32_t init_state (atomic)
882
+ 52 | 76 bytes | padding (to 128)
883
+ 128 | 8 bytes | uint64_t tail (atomic), written by the producer
884
+ 136 | 120 bytes | padding
885
+ 256 | 8 bytes | uint64_t head (atomic), written by the consumer
886
+ 264 | 120 bytes | padding (to 384)
887
+ 384 | elem*size | data array
888
+ ```
889
+
890
+ `size`, `element_size`, the marker and `init_state` sit at the same offsets as in a `SlickQueue`
891
+ segment. Each queue type therefore turns the other's segment away at attach time, with *"holds a
892
+ slick::spsc_queue"* or *"holds a slick::queue"*, instead of timing out. The marker's low nibble is
893
+ reserved for future layout features and must be 0. The C++ side requires `alignof(T) <= 128`.
894
+
784
895
  ## Platform Support
785
896
 
786
897
  ### Fully Supported (Lock-Free)
@@ -920,6 +1031,12 @@ python tests/test_reset_detection.py
920
1031
 
921
1032
  # Wrapping-producer record/size invariants
922
1033
  python tests/test_wrap_invariants.py
1034
+
1035
+ # Segment lifetime: unlink(), remove_shm(), remove(), process exit
1036
+ python tests/test_segment_lifetime.py
1037
+
1038
+ # SpscQueue, local and shared memory
1039
+ python tests/test_spsc.py
923
1040
  ```
924
1041
 
925
1042
  Or run everything with pytest:
@@ -948,6 +1065,9 @@ python tests/test_interop.py --test cpp_producer_python_consumer
948
1065
  python tests/test_interop.py --test multi_producer_interop
949
1066
  python tests/test_interop.py --test stress_interop
950
1067
  python tests/test_interop.py --test cpp_shm_creation
1068
+
1069
+ # SpscQueue against C++ slick::spsc_queue (uses cpp_spsc_tester)
1070
+ python tests/test_spsc_interop.py
951
1071
  ```
952
1072
 
953
1073
  The interop tests verify:
@@ -963,7 +1083,7 @@ The interop tests verify:
963
1083
 
964
1084
  1. **Buffer Cleanup Warning**: You may see a `BufferError: cannot close exported pointers exist` warning during garbage collection. This is a **harmless warning** caused by Python's ctypes creating internal buffer references that persist beyond explicit cleanup. It occurs during program exit and **does not affect functionality, performance, or correctness**. The queue works perfectly despite this warning.
965
1085
 
966
- 2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`. This is a **harmless warning** caused by Python's ctypes creating internal buffer references that persist beyond explicit cleanup. It occurs during program exit and **does not affect functionality, performance, or correctness**. The queue works perfectly despite this warning.
1086
+ 2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`. `SlickQueue` keeps its own segments out of the resource tracker (see [Segment lifetime](#segment-lifetime)), so the warning comes from a `multiprocessing.shared_memory.SharedMemory` your code opened directly, such as one holding an `AtomicCursor`. The tracker unlinks those segments at exit; pass `track=False` (Python 3.13+) to keep them.
967
1087
 
968
1088
  ## Architecture
969
1089
 
@@ -1013,6 +1133,7 @@ queue so consumers keep up, and use `loss_count()` to detect when they have not.
1013
1133
  | read(atomic cursor) multi-consumer | ✅ | ✅ |
1014
1134
  | Feature traits | Template parameter | `traits=` argument |
1015
1135
  | Shared layout marker | `'SLQ1'` / `'SLQ0'` | Same, and validated against C++ peers |
1136
+ | Lossless SPSC queue | `slick::spsc_queue<T>` | `SpscQueue`, same `'SLS0'` layout |
1016
1137
 
1017
1138
  ## Contributing
1018
1139
 
@@ -1,6 +1,6 @@
1
1
  # slick-queue-py
2
2
 
3
- Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue with C++ interoperability through shared memory.
3
+ Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue, plus a lossless single-producer single-consumer (SPSC) queue, with C++ interoperability through shared memory.
4
4
 
5
5
  This is the Python binding for the [SlickQueue C++ library](https://github.com/SlickQuant/slick-queue). The Python implementation maintains exact binary compatibility with the C++ version, enabling seamless interprocess communication between Python and C++ applications.
6
6
 
@@ -14,6 +14,8 @@ This is the Python binding for the [SlickQueue C++ library](https://github.com/S
14
14
  - **Local Memory Mode**: In-process queue using local memory (no shared memory overhead)
15
15
  - **Shared Memory Mode**: Inter-process queue for interprocess communication
16
16
  - **Lock-Free Multi-Producer Multi-Consumer**: True MPMC support using atomic operations
17
+ - **Lossless SPSC Queue**: `SpscQueue` ports C++ `slick::spsc_queue`. It never overwrites an
18
+ unread element and reports a full ring to the producer instead. See [SpscQueue](#spscqueue)
17
19
  - **C++/Python Interoperability**: Python and C++ processes can share the same queue
18
20
  - **Cross-Platform**: Windows and Linux/macOS support (x86-64)
19
21
  - **Memory Layout Compatible**: Exact binary compatibility with C++ `slick::queue<T>`
@@ -564,11 +566,92 @@ Get memoryview for writing to reserved slot.
564
566
 
565
567
  #### `close()`
566
568
 
567
- Close the shared memory connection. Always call this before unlinking.
569
+ Close the shared memory connection. Never deletes the segment.
568
570
 
569
571
  #### `unlink()`
570
572
 
571
- Delete the shared memory segment. Only call from the process that created it.
573
+ Delete the shared memory segment this queue is attached to. Valid before or after `close()`
574
+ (close every handle, then unlink once), and a no-op when repeated, so it cannot delete a segment
575
+ a later run has since created under the same name. A no-op for local memory mode.
576
+
577
+ #### `remove_shm() -> bool`
578
+
579
+ Unlink the segment's name so the next queue created with it starts fresh, matching C++
580
+ `remove_shm()`. Mappings already open, including this queue's own, stay valid. Unlike `unlink()`
581
+ it unlinks by name on every call. Returns `False` for a local-memory queue or if the name is
582
+ already gone, and always `True` for a shared queue on Windows.
583
+
584
+ #### `SlickQueue.remove(name) -> bool`
585
+
586
+ Unlink a segment by name without attaching to it, matching C++
587
+ `slick::shm::shared_memory::remove()`. Typically called at startup to clear a segment a previous
588
+ run left behind. Accepts the `/`-prefixed value of `get_shm_name()`. Always `True` on Windows.
589
+
590
+ #### `get_shm_name() -> Optional[str]`
591
+
592
+ The segment name to pass to C++ (`/`-prefixed on POSIX), or `None` for a local-memory queue.
593
+
594
+ #### Segment lifetime
595
+
596
+ No queue deletes its segment on its own: not `close()`, not the context manager, and not process
597
+ exit, matching C++ slick-queue v2.3.0. On POSIX a name outlives every mapping until it is
598
+ unlinked, so a departing process that unlinked it while a peer was still attached would leave that
599
+ peer reading an orphaned mapping while a restarted process silently created a fresh segment under
600
+ the same name. The process that coordinates the queue's lifetime calls `unlink()`, `remove_shm()`
601
+ or `SlickQueue.remove(name)` once no peer will attach again; until then a restarted process
602
+ reattaches to the existing segment and keeps its contents. On Windows a segment is freed when its
603
+ last handle closes.
604
+
605
+ The queue keeps its segments out of `multiprocessing`'s resource tracker, which would otherwise
606
+ unlink them when the process exits - before Python 3.13, even in a process that only attached. It
607
+ sends the tracker nothing at all: `track=False` on Python 3.13+, and on older POSIX Pythons a
608
+ `SharedMemory` subclass that maps the segment without registering it, so there is no
609
+ register/unregister pair to race between threads or between processes sharing one tracker.
610
+
611
+ ### SpscQueue
612
+
613
+ A lossless single-producer single-consumer ring, compatible with C++ `slick::spsc_queue<T>`. One
614
+ thread or process produces and one consumes. Unlike `SlickQueue` it never overwrites an unread
615
+ element: `try_reserve()` reports a full ring instead.
616
+
617
+ ```python
618
+ import struct
619
+ from slick_queue_py import SpscQueue
620
+
621
+ q = SpscQueue(name='ticks', size=1024, element_size=8) # create, or open with the same geometry
622
+ peer = SpscQueue(name='ticks', element_size=8) # attach, adopting the segment's size
623
+ local = SpscQueue(size=1024, element_size=8) # local memory
624
+
625
+ # Producer
626
+ i = q.try_reserve() # SpscQueue.invalid_index when full; reserve() waits instead
627
+ if i != SpscQueue.invalid_index:
628
+ q[i][:8] = struct.pack('<Q', 42)
629
+ q.publish(i)
630
+
631
+ # Consumer
632
+ data = peer.read() # next element as bytes, or None
633
+ data, dropped = peer.read_last() # newest element, and how many unread ones it skipped
634
+ ```
635
+
636
+ - `reserve(n)` and `try_reserve(n)` reserve `n` separate elements. Write element `k` through
637
+ `q[i + k]`, because a batch may wrap around the end of the ring. Publish in reservation order.
638
+ Publishing a later batch first also exposes the earlier one, and publishing the earlier one
639
+ afterwards is a no-op.
640
+ - `read()` and `read_last()` copy the element and hand its slot back before they return. C++ holds
641
+ the slot until the consumer's next call, so here the full `size` is always usable and there is no
642
+ `release()`.
643
+ - `read_last()` loads the producer's tail and moves the read cursor there, so its cost doesn't grow
644
+ with the backlog. It returns `(None, 0)` if nothing was published since the last read. Note
645
+ that the second item counts skipped elements, where `SlickQueue.read_last()` returns a size.
646
+ - Both cursors live in the segment, and a peer that reattaches resumes where the previous one
647
+ stopped. Each object caches its own side's cursor, so keep one live object per side. After
648
+ another process has produced or consumed, attach a fresh object rather than reusing an old one.
649
+ - `reset()` empties the queue. It isn't thread-safe, and a peer in another process must reattach.
650
+ - `close()`, `unlink()`, `remove_shm()`, `SpscQueue.remove(name)`, `get_shm_name()` and the
651
+ context manager behave as they do for `SlickQueue`. Using the queue after `close()` raises
652
+ `RuntimeError("queue is closed")`.
653
+ - `traits=` takes an `SpscQueueTraits` subclass. Its one setting is `enable_cpu_relax` (default
654
+ `True`), which yields the GIL while `reserve()` waits. `q.traits` is a read-only snapshot.
572
655
 
573
656
  ### AtomicCursor
574
657
 
@@ -745,6 +828,34 @@ Bits 2-3 of the nibble are reserved and must be 0; a marker that sets one is rej
745
828
  newer than this build understands. Segments created before slick-queue v1.4.0 carry no
746
829
  marker at all and are rejected at attach time.
747
830
 
831
+ ### SpscQueue layout
832
+
833
+ `SpscQueue` matches C++ `slick::spsc_queue<T>`. Each cursor sits on its own 128-byte line on every
834
+ platform:
835
+
836
+ ```
837
+ Offset | Size | Content
838
+ -------|---------------|------------------
839
+ 0 | 8 bytes | reserved (0)
840
+ 8 | 4 bytes | uint32_t size (queue capacity)
841
+ 12 | 4 bytes | uint32_t element_size
842
+ 16 | 8 bytes | reserved (0)
843
+ 24 | 4 bytes | uint32_t header_magic - 'SLS0' (0x534C5330)
844
+ 28 | 20 bytes | reserved (0)
845
+ 48 | 4 bytes | uint32_t init_state (atomic)
846
+ 52 | 76 bytes | padding (to 128)
847
+ 128 | 8 bytes | uint64_t tail (atomic), written by the producer
848
+ 136 | 120 bytes | padding
849
+ 256 | 8 bytes | uint64_t head (atomic), written by the consumer
850
+ 264 | 120 bytes | padding (to 384)
851
+ 384 | elem*size | data array
852
+ ```
853
+
854
+ `size`, `element_size`, the marker and `init_state` sit at the same offsets as in a `SlickQueue`
855
+ segment. Each queue type therefore turns the other's segment away at attach time, with *"holds a
856
+ slick::spsc_queue"* or *"holds a slick::queue"*, instead of timing out. The marker's low nibble is
857
+ reserved for future layout features and must be 0. The C++ side requires `alignof(T) <= 128`.
858
+
748
859
  ## Platform Support
749
860
 
750
861
  ### Fully Supported (Lock-Free)
@@ -884,6 +995,12 @@ python tests/test_reset_detection.py
884
995
 
885
996
  # Wrapping-producer record/size invariants
886
997
  python tests/test_wrap_invariants.py
998
+
999
+ # Segment lifetime: unlink(), remove_shm(), remove(), process exit
1000
+ python tests/test_segment_lifetime.py
1001
+
1002
+ # SpscQueue, local and shared memory
1003
+ python tests/test_spsc.py
887
1004
  ```
888
1005
 
889
1006
  Or run everything with pytest:
@@ -912,6 +1029,9 @@ python tests/test_interop.py --test cpp_producer_python_consumer
912
1029
  python tests/test_interop.py --test multi_producer_interop
913
1030
  python tests/test_interop.py --test stress_interop
914
1031
  python tests/test_interop.py --test cpp_shm_creation
1032
+
1033
+ # SpscQueue against C++ slick::spsc_queue (uses cpp_spsc_tester)
1034
+ python tests/test_spsc_interop.py
915
1035
  ```
916
1036
 
917
1037
  The interop tests verify:
@@ -927,7 +1047,7 @@ The interop tests verify:
927
1047
 
928
1048
  1. **Buffer Cleanup Warning**: You may see a `BufferError: cannot close exported pointers exist` warning during garbage collection. This is a **harmless warning** caused by Python's ctypes creating internal buffer references that persist beyond explicit cleanup. It occurs during program exit and **does not affect functionality, performance, or correctness**. The queue works perfectly despite this warning.
929
1049
 
930
- 2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`. This is a **harmless warning** caused by Python's ctypes creating internal buffer references that persist beyond explicit cleanup. It occurs during program exit and **does not affect functionality, performance, or correctness**. The queue works perfectly despite this warning.
1050
+ 2. **UserWarning**: On Linux you may see `UserWarning: resource_tracker: There appear to be 4 leaked shared_memory objects to clean up at shutdown`. `SlickQueue` keeps its own segments out of the resource tracker (see [Segment lifetime](#segment-lifetime)), so the warning comes from a `multiprocessing.shared_memory.SharedMemory` your code opened directly, such as one holding an `AtomicCursor`. The tracker unlinks those segments at exit; pass `track=False` (Python 3.13+) to keep them.
931
1051
 
932
1052
  ## Architecture
933
1053
 
@@ -977,6 +1097,7 @@ queue so consumers keep up, and use `loss_count()` to detect when they have not.
977
1097
  | read(atomic cursor) multi-consumer | ✅ | ✅ |
978
1098
  | Feature traits | Template parameter | `traits=` argument |
979
1099
  | Shared layout marker | `'SLQ1'` / `'SLQ0'` | Same, and validated against C++ peers |
1100
+ | Lossless SPSC queue | `slick::spsc_queue<T>` | `SpscQueue`, same `'SLS0'` layout |
980
1101
 
981
1102
  ## Contributing
982
1103
 
@@ -25,6 +25,7 @@ tests/cpp_consumer.cpp
25
25
  tests/cpp_multi_producer.cpp
26
26
  tests/cpp_producer.cpp
27
27
  tests/cpp_read_last_tester.cpp
28
+ tests/cpp_spsc_tester.cpp
28
29
  tests/cpp_work_stealing_consumer.cpp
29
30
  tests/run_test.py
30
31
  tests/test_atomic_cursor.py
@@ -39,5 +40,8 @@ tests/test_queue.py
39
40
  tests/test_read_last.py
40
41
  tests/test_read_last_interop.py
41
42
  tests/test_reset_detection.py
43
+ tests/test_segment_lifetime.py
44
+ tests/test_spsc.py
45
+ tests/test_spsc_interop.py
42
46
  tests/test_traits.py
43
47
  tests/test_wrap_invariants.py