slick-stream-buffer-multiplexer-py 2.2.0__tar.gz → 2.2.0.post1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/API_DIFFERENCES.md +6 -0
  2. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/PKG-INFO +46 -18
  3. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/README.md +43 -15
  4. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/pyproject.toml +2 -2
  5. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/requirements.txt +2 -2
  6. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/setup.py +2 -2
  7. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/slick_stream_buffer_multiplexer_py.py +29 -30
  8. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer_shm.py +145 -0
  9. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/LICENSE +0 -0
  10. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/MANIFEST.in +0 -0
  11. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/setup.cfg +0 -0
  12. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/slick_stream_buffer_multiplexer_py.egg-info/SOURCES.txt +0 -0
  13. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_interop.py +0 -0
  14. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer.py +0 -0
  15. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer_mpmc.py +0 -0
  16. {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer_queue.py +0 -0
@@ -182,3 +182,9 @@ Python adds explicit lifecycle management that C++ handles via RAII:
182
182
  `remove(name)` is the one lifecycle call both sides share, with the same contract: unlink a
183
183
  segment by name without attaching to it, returning `True` on success and, on Windows, always
184
184
  `True` because a section dies with its last handle.
185
+
186
+ Neither side deletes a segment on its own: not the C++ destructors (since v2.1.0), and not
187
+ Python's `close()`, context manager or process exit. Python's dependencies keep their segments out
188
+ of `multiprocessing`'s resource tracker for this, which would otherwise unlink them when the
189
+ process exits. Each inner ring's own `remove_shm()` is reachable on both sides, through
190
+ `stream_buffer` / `queue` in Python and `stream_buffer()` / `queue()` in C++.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: slick-stream-buffer-multiplexer-py
3
- Version: 2.2.0
3
+ Version: 2.2.0.post1
4
4
  Summary: Lock-free MPMC byte stream multiplexer with C++ interoperability via shared memory
5
5
  Home-page: https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py
6
6
  Author: Slick Quant
@@ -28,8 +28,8 @@ Classifier: Operating System :: MacOS
28
28
  Requires-Python: >=3.8
29
29
  Description-Content-Type: text/markdown
30
30
  License-File: LICENSE
31
- Requires-Dist: slick-queue-py>=2.1.0
32
- Requires-Dist: slick-stream-buffer-py>=2.0.0
31
+ Requires-Dist: slick-queue-py>=2.3.0
32
+ Requires-Dist: slick-stream-buffer-py>=2.1.0
33
33
  Dynamic: author
34
34
  Dynamic: home-page
35
35
  Dynamic: license-file
@@ -46,7 +46,7 @@ every peer of a shared queue agrees on `enable_read_last` (see
46
46
  [`enable_read_last` must match on every peer](#enable_read_last-must-match-on-every-peer)).
47
47
 
48
48
  Tracks C++ `slick/stream_buffer_multiplexer.hpp` **v2.2.0**, and requires slick-queue-py >=
49
- 2.1.0 and slick-stream-buffer-py >= 2.0.0. The shared-memory formats are unchanged
49
+ 2.3.0 and slick-stream-buffer-py >= 2.1.0. The shared-memory formats are unchanged
50
50
  (`'SLQ1'` / `'SSB1'`), so segments written by v0.1.0 or by a C++ v1.x process are still
51
51
  readable with the default traits.
52
52
 
@@ -84,9 +84,10 @@ records referencing other producer ids are silently skipped (not counted as loss
84
84
  ## Requirements
85
85
 
86
86
  - Python 3.8+
87
- - [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.1.0 (queue producers need
88
- its `items_per_slot`)
89
- - [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.0.0
87
+ - [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.3.0 (queue producers need
88
+ its `items_per_slot`; 2.3.0 is the release whose segments outlive their creator)
89
+ - [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.1.0 (likewise
90
+ for segment lifetime - see [Segment lifetime](#segment-lifetime))
90
91
 
91
92
  Both dependencies bundle a small C++ extension for `std::atomic` operations; this package itself
92
93
  is pure Python.
@@ -178,6 +179,8 @@ accessors return `None` for the other kind's ids.
178
179
  from slick_stream_buffer_multiplexer_py import AtomicCursor
179
180
  from multiprocessing.shared_memory import SharedMemory
180
181
 
182
+ # On Python 3.13+ add track=False, or the resource tracker unlinks the cursor's
183
+ # segment when this process exits - see Segment lifetime
181
184
  cursor_shm = SharedMemory(name="mux_cursor", create=True, size=8)
182
185
  shared_cursor = AtomicCursor(cursor_shm.buf, 0)
183
186
 
@@ -288,28 +291,53 @@ inner_loss = sum(p.loss_count() for p in mux.get_producer_buffers().values())
288
291
  queues = {pid: p.size for pid, p in mux.get_producer_queues().items()}
289
292
  ```
290
293
 
291
- ## Shutdown
294
+ ## Segment lifetime
292
295
 
293
- Close every handle, then unlink once — the usual `shared_memory` idiom:
296
+ Closing a multiplexer or a producer handle, leaving a `with` block, or exiting the process never
297
+ deletes a shared-memory segment - not even one this process created - matching C++ v2.1.0 and
298
+ later. On POSIX a name therefore stays in `/dev/shm` until it is removed explicitly, and a creator
299
+ that finds it attaches to the existing segment instead of starting fresh: the shared record queue
300
+ still holds its records, a producer's `own_buffer` is `False` - for either kind of producer - and a
301
+ creator passing a different geometry raises. That is what lets a restarted producer rejoin
302
+ consumers that are still attached.
303
+
304
+ slick-queue-py (2.3.0+) and slick-stream-buffer-py (2.1.0+) keep their segments out of
305
+ `multiprocessing`'s resource tracker for this, since it would otherwise unlink them when the
306
+ process exits - before Python 3.13, even in a process that only attached. A `SharedMemory` your own
307
+ code opens, such as one holding an `AtomicCursor`, is still tracked unless you pass `track=False`.
308
+
309
+ Whichever process coordinates the lifetime deletes the segments - at shutdown once no peer will
310
+ attach again, by closing every handle and then unlinking once (the usual `shared_memory` idiom):
294
311
 
295
312
  ```python
296
313
  mux.close() # or leave a `with StreamBufferMultiplexer(...) as mux:` block
297
314
  mux.unlink() # deletes the record queue and every registered producer's segment, both kinds
298
315
  ```
299
316
 
300
- `close()` keeps the registration table so `unlink()` can still reach each producer's segment, and
301
- `unlink()` runs once: repeating it never deletes a segment a later run has created under the same
302
- name. On Windows `unlink()` is a no-op — a section disappears once its last handle closes.
317
+ or at startup, by name, for a clean start (see below). `close()` keeps the registration table so
318
+ `unlink()` can still reach each producer's segment, and `unlink()` runs once: repeating it never
319
+ deletes a segment a later run has created under the same name. On Windows `unlink()` is a no-op —
320
+ a section disappears once its last handle closes.
321
+
322
+ > **Upgrading from 2.0.** Before slick-queue-py 2.3.0 and slick-stream-buffer-py 2.1.0, a segment
323
+ > on POSIX was unlinked by the resource tracker when the process that created it exited - and,
324
+ > before Python 3.13, when any process that attached to it did. A producer that exited first
325
+ > therefore orphaned every consumer still attached. Code that relied on that cleanup now leaves
326
+ > segments behind and attaches to them on its next run - call `unlink()` at shutdown or
327
+ > `remove(name)` at startup.
303
328
 
304
- ## Stale-segment recovery
329
+ ## Removing segments by name
305
330
 
306
- `StreamBufferMultiplexer.remove(name)` unlinks a segment by name — it works for every kind this
307
- class creates, a shared record queue's, a producer stream buffer's and a producer queue's, since
308
- all are plain shared memory segments underneath. Use it when a previous run died mid-initialization and left a
309
- segment wedged, which construction over that name reports by raising:
331
+ `StreamBufferMultiplexer.remove(name)` unlinks a segment by name without attaching to it — it
332
+ works for every kind this class creates, a shared record queue's, a producer stream buffer's and a
333
+ producer queue's, since all are plain shared memory segments underneath. Use it at startup for a
334
+ clean start, and to clear a segment a previous run left wedged by dying mid-initialization, which
335
+ construction over that name reports by raising:
310
336
 
311
337
  ```python
312
- StreamBufferMultiplexer.remove("md_records") # clear what a dead run left
338
+ StreamBufferMultiplexer.remove("md_records") # shared record queue
339
+ StreamBufferMultiplexer.remove("md_p0") # a producer's stream buffer
340
+ StreamBufferMultiplexer.remove("md_q2") # a producer's queue
313
341
  mux = StreamBufferMultiplexer(shared_queue_size=1024, name="md_records")
314
342
  ```
315
343
 
@@ -9,7 +9,7 @@ every peer of a shared queue agrees on `enable_read_last` (see
9
9
  [`enable_read_last` must match on every peer](#enable_read_last-must-match-on-every-peer)).
10
10
 
11
11
  Tracks C++ `slick/stream_buffer_multiplexer.hpp` **v2.2.0**, and requires slick-queue-py >=
12
- 2.1.0 and slick-stream-buffer-py >= 2.0.0. The shared-memory formats are unchanged
12
+ 2.3.0 and slick-stream-buffer-py >= 2.1.0. The shared-memory formats are unchanged
13
13
  (`'SLQ1'` / `'SSB1'`), so segments written by v0.1.0 or by a C++ v1.x process are still
14
14
  readable with the default traits.
15
15
 
@@ -47,9 +47,10 @@ records referencing other producer ids are silently skipped (not counted as loss
47
47
  ## Requirements
48
48
 
49
49
  - Python 3.8+
50
- - [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.1.0 (queue producers need
51
- its `items_per_slot`)
52
- - [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.0.0
50
+ - [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.3.0 (queue producers need
51
+ its `items_per_slot`; 2.3.0 is the release whose segments outlive their creator)
52
+ - [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.1.0 (likewise
53
+ for segment lifetime - see [Segment lifetime](#segment-lifetime))
53
54
 
54
55
  Both dependencies bundle a small C++ extension for `std::atomic` operations; this package itself
55
56
  is pure Python.
@@ -141,6 +142,8 @@ accessors return `None` for the other kind's ids.
141
142
  from slick_stream_buffer_multiplexer_py import AtomicCursor
142
143
  from multiprocessing.shared_memory import SharedMemory
143
144
 
145
+ # On Python 3.13+ add track=False, or the resource tracker unlinks the cursor's
146
+ # segment when this process exits - see Segment lifetime
144
147
  cursor_shm = SharedMemory(name="mux_cursor", create=True, size=8)
145
148
  shared_cursor = AtomicCursor(cursor_shm.buf, 0)
146
149
 
@@ -251,28 +254,53 @@ inner_loss = sum(p.loss_count() for p in mux.get_producer_buffers().values())
251
254
  queues = {pid: p.size for pid, p in mux.get_producer_queues().items()}
252
255
  ```
253
256
 
254
- ## Shutdown
257
+ ## Segment lifetime
255
258
 
256
- Close every handle, then unlink once — the usual `shared_memory` idiom:
259
+ Closing a multiplexer or a producer handle, leaving a `with` block, or exiting the process never
260
+ deletes a shared-memory segment - not even one this process created - matching C++ v2.1.0 and
261
+ later. On POSIX a name therefore stays in `/dev/shm` until it is removed explicitly, and a creator
262
+ that finds it attaches to the existing segment instead of starting fresh: the shared record queue
263
+ still holds its records, a producer's `own_buffer` is `False` - for either kind of producer - and a
264
+ creator passing a different geometry raises. That is what lets a restarted producer rejoin
265
+ consumers that are still attached.
266
+
267
+ slick-queue-py (2.3.0+) and slick-stream-buffer-py (2.1.0+) keep their segments out of
268
+ `multiprocessing`'s resource tracker for this, since it would otherwise unlink them when the
269
+ process exits - before Python 3.13, even in a process that only attached. A `SharedMemory` your own
270
+ code opens, such as one holding an `AtomicCursor`, is still tracked unless you pass `track=False`.
271
+
272
+ Whichever process coordinates the lifetime deletes the segments - at shutdown once no peer will
273
+ attach again, by closing every handle and then unlinking once (the usual `shared_memory` idiom):
257
274
 
258
275
  ```python
259
276
  mux.close() # or leave a `with StreamBufferMultiplexer(...) as mux:` block
260
277
  mux.unlink() # deletes the record queue and every registered producer's segment, both kinds
261
278
  ```
262
279
 
263
- `close()` keeps the registration table so `unlink()` can still reach each producer's segment, and
264
- `unlink()` runs once: repeating it never deletes a segment a later run has created under the same
265
- name. On Windows `unlink()` is a no-op — a section disappears once its last handle closes.
280
+ or at startup, by name, for a clean start (see below). `close()` keeps the registration table so
281
+ `unlink()` can still reach each producer's segment, and `unlink()` runs once: repeating it never
282
+ deletes a segment a later run has created under the same name. On Windows `unlink()` is a no-op —
283
+ a section disappears once its last handle closes.
284
+
285
+ > **Upgrading from 2.0.** Before slick-queue-py 2.3.0 and slick-stream-buffer-py 2.1.0, a segment
286
+ > on POSIX was unlinked by the resource tracker when the process that created it exited - and,
287
+ > before Python 3.13, when any process that attached to it did. A producer that exited first
288
+ > therefore orphaned every consumer still attached. Code that relied on that cleanup now leaves
289
+ > segments behind and attaches to them on its next run - call `unlink()` at shutdown or
290
+ > `remove(name)` at startup.
266
291
 
267
- ## Stale-segment recovery
292
+ ## Removing segments by name
268
293
 
269
- `StreamBufferMultiplexer.remove(name)` unlinks a segment by name — it works for every kind this
270
- class creates, a shared record queue's, a producer stream buffer's and a producer queue's, since
271
- all are plain shared memory segments underneath. Use it when a previous run died mid-initialization and left a
272
- segment wedged, which construction over that name reports by raising:
294
+ `StreamBufferMultiplexer.remove(name)` unlinks a segment by name without attaching to it — it
295
+ works for every kind this class creates, a shared record queue's, a producer stream buffer's and a
296
+ producer queue's, since all are plain shared memory segments underneath. Use it at startup for a
297
+ clean start, and to clear a segment a previous run left wedged by dying mid-initialization, which
298
+ construction over that name reports by raising:
273
299
 
274
300
  ```python
275
- StreamBufferMultiplexer.remove("md_records") # clear what a dead run left
301
+ StreamBufferMultiplexer.remove("md_records") # shared record queue
302
+ StreamBufferMultiplexer.remove("md_p0") # a producer's stream buffer
303
+ StreamBufferMultiplexer.remove("md_q2") # a producer's queue
276
304
  mux = StreamBufferMultiplexer(shared_queue_size=1024, name="md_records")
277
305
  ```
278
306
 
@@ -30,8 +30,8 @@ classifiers = [
30
30
  "Operating System :: MacOS",
31
31
  ]
32
32
  dependencies = [
33
- "slick-queue-py>=2.1.0",
34
- "slick-stream-buffer-py>=2.0.0",
33
+ "slick-queue-py>=2.3.0",
34
+ "slick-stream-buffer-py>=2.1.0",
35
35
  ]
36
36
 
37
37
  [project.urls]
@@ -1,3 +1,3 @@
1
1
  # Runtime dependencies (binary-compatible Python ports of the C++ dependencies)
2
- slick-queue-py>=2.1.0
3
- slick-stream-buffer-py>=2.0.0
2
+ slick-queue-py>=2.3.0
3
+ slick-stream-buffer-py>=2.1.0
@@ -52,8 +52,8 @@ setup(
52
52
  keywords='stream-buffer, multiplexer, lock-free, atomic, shared-memory, ipc, multiprocessing, mpmc',
53
53
  py_modules=['slick_stream_buffer_multiplexer_py'],
54
54
  install_requires=[
55
- 'slick-queue-py>=2.1.0',
56
- 'slick-stream-buffer-py>=2.0.0',
55
+ 'slick-queue-py>=2.3.0',
56
+ 'slick-stream-buffer-py>=2.1.0',
57
57
  ],
58
58
  python_requires='>=3.8',
59
59
  license='MIT',
@@ -76,10 +76,16 @@ Caveats (same as C++):
76
76
  propagated from SlickStreamBuffer.consume() before any state moves - so
77
77
  nothing is published to either the producer's ring or the shared record
78
78
  queue. A queue message is limited to the queue's size by SlickQueue.reserve().
79
+ - No segment is ever unlinked on its own - not by close(), the context
80
+ manager or process exit, including through multiprocessing's resource
81
+ tracker, which slick-queue-py and slick-stream-buffer-py keep their segments
82
+ out of. A restarted producer therefore reattaches to the segments its
83
+ consumers still map. The coordinating process deletes them with unlink() or
84
+ remove(name).
79
85
  """
80
86
  from __future__ import annotations
81
87
 
82
- __version__ = '2.2.0'
88
+ __version__ = '2.2.0.post1'
83
89
 
84
90
  import enum
85
91
  import operator
@@ -350,11 +356,6 @@ class ProducerQueue(_Producer):
350
356
  # items_per_slot - 1, for _dereference(): read() advances a cursor by the
351
357
  # whole reservation, which is the message length rounded up to this.
352
358
  self._item_mask = queue.items_per_slot - 1
353
- # the queue's SharedMemory, held for unlink() for the same reason the
354
- # multiplexer holds its record queue's: SlickQueue.close() drops its own
355
- # handle, after which SlickQueue.unlink() silently does nothing. None for
356
- # local memory.
357
- self._shm = getattr(queue, '_shm', None)
358
359
 
359
360
  # ------------------------------------------------------------------
360
361
  # Producer side (any number of threads) - forwards to the inner queue
@@ -457,15 +458,11 @@ class ProducerQueue(_Producer):
457
458
 
458
459
  def unlink(self) -> None:
459
460
  """Unlink (delete) the underlying shared memory segment, if any. Valid
460
- before or after close(), and a no-op when repeated."""
461
- shm = self._shm
462
- if shm is None:
463
- return
464
- self._shm = None # unlink once, however many times this is called
465
- try:
466
- shm.unlink()
467
- except Exception:
468
- pass
461
+ before or after close(), and a no-op when repeated (see
462
+ SlickQueue.unlink()). The once-only guard lives on the queue, so an
463
+ adopted queue registered with several multiplexers is still unlinked
464
+ once."""
465
+ self._queue.unlink()
469
466
 
470
467
  def _dereference(self, sequence: int) -> MultiplexRecord:
471
468
  """Dereference a shared-queue record into this producer's queue. Falsy
@@ -552,10 +549,6 @@ class StreamBufferMultiplexer:
552
549
  raise ValueError("shared_queue_size or name must be provided")
553
550
  self._shared_queue = SlickQueue(name=name, element_size=RECORD_SIZE,
554
551
  traits=traits)
555
- # the queue's SharedMemory, held for unlink(): SlickQueue.close() drops
556
- # its own handle, after which SlickQueue.unlink() silently does nothing -
557
- # yet close-then-unlink is the documented order. None for local memory.
558
- self._shared_queue_shm = getattr(self._shared_queue, '_shm', None)
559
552
 
560
553
  # One table per kind - what get_producer_buffers() / get_producer_queues()
561
554
  # hand out - plus one of every producer for read(), the counterpart of the
@@ -586,15 +579,21 @@ class StreamBufferMultiplexer:
586
579
 
587
580
  @staticmethod
588
581
  def remove(name: str) -> bool:
589
- """Remove a shared memory segment by name, the stale-segment recovery step.
582
+ """Remove a shared memory segment by name: the explicit cleanup step, and
583
+ the stale-segment recovery step.
590
584
 
591
585
  Applies to every kind of name this class creates - a shared record
592
586
  queue's, a producer stream buffer's and a producer queue's - since all
593
- are plain shared memory segments underneath. Use it when a previous run died
594
- mid-initialization and left a segment wedged, which construction over
595
- that name reports by raising::
596
-
597
- StreamBufferMultiplexer.remove("md_records") # clear a dead run
587
+ are plain shared memory segments underneath. Nothing unlinks a segment
588
+ on close() or process exit, so on POSIX a name outlives every process
589
+ that used it and the next creator attaches to it instead of starting
590
+ fresh. Call this from whichever process coordinates the lifetime: at
591
+ startup for a clean start, or at shutdown once no peer will attach
592
+ again. It also clears a segment a previous run left wedged by dying
593
+ mid-initialization, which construction over that name reports by
594
+ raising::
595
+
596
+ StreamBufferMultiplexer.remove("md_records") # clear what the last run left
598
597
  mux = StreamBufferMultiplexer(shared_queue_size=1024, name="md_records")
599
598
 
600
599
  Returns:
@@ -845,6 +844,10 @@ class StreamBufferMultiplexer:
845
844
  """Unlink (delete) the shared record queue segment and all registered
846
845
  producers' segments, of both kinds.
847
846
 
847
+ No multiplexer does this on its own - not close(), not the context
848
+ manager and not process exit, as in C++ - so call it from the process
849
+ that coordinates the segments' lifetime, once no peer will attach again.
850
+
848
851
  Valid before or after close(); the documented order is the
849
852
  ``shared_memory`` idiom - close every handle, then unlink once. Deleting
850
853
  a name never touches mappings that already exist. Unlinking twice is a
@@ -856,12 +859,8 @@ class StreamBufferMultiplexer:
856
859
  producer.unlink() # by saved name, once - valid after close()
857
860
  except Exception:
858
861
  pass
859
- shm = self._shared_queue_shm
860
- if shm is None:
861
- return
862
- self._shared_queue_shm = None # unlink once, however many times this is called
863
862
  try:
864
- shm.unlink()
863
+ self._shared_queue.unlink() # likewise
865
864
  except Exception:
866
865
  pass
867
866
 
@@ -521,6 +521,147 @@ def test_shm_unlink_after_close_covers_queue_producers():
521
521
  successor.unlink()
522
522
 
523
523
 
524
+ # ---------------------------------------------------------------------------
525
+ # Segment lifetime: nothing unlinks a segment on its own (C++ v2.1.0+)
526
+ # ---------------------------------------------------------------------------
527
+
528
+ def _remove_all(names) -> None:
529
+ for n in names:
530
+ StreamBufferMultiplexer.remove(n)
531
+
532
+
533
+ def _read_all(q: str, p0: str, q1: str):
534
+ """Open the record queue and both producers by name and read every message."""
535
+ mux = StreamBufferMultiplexer(name=q)
536
+ try:
537
+ mux.add_producer(0, name=p0)
538
+ mux.add_queue_producer(1, name=q1)
539
+ out = []
540
+ cursor = 0
541
+ while True:
542
+ rec, cursor = mux.read(cursor)
543
+ if not rec:
544
+ return out
545
+ out.append((rec.producer_id, bytes(rec.data)))
546
+ finally:
547
+ mux.close()
548
+
549
+
550
+ # A fresh interpreter, so it has its own resource tracker - a multiprocessing
551
+ # child would share the parent's, and the parent's exit would be what unlinked
552
+ # the names.
553
+ _LIFETIME_CHILD = r"""
554
+ import sys
555
+ from pathlib import Path
556
+ root = Path(sys.argv[1])
557
+ for p in (root, root.parent / "slick-queue-py", root.parent / "slick-stream-buffer-py"):
558
+ sys.path.insert(0, str(p))
559
+ from slick_stream_buffer_multiplexer_py import StreamBufferMultiplexer
560
+ q, p0, q1, mode, tag = sys.argv[2:7]
561
+ if mode == 'create':
562
+ mux = StreamBufferMultiplexer(shared_queue_size=64, name=q)
563
+ pb = mux.add_producer(0, capacity=1024, control_size=16, name=p0)
564
+ pq = mux.add_queue_producer(1, size=1024, items_per_slot=16, name=q1)
565
+ assert pb.own_buffer and pq.own_buffer, "child expected to create the segments"
566
+ else:
567
+ mux = StreamBufferMultiplexer(name=q)
568
+ pb = mux.add_producer(0, name=p0)
569
+ pq = mux.add_queue_producer(1, name=q1)
570
+ data = tag.encode()
571
+ mv = pb.prepare(len(data))
572
+ mv[:] = data
573
+ pb.commit(len(data))
574
+ pb.consume(len(data))
575
+ i = pq.reserve(len(data))
576
+ pq[i][:len(data)] = data
577
+ pq.publish(i, len(data))
578
+ mux.close()
579
+ """
580
+
581
+
582
+ def _run_lifetime_child(q: str, p0: str, q1: str, mode: str, tag: str) -> None:
583
+ import subprocess
584
+ result = subprocess.run(
585
+ [sys.executable, "-c", _LIFETIME_CHILD, str(_root), q, p0, q1, mode, tag],
586
+ capture_output=True, text=True, timeout=60)
587
+ assert result.returncode == 0, f"child failed:\n{result.stdout}\n{result.stderr}"
588
+ # The tracker warns about every registered name it cleans up at shutdown
589
+ assert "resource_tracker" not in result.stderr, result.stderr
590
+
591
+
592
+ def test_shm_unlink_leaves_resource_tracker_alone():
593
+ """Regression: unlink() deleted the record queue and each queue producer's
594
+ segment through SharedMemory.unlink(), which before Python 3.13 also sends
595
+ the resource tracker an UNREGISTER. slick-queue-py keeps no registration
596
+ since 2.3.0, so the tracker raised KeyError and printed a traceback from its
597
+ own process. Routing through SlickQueue.unlink() sends nothing."""
598
+ from multiprocessing import resource_tracker
599
+ names = ("mux_py_ut_records", "mux_py_ut_p0", "mux_py_ut_q1")
600
+ _remove_all(names)
601
+ mux = StreamBufferMultiplexer(shared_queue_size=64, name=names[0])
602
+ mux.add_producer(0, capacity=1024, control_size=16, name=names[1])
603
+ mux.add_queue_producer(1, size=1024, items_per_slot=16, name=names[2])
604
+ mux.close()
605
+
606
+ calls = []
607
+ original = resource_tracker.unregister
608
+ resource_tracker.unregister = lambda n, rtype: calls.append((n, rtype))
609
+ try:
610
+ mux.unlink()
611
+ finally:
612
+ resource_tracker.unregister = original
613
+ assert calls == [], calls
614
+
615
+ if sys.platform != 'win32':
616
+ leaked = [n for n in names if StreamBufferMultiplexer.remove(n)]
617
+ assert not leaked, f"segments outlived close() + unlink(): {leaked}"
618
+
619
+
620
+ def test_shm_creator_exit_keeps_segments():
621
+ """A creator's exit used to unlink its names on POSIX through the resource
622
+ tracker - the record queue and both kinds of producer segment - stranding
623
+ every consumer still attached. On Windows a section dies with its last
624
+ handle, which no one else holds here."""
625
+ if sys.platform == 'win32':
626
+ return
627
+ names = ("mux_py_cx_records", "mux_py_cx_p0", "mux_py_cx_q1")
628
+ _remove_all(names)
629
+ try:
630
+ _run_lifetime_child(*names, 'create', "from child")
631
+ assert _read_all(*names) == [(0, b"from child"), (1, b"from child")]
632
+
633
+ # A restarted creator reattaches to all three and keeps their contents
634
+ again = StreamBufferMultiplexer(shared_queue_size=64, name=names[0])
635
+ try:
636
+ assert not again.shared_queue._own
637
+ pb = again.add_producer(0, capacity=1024, control_size=16, name=names[1])
638
+ pq = again.add_queue_producer(1, size=1024, items_per_slot=16, name=names[2])
639
+ assert not pb.own_buffer
640
+ assert not pq.own_buffer
641
+ finally:
642
+ again.close()
643
+ finally:
644
+ _remove_all(names)
645
+
646
+
647
+ def test_shm_attacher_exit_keeps_segments():
648
+ """Before Python 3.13 attaching registered every name with the attacher's
649
+ resource tracker too, so any peer's exit unlinked the segments on POSIX."""
650
+ names = ("mux_py_ax_records", "mux_py_ax_p0", "mux_py_ax_q1")
651
+ _remove_all(names)
652
+ creator = StreamBufferMultiplexer(shared_queue_size=64, name=names[0])
653
+ try:
654
+ creator.add_producer(0, capacity=1024, control_size=16, name=names[1])
655
+ creator.add_queue_producer(1, size=1024, items_per_slot=16, name=names[2])
656
+ _run_lifetime_child(*names, 'open', "first")
657
+ _run_lifetime_child(*names, 'open', "second")
658
+ assert _read_all(*names) == [
659
+ (0, b"first"), (1, b"first"), (0, b"second"), (1, b"second")]
660
+ finally:
661
+ creator.close()
662
+ creator.unlink()
663
+
664
+
524
665
  def run_all_tests():
525
666
  tests = [
526
667
  ("ShmRoundtrip", test_shm_roundtrip),
@@ -536,6 +677,10 @@ def run_all_tests():
536
677
  ("ShmQueueCrossProcessRoundtrip", test_shm_queue_cross_process_roundtrip),
537
678
  ("ShmUnlinkAfterCloseCoversQueueProducers",
538
679
  test_shm_unlink_after_close_covers_queue_producers),
680
+ ("ShmUnlinkLeavesResourceTrackerAlone",
681
+ test_shm_unlink_leaves_resource_tracker_alone),
682
+ ("ShmCreatorExitKeepsSegments", test_shm_creator_exit_keeps_segments),
683
+ ("ShmAttacherExitKeepsSegments", test_shm_attacher_exit_keeps_segments),
539
684
  ]
540
685
 
541
686
  print("=" * 70)