slick-stream-buffer-multiplexer-py 2.2.0__tar.gz → 2.2.0.post1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/API_DIFFERENCES.md +6 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/PKG-INFO +46 -18
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/README.md +43 -15
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/pyproject.toml +2 -2
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/requirements.txt +2 -2
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/setup.py +2 -2
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/slick_stream_buffer_multiplexer_py.py +29 -30
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer_shm.py +145 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/LICENSE +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/MANIFEST.in +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/setup.cfg +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/slick_stream_buffer_multiplexer_py.egg-info/SOURCES.txt +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_interop.py +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer.py +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer_mpmc.py +0 -0
- {slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/tests/test_multiplexer_queue.py +0 -0
|
@@ -182,3 +182,9 @@ Python adds explicit lifecycle management that C++ handles via RAII:
|
|
|
182
182
|
`remove(name)` is the one lifecycle call both sides share, with the same contract: unlink a
|
|
183
183
|
segment by name without attaching to it, returning `True` on success and, on Windows, always
|
|
184
184
|
`True` because a section dies with its last handle.
|
|
185
|
+
|
|
186
|
+
Neither side deletes a segment on its own: not the C++ destructors (since v2.1.0), and not
|
|
187
|
+
Python's `close()`, context manager or process exit. Python's dependencies keep their segments out
|
|
188
|
+
of `multiprocessing`'s resource tracker for this, which would otherwise unlink them when the
|
|
189
|
+
process exits. Each inner ring's own `remove_shm()` is reachable on both sides, through
|
|
190
|
+
`stream_buffer` / `queue` in Python and `stream_buffer()` / `queue()` in C++.
|
{slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: slick-stream-buffer-multiplexer-py
|
|
3
|
-
Version: 2.2.0
|
|
3
|
+
Version: 2.2.0.post1
|
|
4
4
|
Summary: Lock-free MPMC byte stream multiplexer with C++ interoperability via shared memory
|
|
5
5
|
Home-page: https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py
|
|
6
6
|
Author: Slick Quant
|
|
@@ -28,8 +28,8 @@ Classifier: Operating System :: MacOS
|
|
|
28
28
|
Requires-Python: >=3.8
|
|
29
29
|
Description-Content-Type: text/markdown
|
|
30
30
|
License-File: LICENSE
|
|
31
|
-
Requires-Dist: slick-queue-py>=2.
|
|
32
|
-
Requires-Dist: slick-stream-buffer-py>=2.
|
|
31
|
+
Requires-Dist: slick-queue-py>=2.3.0
|
|
32
|
+
Requires-Dist: slick-stream-buffer-py>=2.1.0
|
|
33
33
|
Dynamic: author
|
|
34
34
|
Dynamic: home-page
|
|
35
35
|
Dynamic: license-file
|
|
@@ -46,7 +46,7 @@ every peer of a shared queue agrees on `enable_read_last` (see
|
|
|
46
46
|
[`enable_read_last` must match on every peer](#enable_read_last-must-match-on-every-peer)).
|
|
47
47
|
|
|
48
48
|
Tracks C++ `slick/stream_buffer_multiplexer.hpp` **v2.2.0**, and requires slick-queue-py >=
|
|
49
|
-
2.
|
|
49
|
+
2.3.0 and slick-stream-buffer-py >= 2.1.0. The shared-memory formats are unchanged
|
|
50
50
|
(`'SLQ1'` / `'SSB1'`), so segments written by v0.1.0 or by a C++ v1.x process are still
|
|
51
51
|
readable with the default traits.
|
|
52
52
|
|
|
@@ -84,9 +84,10 @@ records referencing other producer ids are silently skipped (not counted as loss
|
|
|
84
84
|
## Requirements
|
|
85
85
|
|
|
86
86
|
- Python 3.8+
|
|
87
|
-
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.
|
|
88
|
-
its `items_per_slot
|
|
89
|
-
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.
|
|
87
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.3.0 (queue producers need
|
|
88
|
+
its `items_per_slot`; 2.3.0 is the release whose segments outlive their creator)
|
|
89
|
+
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.1.0 (likewise
|
|
90
|
+
for segment lifetime - see [Segment lifetime](#segment-lifetime))
|
|
90
91
|
|
|
91
92
|
Both dependencies bundle a small C++ extension for `std::atomic` operations; this package itself
|
|
92
93
|
is pure Python.
|
|
@@ -178,6 +179,8 @@ accessors return `None` for the other kind's ids.
|
|
|
178
179
|
from slick_stream_buffer_multiplexer_py import AtomicCursor
|
|
179
180
|
from multiprocessing.shared_memory import SharedMemory
|
|
180
181
|
|
|
182
|
+
# On Python 3.13+ add track=False, or the resource tracker unlinks the cursor's
|
|
183
|
+
# segment when this process exits - see Segment lifetime
|
|
181
184
|
cursor_shm = SharedMemory(name="mux_cursor", create=True, size=8)
|
|
182
185
|
shared_cursor = AtomicCursor(cursor_shm.buf, 0)
|
|
183
186
|
|
|
@@ -288,28 +291,53 @@ inner_loss = sum(p.loss_count() for p in mux.get_producer_buffers().values())
|
|
|
288
291
|
queues = {pid: p.size for pid, p in mux.get_producer_queues().items()}
|
|
289
292
|
```
|
|
290
293
|
|
|
291
|
-
##
|
|
294
|
+
## Segment lifetime
|
|
292
295
|
|
|
293
|
-
|
|
296
|
+
Closing a multiplexer or a producer handle, leaving a `with` block, or exiting the process never
|
|
297
|
+
deletes a shared-memory segment - not even one this process created - matching C++ v2.1.0 and
|
|
298
|
+
later. On POSIX a name therefore stays in `/dev/shm` until it is removed explicitly, and a creator
|
|
299
|
+
that finds it attaches to the existing segment instead of starting fresh: the shared record queue
|
|
300
|
+
still holds its records, a producer's `own_buffer` is `False` - for either kind of producer - and a
|
|
301
|
+
creator passing a different geometry raises. That is what lets a restarted producer rejoin
|
|
302
|
+
consumers that are still attached.
|
|
303
|
+
|
|
304
|
+
slick-queue-py (2.3.0+) and slick-stream-buffer-py (2.1.0+) keep their segments out of
|
|
305
|
+
`multiprocessing`'s resource tracker for this, since it would otherwise unlink them when the
|
|
306
|
+
process exits - before Python 3.13, even in a process that only attached. A `SharedMemory` your own
|
|
307
|
+
code opens, such as one holding an `AtomicCursor`, is still tracked unless you pass `track=False`.
|
|
308
|
+
|
|
309
|
+
Whichever process coordinates the lifetime deletes the segments - at shutdown once no peer will
|
|
310
|
+
attach again, by closing every handle and then unlinking once (the usual `shared_memory` idiom):
|
|
294
311
|
|
|
295
312
|
```python
|
|
296
313
|
mux.close() # or leave a `with StreamBufferMultiplexer(...) as mux:` block
|
|
297
314
|
mux.unlink() # deletes the record queue and every registered producer's segment, both kinds
|
|
298
315
|
```
|
|
299
316
|
|
|
300
|
-
`close()` keeps the registration table so
|
|
301
|
-
`unlink()`
|
|
302
|
-
name. On Windows `unlink()` is a no-op —
|
|
317
|
+
or at startup, by name, for a clean start (see below). `close()` keeps the registration table so
|
|
318
|
+
`unlink()` can still reach each producer's segment, and `unlink()` runs once: repeating it never
|
|
319
|
+
deletes a segment a later run has created under the same name. On Windows `unlink()` is a no-op —
|
|
320
|
+
a section disappears once its last handle closes.
|
|
321
|
+
|
|
322
|
+
> **Upgrading from 2.0.** Before slick-queue-py 2.3.0 and slick-stream-buffer-py 2.1.0, a segment
|
|
323
|
+
> on POSIX was unlinked by the resource tracker when the process that created it exited - and,
|
|
324
|
+
> before Python 3.13, when any process that attached to it did. A producer that exited first
|
|
325
|
+
> therefore orphaned every consumer still attached. Code that relied on that cleanup now leaves
|
|
326
|
+
> segments behind and attaches to them on its next run - call `unlink()` at shutdown or
|
|
327
|
+
> `remove(name)` at startup.
|
|
303
328
|
|
|
304
|
-
##
|
|
329
|
+
## Removing segments by name
|
|
305
330
|
|
|
306
|
-
`StreamBufferMultiplexer.remove(name)` unlinks a segment by name
|
|
307
|
-
class creates, a shared record queue's, a producer stream buffer's and a
|
|
308
|
-
all are plain shared memory segments underneath. Use it
|
|
309
|
-
|
|
331
|
+
`StreamBufferMultiplexer.remove(name)` unlinks a segment by name without attaching to it — it
|
|
332
|
+
works for every kind this class creates, a shared record queue's, a producer stream buffer's and a
|
|
333
|
+
producer queue's, since all are plain shared memory segments underneath. Use it at startup for a
|
|
334
|
+
clean start, and to clear a segment a previous run left wedged by dying mid-initialization, which
|
|
335
|
+
construction over that name reports by raising:
|
|
310
336
|
|
|
311
337
|
```python
|
|
312
|
-
StreamBufferMultiplexer.remove("md_records") #
|
|
338
|
+
StreamBufferMultiplexer.remove("md_records") # shared record queue
|
|
339
|
+
StreamBufferMultiplexer.remove("md_p0") # a producer's stream buffer
|
|
340
|
+
StreamBufferMultiplexer.remove("md_q2") # a producer's queue
|
|
313
341
|
mux = StreamBufferMultiplexer(shared_queue_size=1024, name="md_records")
|
|
314
342
|
```
|
|
315
343
|
|
|
@@ -9,7 +9,7 @@ every peer of a shared queue agrees on `enable_read_last` (see
|
|
|
9
9
|
[`enable_read_last` must match on every peer](#enable_read_last-must-match-on-every-peer)).
|
|
10
10
|
|
|
11
11
|
Tracks C++ `slick/stream_buffer_multiplexer.hpp` **v2.2.0**, and requires slick-queue-py >=
|
|
12
|
-
2.
|
|
12
|
+
2.3.0 and slick-stream-buffer-py >= 2.1.0. The shared-memory formats are unchanged
|
|
13
13
|
(`'SLQ1'` / `'SSB1'`), so segments written by v0.1.0 or by a C++ v1.x process are still
|
|
14
14
|
readable with the default traits.
|
|
15
15
|
|
|
@@ -47,9 +47,10 @@ records referencing other producer ids are silently skipped (not counted as loss
|
|
|
47
47
|
## Requirements
|
|
48
48
|
|
|
49
49
|
- Python 3.8+
|
|
50
|
-
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.
|
|
51
|
-
its `items_per_slot
|
|
52
|
-
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.
|
|
50
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 2.3.0 (queue producers need
|
|
51
|
+
its `items_per_slot`; 2.3.0 is the release whose segments outlive their creator)
|
|
52
|
+
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 2.1.0 (likewise
|
|
53
|
+
for segment lifetime - see [Segment lifetime](#segment-lifetime))
|
|
53
54
|
|
|
54
55
|
Both dependencies bundle a small C++ extension for `std::atomic` operations; this package itself
|
|
55
56
|
is pure Python.
|
|
@@ -141,6 +142,8 @@ accessors return `None` for the other kind's ids.
|
|
|
141
142
|
from slick_stream_buffer_multiplexer_py import AtomicCursor
|
|
142
143
|
from multiprocessing.shared_memory import SharedMemory
|
|
143
144
|
|
|
145
|
+
# On Python 3.13+ add track=False, or the resource tracker unlinks the cursor's
|
|
146
|
+
# segment when this process exits - see Segment lifetime
|
|
144
147
|
cursor_shm = SharedMemory(name="mux_cursor", create=True, size=8)
|
|
145
148
|
shared_cursor = AtomicCursor(cursor_shm.buf, 0)
|
|
146
149
|
|
|
@@ -251,28 +254,53 @@ inner_loss = sum(p.loss_count() for p in mux.get_producer_buffers().values())
|
|
|
251
254
|
queues = {pid: p.size for pid, p in mux.get_producer_queues().items()}
|
|
252
255
|
```
|
|
253
256
|
|
|
254
|
-
##
|
|
257
|
+
## Segment lifetime
|
|
255
258
|
|
|
256
|
-
|
|
259
|
+
Closing a multiplexer or a producer handle, leaving a `with` block, or exiting the process never
|
|
260
|
+
deletes a shared-memory segment - not even one this process created - matching C++ v2.1.0 and
|
|
261
|
+
later. On POSIX a name therefore stays in `/dev/shm` until it is removed explicitly, and a creator
|
|
262
|
+
that finds it attaches to the existing segment instead of starting fresh: the shared record queue
|
|
263
|
+
still holds its records, a producer's `own_buffer` is `False` - for either kind of producer - and a
|
|
264
|
+
creator passing a different geometry raises. That is what lets a restarted producer rejoin
|
|
265
|
+
consumers that are still attached.
|
|
266
|
+
|
|
267
|
+
slick-queue-py (2.3.0+) and slick-stream-buffer-py (2.1.0+) keep their segments out of
|
|
268
|
+
`multiprocessing`'s resource tracker for this, since it would otherwise unlink them when the
|
|
269
|
+
process exits - before Python 3.13, even in a process that only attached. A `SharedMemory` your own
|
|
270
|
+
code opens, such as one holding an `AtomicCursor`, is still tracked unless you pass `track=False`.
|
|
271
|
+
|
|
272
|
+
Whichever process coordinates the lifetime deletes the segments - at shutdown once no peer will
|
|
273
|
+
attach again, by closing every handle and then unlinking once (the usual `shared_memory` idiom):
|
|
257
274
|
|
|
258
275
|
```python
|
|
259
276
|
mux.close() # or leave a `with StreamBufferMultiplexer(...) as mux:` block
|
|
260
277
|
mux.unlink() # deletes the record queue and every registered producer's segment, both kinds
|
|
261
278
|
```
|
|
262
279
|
|
|
263
|
-
`close()` keeps the registration table so
|
|
264
|
-
`unlink()`
|
|
265
|
-
name. On Windows `unlink()` is a no-op —
|
|
280
|
+
or at startup, by name, for a clean start (see below). `close()` keeps the registration table so
|
|
281
|
+
`unlink()` can still reach each producer's segment, and `unlink()` runs once: repeating it never
|
|
282
|
+
deletes a segment a later run has created under the same name. On Windows `unlink()` is a no-op —
|
|
283
|
+
a section disappears once its last handle closes.
|
|
284
|
+
|
|
285
|
+
> **Upgrading from 2.0.** Before slick-queue-py 2.3.0 and slick-stream-buffer-py 2.1.0, a segment
|
|
286
|
+
> on POSIX was unlinked by the resource tracker when the process that created it exited - and,
|
|
287
|
+
> before Python 3.13, when any process that attached to it did. A producer that exited first
|
|
288
|
+
> therefore orphaned every consumer still attached. Code that relied on that cleanup now leaves
|
|
289
|
+
> segments behind and attaches to them on its next run - call `unlink()` at shutdown or
|
|
290
|
+
> `remove(name)` at startup.
|
|
266
291
|
|
|
267
|
-
##
|
|
292
|
+
## Removing segments by name
|
|
268
293
|
|
|
269
|
-
`StreamBufferMultiplexer.remove(name)` unlinks a segment by name
|
|
270
|
-
class creates, a shared record queue's, a producer stream buffer's and a
|
|
271
|
-
all are plain shared memory segments underneath. Use it
|
|
272
|
-
|
|
294
|
+
`StreamBufferMultiplexer.remove(name)` unlinks a segment by name without attaching to it — it
|
|
295
|
+
works for every kind this class creates, a shared record queue's, a producer stream buffer's and a
|
|
296
|
+
producer queue's, since all are plain shared memory segments underneath. Use it at startup for a
|
|
297
|
+
clean start, and to clear a segment a previous run left wedged by dying mid-initialization, which
|
|
298
|
+
construction over that name reports by raising:
|
|
273
299
|
|
|
274
300
|
```python
|
|
275
|
-
StreamBufferMultiplexer.remove("md_records") #
|
|
301
|
+
StreamBufferMultiplexer.remove("md_records") # shared record queue
|
|
302
|
+
StreamBufferMultiplexer.remove("md_p0") # a producer's stream buffer
|
|
303
|
+
StreamBufferMultiplexer.remove("md_q2") # a producer's queue
|
|
276
304
|
mux = StreamBufferMultiplexer(shared_queue_size=1024, name="md_records")
|
|
277
305
|
```
|
|
278
306
|
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
# Runtime dependencies (binary-compatible Python ports of the C++ dependencies)
|
|
2
|
-
slick-queue-py>=2.
|
|
3
|
-
slick-stream-buffer-py>=2.
|
|
2
|
+
slick-queue-py>=2.3.0
|
|
3
|
+
slick-stream-buffer-py>=2.1.0
|
{slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/setup.py
RENAMED
|
@@ -52,8 +52,8 @@ setup(
|
|
|
52
52
|
keywords='stream-buffer, multiplexer, lock-free, atomic, shared-memory, ipc, multiprocessing, mpmc',
|
|
53
53
|
py_modules=['slick_stream_buffer_multiplexer_py'],
|
|
54
54
|
install_requires=[
|
|
55
|
-
'slick-queue-py>=2.
|
|
56
|
-
'slick-stream-buffer-py>=2.
|
|
55
|
+
'slick-queue-py>=2.3.0',
|
|
56
|
+
'slick-stream-buffer-py>=2.1.0',
|
|
57
57
|
],
|
|
58
58
|
python_requires='>=3.8',
|
|
59
59
|
license='MIT',
|
|
@@ -76,10 +76,16 @@ Caveats (same as C++):
|
|
|
76
76
|
propagated from SlickStreamBuffer.consume() before any state moves - so
|
|
77
77
|
nothing is published to either the producer's ring or the shared record
|
|
78
78
|
queue. A queue message is limited to the queue's size by SlickQueue.reserve().
|
|
79
|
+
- No segment is ever unlinked on its own - not by close(), the context
|
|
80
|
+
manager or process exit, including through multiprocessing's resource
|
|
81
|
+
tracker, which slick-queue-py and slick-stream-buffer-py keep their segments
|
|
82
|
+
out of. A restarted producer therefore reattaches to the segments its
|
|
83
|
+
consumers still map. The coordinating process deletes them with unlink() or
|
|
84
|
+
remove(name).
|
|
79
85
|
"""
|
|
80
86
|
from __future__ import annotations
|
|
81
87
|
|
|
82
|
-
__version__ = '2.2.0'
|
|
88
|
+
__version__ = '2.2.0.post1'
|
|
83
89
|
|
|
84
90
|
import enum
|
|
85
91
|
import operator
|
|
@@ -350,11 +356,6 @@ class ProducerQueue(_Producer):
|
|
|
350
356
|
# items_per_slot - 1, for _dereference(): read() advances a cursor by the
|
|
351
357
|
# whole reservation, which is the message length rounded up to this.
|
|
352
358
|
self._item_mask = queue.items_per_slot - 1
|
|
353
|
-
# the queue's SharedMemory, held for unlink() for the same reason the
|
|
354
|
-
# multiplexer holds its record queue's: SlickQueue.close() drops its own
|
|
355
|
-
# handle, after which SlickQueue.unlink() silently does nothing. None for
|
|
356
|
-
# local memory.
|
|
357
|
-
self._shm = getattr(queue, '_shm', None)
|
|
358
359
|
|
|
359
360
|
# ------------------------------------------------------------------
|
|
360
361
|
# Producer side (any number of threads) - forwards to the inner queue
|
|
@@ -457,15 +458,11 @@ class ProducerQueue(_Producer):
|
|
|
457
458
|
|
|
458
459
|
def unlink(self) -> None:
|
|
459
460
|
"""Unlink (delete) the underlying shared memory segment, if any. Valid
|
|
460
|
-
before or after close(), and a no-op when repeated
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
self.
|
|
465
|
-
try:
|
|
466
|
-
shm.unlink()
|
|
467
|
-
except Exception:
|
|
468
|
-
pass
|
|
461
|
+
before or after close(), and a no-op when repeated (see
|
|
462
|
+
SlickQueue.unlink()). The once-only guard lives on the queue, so an
|
|
463
|
+
adopted queue registered with several multiplexers is still unlinked
|
|
464
|
+
once."""
|
|
465
|
+
self._queue.unlink()
|
|
469
466
|
|
|
470
467
|
def _dereference(self, sequence: int) -> MultiplexRecord:
|
|
471
468
|
"""Dereference a shared-queue record into this producer's queue. Falsy
|
|
@@ -552,10 +549,6 @@ class StreamBufferMultiplexer:
|
|
|
552
549
|
raise ValueError("shared_queue_size or name must be provided")
|
|
553
550
|
self._shared_queue = SlickQueue(name=name, element_size=RECORD_SIZE,
|
|
554
551
|
traits=traits)
|
|
555
|
-
# the queue's SharedMemory, held for unlink(): SlickQueue.close() drops
|
|
556
|
-
# its own handle, after which SlickQueue.unlink() silently does nothing -
|
|
557
|
-
# yet close-then-unlink is the documented order. None for local memory.
|
|
558
|
-
self._shared_queue_shm = getattr(self._shared_queue, '_shm', None)
|
|
559
552
|
|
|
560
553
|
# One table per kind - what get_producer_buffers() / get_producer_queues()
|
|
561
554
|
# hand out - plus one of every producer for read(), the counterpart of the
|
|
@@ -586,15 +579,21 @@ class StreamBufferMultiplexer:
|
|
|
586
579
|
|
|
587
580
|
@staticmethod
|
|
588
581
|
def remove(name: str) -> bool:
|
|
589
|
-
"""Remove a shared memory segment by name
|
|
582
|
+
"""Remove a shared memory segment by name: the explicit cleanup step, and
|
|
583
|
+
the stale-segment recovery step.
|
|
590
584
|
|
|
591
585
|
Applies to every kind of name this class creates - a shared record
|
|
592
586
|
queue's, a producer stream buffer's and a producer queue's - since all
|
|
593
|
-
are plain shared memory segments underneath.
|
|
594
|
-
|
|
595
|
-
that
|
|
596
|
-
|
|
597
|
-
|
|
587
|
+
are plain shared memory segments underneath. Nothing unlinks a segment
|
|
588
|
+
on close() or process exit, so on POSIX a name outlives every process
|
|
589
|
+
that used it and the next creator attaches to it instead of starting
|
|
590
|
+
fresh. Call this from whichever process coordinates the lifetime: at
|
|
591
|
+
startup for a clean start, or at shutdown once no peer will attach
|
|
592
|
+
again. It also clears a segment a previous run left wedged by dying
|
|
593
|
+
mid-initialization, which construction over that name reports by
|
|
594
|
+
raising::
|
|
595
|
+
|
|
596
|
+
StreamBufferMultiplexer.remove("md_records") # clear what the last run left
|
|
598
597
|
mux = StreamBufferMultiplexer(shared_queue_size=1024, name="md_records")
|
|
599
598
|
|
|
600
599
|
Returns:
|
|
@@ -845,6 +844,10 @@ class StreamBufferMultiplexer:
|
|
|
845
844
|
"""Unlink (delete) the shared record queue segment and all registered
|
|
846
845
|
producers' segments, of both kinds.
|
|
847
846
|
|
|
847
|
+
No multiplexer does this on its own - not close(), not the context
|
|
848
|
+
manager and not process exit, as in C++ - so call it from the process
|
|
849
|
+
that coordinates the segments' lifetime, once no peer will attach again.
|
|
850
|
+
|
|
848
851
|
Valid before or after close(); the documented order is the
|
|
849
852
|
``shared_memory`` idiom - close every handle, then unlink once. Deleting
|
|
850
853
|
a name never touches mappings that already exist. Unlinking twice is a
|
|
@@ -856,12 +859,8 @@ class StreamBufferMultiplexer:
|
|
|
856
859
|
producer.unlink() # by saved name, once - valid after close()
|
|
857
860
|
except Exception:
|
|
858
861
|
pass
|
|
859
|
-
shm = self._shared_queue_shm
|
|
860
|
-
if shm is None:
|
|
861
|
-
return
|
|
862
|
-
self._shared_queue_shm = None # unlink once, however many times this is called
|
|
863
862
|
try:
|
|
864
|
-
|
|
863
|
+
self._shared_queue.unlink() # likewise
|
|
865
864
|
except Exception:
|
|
866
865
|
pass
|
|
867
866
|
|
|
@@ -521,6 +521,147 @@ def test_shm_unlink_after_close_covers_queue_producers():
|
|
|
521
521
|
successor.unlink()
|
|
522
522
|
|
|
523
523
|
|
|
524
|
+
# ---------------------------------------------------------------------------
|
|
525
|
+
# Segment lifetime: nothing unlinks a segment on its own (C++ v2.1.0+)
|
|
526
|
+
# ---------------------------------------------------------------------------
|
|
527
|
+
|
|
528
|
+
def _remove_all(names) -> None:
|
|
529
|
+
for n in names:
|
|
530
|
+
StreamBufferMultiplexer.remove(n)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _read_all(q: str, p0: str, q1: str):
|
|
534
|
+
"""Open the record queue and both producers by name and read every message."""
|
|
535
|
+
mux = StreamBufferMultiplexer(name=q)
|
|
536
|
+
try:
|
|
537
|
+
mux.add_producer(0, name=p0)
|
|
538
|
+
mux.add_queue_producer(1, name=q1)
|
|
539
|
+
out = []
|
|
540
|
+
cursor = 0
|
|
541
|
+
while True:
|
|
542
|
+
rec, cursor = mux.read(cursor)
|
|
543
|
+
if not rec:
|
|
544
|
+
return out
|
|
545
|
+
out.append((rec.producer_id, bytes(rec.data)))
|
|
546
|
+
finally:
|
|
547
|
+
mux.close()
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
# A fresh interpreter, so it has its own resource tracker - a multiprocessing
|
|
551
|
+
# child would share the parent's, and the parent's exit would be what unlinked
|
|
552
|
+
# the names.
|
|
553
|
+
_LIFETIME_CHILD = r"""
|
|
554
|
+
import sys
|
|
555
|
+
from pathlib import Path
|
|
556
|
+
root = Path(sys.argv[1])
|
|
557
|
+
for p in (root, root.parent / "slick-queue-py", root.parent / "slick-stream-buffer-py"):
|
|
558
|
+
sys.path.insert(0, str(p))
|
|
559
|
+
from slick_stream_buffer_multiplexer_py import StreamBufferMultiplexer
|
|
560
|
+
q, p0, q1, mode, tag = sys.argv[2:7]
|
|
561
|
+
if mode == 'create':
|
|
562
|
+
mux = StreamBufferMultiplexer(shared_queue_size=64, name=q)
|
|
563
|
+
pb = mux.add_producer(0, capacity=1024, control_size=16, name=p0)
|
|
564
|
+
pq = mux.add_queue_producer(1, size=1024, items_per_slot=16, name=q1)
|
|
565
|
+
assert pb.own_buffer and pq.own_buffer, "child expected to create the segments"
|
|
566
|
+
else:
|
|
567
|
+
mux = StreamBufferMultiplexer(name=q)
|
|
568
|
+
pb = mux.add_producer(0, name=p0)
|
|
569
|
+
pq = mux.add_queue_producer(1, name=q1)
|
|
570
|
+
data = tag.encode()
|
|
571
|
+
mv = pb.prepare(len(data))
|
|
572
|
+
mv[:] = data
|
|
573
|
+
pb.commit(len(data))
|
|
574
|
+
pb.consume(len(data))
|
|
575
|
+
i = pq.reserve(len(data))
|
|
576
|
+
pq[i][:len(data)] = data
|
|
577
|
+
pq.publish(i, len(data))
|
|
578
|
+
mux.close()
|
|
579
|
+
"""
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _run_lifetime_child(q: str, p0: str, q1: str, mode: str, tag: str) -> None:
|
|
583
|
+
import subprocess
|
|
584
|
+
result = subprocess.run(
|
|
585
|
+
[sys.executable, "-c", _LIFETIME_CHILD, str(_root), q, p0, q1, mode, tag],
|
|
586
|
+
capture_output=True, text=True, timeout=60)
|
|
587
|
+
assert result.returncode == 0, f"child failed:\n{result.stdout}\n{result.stderr}"
|
|
588
|
+
# The tracker warns about every registered name it cleans up at shutdown
|
|
589
|
+
assert "resource_tracker" not in result.stderr, result.stderr
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def test_shm_unlink_leaves_resource_tracker_alone():
|
|
593
|
+
"""Regression: unlink() deleted the record queue and each queue producer's
|
|
594
|
+
segment through SharedMemory.unlink(), which before Python 3.13 also sends
|
|
595
|
+
the resource tracker an UNREGISTER. slick-queue-py keeps no registration
|
|
596
|
+
since 2.3.0, so the tracker raised KeyError and printed a traceback from its
|
|
597
|
+
own process. Routing through SlickQueue.unlink() sends nothing."""
|
|
598
|
+
from multiprocessing import resource_tracker
|
|
599
|
+
names = ("mux_py_ut_records", "mux_py_ut_p0", "mux_py_ut_q1")
|
|
600
|
+
_remove_all(names)
|
|
601
|
+
mux = StreamBufferMultiplexer(shared_queue_size=64, name=names[0])
|
|
602
|
+
mux.add_producer(0, capacity=1024, control_size=16, name=names[1])
|
|
603
|
+
mux.add_queue_producer(1, size=1024, items_per_slot=16, name=names[2])
|
|
604
|
+
mux.close()
|
|
605
|
+
|
|
606
|
+
calls = []
|
|
607
|
+
original = resource_tracker.unregister
|
|
608
|
+
resource_tracker.unregister = lambda n, rtype: calls.append((n, rtype))
|
|
609
|
+
try:
|
|
610
|
+
mux.unlink()
|
|
611
|
+
finally:
|
|
612
|
+
resource_tracker.unregister = original
|
|
613
|
+
assert calls == [], calls
|
|
614
|
+
|
|
615
|
+
if sys.platform != 'win32':
|
|
616
|
+
leaked = [n for n in names if StreamBufferMultiplexer.remove(n)]
|
|
617
|
+
assert not leaked, f"segments outlived close() + unlink(): {leaked}"
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def test_shm_creator_exit_keeps_segments():
|
|
621
|
+
"""A creator's exit used to unlink its names on POSIX through the resource
|
|
622
|
+
tracker - the record queue and both kinds of producer segment - stranding
|
|
623
|
+
every consumer still attached. On Windows a section dies with its last
|
|
624
|
+
handle, which no one else holds here."""
|
|
625
|
+
if sys.platform == 'win32':
|
|
626
|
+
return
|
|
627
|
+
names = ("mux_py_cx_records", "mux_py_cx_p0", "mux_py_cx_q1")
|
|
628
|
+
_remove_all(names)
|
|
629
|
+
try:
|
|
630
|
+
_run_lifetime_child(*names, 'create', "from child")
|
|
631
|
+
assert _read_all(*names) == [(0, b"from child"), (1, b"from child")]
|
|
632
|
+
|
|
633
|
+
# A restarted creator reattaches to all three and keeps their contents
|
|
634
|
+
again = StreamBufferMultiplexer(shared_queue_size=64, name=names[0])
|
|
635
|
+
try:
|
|
636
|
+
assert not again.shared_queue._own
|
|
637
|
+
pb = again.add_producer(0, capacity=1024, control_size=16, name=names[1])
|
|
638
|
+
pq = again.add_queue_producer(1, size=1024, items_per_slot=16, name=names[2])
|
|
639
|
+
assert not pb.own_buffer
|
|
640
|
+
assert not pq.own_buffer
|
|
641
|
+
finally:
|
|
642
|
+
again.close()
|
|
643
|
+
finally:
|
|
644
|
+
_remove_all(names)
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def test_shm_attacher_exit_keeps_segments():
|
|
648
|
+
"""Before Python 3.13 attaching registered every name with the attacher's
|
|
649
|
+
resource tracker too, so any peer's exit unlinked the segments on POSIX."""
|
|
650
|
+
names = ("mux_py_ax_records", "mux_py_ax_p0", "mux_py_ax_q1")
|
|
651
|
+
_remove_all(names)
|
|
652
|
+
creator = StreamBufferMultiplexer(shared_queue_size=64, name=names[0])
|
|
653
|
+
try:
|
|
654
|
+
creator.add_producer(0, capacity=1024, control_size=16, name=names[1])
|
|
655
|
+
creator.add_queue_producer(1, size=1024, items_per_slot=16, name=names[2])
|
|
656
|
+
_run_lifetime_child(*names, 'open', "first")
|
|
657
|
+
_run_lifetime_child(*names, 'open', "second")
|
|
658
|
+
assert _read_all(*names) == [
|
|
659
|
+
(0, b"first"), (1, b"first"), (0, b"second"), (1, b"second")]
|
|
660
|
+
finally:
|
|
661
|
+
creator.close()
|
|
662
|
+
creator.unlink()
|
|
663
|
+
|
|
664
|
+
|
|
524
665
|
def run_all_tests():
|
|
525
666
|
tests = [
|
|
526
667
|
("ShmRoundtrip", test_shm_roundtrip),
|
|
@@ -536,6 +677,10 @@ def run_all_tests():
|
|
|
536
677
|
("ShmQueueCrossProcessRoundtrip", test_shm_queue_cross_process_roundtrip),
|
|
537
678
|
("ShmUnlinkAfterCloseCoversQueueProducers",
|
|
538
679
|
test_shm_unlink_after_close_covers_queue_producers),
|
|
680
|
+
("ShmUnlinkLeavesResourceTrackerAlone",
|
|
681
|
+
test_shm_unlink_leaves_resource_tracker_alone),
|
|
682
|
+
("ShmCreatorExitKeepsSegments", test_shm_creator_exit_keeps_segments),
|
|
683
|
+
("ShmAttacherExitKeepsSegments", test_shm_attacher_exit_keeps_segments),
|
|
539
684
|
]
|
|
540
685
|
|
|
541
686
|
print("=" * 70)
|
{slick_stream_buffer_multiplexer_py-2.2.0 → slick_stream_buffer_multiplexer_py-2.2.0.post1}/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|