slick-stream-buffer-multiplexer-py 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slick_stream_buffer_multiplexer_py-0.1.0/API_DIFFERENCES.md +78 -0
- slick_stream_buffer_multiplexer_py-0.1.0/LICENSE +21 -0
- slick_stream_buffer_multiplexer_py-0.1.0/MANIFEST.in +26 -0
- slick_stream_buffer_multiplexer_py-0.1.0/PKG-INFO +216 -0
- slick_stream_buffer_multiplexer_py-0.1.0/README.md +179 -0
- slick_stream_buffer_multiplexer_py-0.1.0/pyproject.toml +47 -0
- slick_stream_buffer_multiplexer_py-0.1.0/requirements.txt +3 -0
- slick_stream_buffer_multiplexer_py-0.1.0/setup.cfg +4 -0
- slick_stream_buffer_multiplexer_py-0.1.0/setup.py +60 -0
- slick_stream_buffer_multiplexer_py-0.1.0/slick_stream_buffer_multiplexer_py.egg-info/SOURCES.txt +12 -0
- slick_stream_buffer_multiplexer_py-0.1.0/slick_stream_buffer_multiplexer_py.py +407 -0
- slick_stream_buffer_multiplexer_py-0.1.0/tests/test_interop.py +321 -0
- slick_stream_buffer_multiplexer_py-0.1.0/tests/test_multiplexer.py +252 -0
- slick_stream_buffer_multiplexer_py-0.1.0/tests/test_multiplexer_mpmc.py +236 -0
- slick_stream_buffer_multiplexer_py-0.1.0/tests/test_multiplexer_shm.py +292 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# API Differences: Python vs C++
|
|
2
|
+
|
|
3
|
+
The Python implementation is binary-compatible with the C++
|
|
4
|
+
`slick::stream_buffer_multiplexer` at the shared memory level — the shared record queue
|
|
5
|
+
('SLQ1', 16-byte elements) and every producer segment ('SSB1') match byte-for-byte. The
|
|
6
|
+
*language-level* API differs in the following ways.
|
|
7
|
+
|
|
8
|
+
## Construction
|
|
9
|
+
|
|
10
|
+
| C++ | Python |
|
|
11
|
+
|---|---|
|
|
12
|
+
| `stream_buffer_multiplexer mux(size)` | `StreamBufferMultiplexer(shared_queue_size=size)` |
|
|
13
|
+
| `stream_buffer_multiplexer mux(size, "name")` | `StreamBufferMultiplexer(shared_queue_size=size, name="name")` |
|
|
14
|
+
| `stream_buffer_multiplexer mux("name")` | `StreamBufferMultiplexer(name="name")` |
|
|
15
|
+
| `add_producer(id, cap, ctl)` | `add_producer(id, capacity=cap, control_size=ctl)` |
|
|
16
|
+
| `add_producer(id, cap, ctl, "name")` | `add_producer(id, capacity=cap, control_size=ctl, name="name")` |
|
|
17
|
+
| `add_producer(id, "name")` | `add_producer(id, name="name")` |
|
|
18
|
+
| throws `std::invalid_argument` (duplicate id, bad sizes) | raises `ValueError` |
|
|
19
|
+
| throws `std::runtime_error` (shm failures) | raises `RuntimeError`, `ValueError` (queue geometry mismatch), or `FileNotFoundError` (missing segment) |
|
|
20
|
+
|
|
21
|
+
## Consumer cursor is passed by value
|
|
22
|
+
|
|
23
|
+
```cpp
|
|
24
|
+
// C++
|
|
25
|
+
uint64_t cursor = 0;
|
|
26
|
+
auto rec = mux.read(cursor); // cursor updated in place
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
# Python
|
|
31
|
+
cursor = 0
|
|
32
|
+
rec, cursor = mux.read(cursor) # updated cursor returned
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
For the work-stealing overload, pass an `AtomicCursor` (from slick-queue-py); the second
|
|
36
|
+
return value is then the claimed shared-queue index (`-1` when nothing was available):
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
rec, idx = mux.read(shared_cursor)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Zero-copy vs copies
|
|
43
|
+
|
|
44
|
+
Same policy as slick-stream-buffer-py: `prepare()` and `data()` return zero-copy
|
|
45
|
+
memoryviews into the producer's ring; `read()` results (`MultiplexRecord.data`) and
|
|
46
|
+
`consume()` results are **bytes copies**, safe by default under the lossy overwrite
|
|
47
|
+
semantics.
|
|
48
|
+
|
|
49
|
+
## Producer buffer accessors
|
|
50
|
+
|
|
51
|
+
C++ exposes both `stream_buffer()` (reference) and `stream_buffer_ptr()`
|
|
52
|
+
(`std::shared_ptr`); Python has no owning/non-owning distinction, so there is a single
|
|
53
|
+
`producer.stream_buffer` property returning the underlying `SlickStreamBuffer`.
|
|
54
|
+
Similarly, `get_producer_buffer()` and `find_producer()` both return the same
|
|
55
|
+
`ProducerBuffer` object (or `None`).
|
|
56
|
+
|
|
57
|
+
## Producer lookup
|
|
58
|
+
|
|
59
|
+
C++ uses a dense vector for producer ids < 4096 with a hash-map fallback; Python uses a
|
|
60
|
+
plain dict for all ids (already O(1) — the dense-path micro-optimization is meaningless
|
|
61
|
+
in Python).
|
|
62
|
+
|
|
63
|
+
## loss_count()
|
|
64
|
+
|
|
65
|
+
C++ compiles the multiplexer-level counter out unless
|
|
66
|
+
`SLICK_STREAM_BUFFER_MULTIPLEXER_ENABLE_LOSS_DETECTION` is enabled (debug default);
|
|
67
|
+
Python always counts. Same for the shared queue's wrap-loss term (slick-queue-py always
|
|
68
|
+
counts). All counters are per-instance, not shared through the segments.
|
|
69
|
+
|
|
70
|
+
## Lifecycle
|
|
71
|
+
|
|
72
|
+
Python adds explicit lifecycle management that C++ handles via RAII:
|
|
73
|
+
|
|
74
|
+
- `close()` — detach from all producer segments and the queue segment
|
|
75
|
+
- `unlink()` — delete the queue segment and all registered producers' segments
|
|
76
|
+
- context manager support: `with StreamBufferMultiplexer(...) as mux: ...`
|
|
77
|
+
- `get_shm_name()` — on the multiplexer (queue segment) and on each `ProducerBuffer`,
|
|
78
|
+
the exact names to pass to C++ (`/`-prefixed on POSIX)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Slick Quant
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Include documentation
|
|
2
|
+
include README.md
|
|
3
|
+
include LICENSE
|
|
4
|
+
include API_DIFFERENCES.md
|
|
5
|
+
|
|
6
|
+
# Include packaging files
|
|
7
|
+
include setup.py
|
|
8
|
+
include pyproject.toml
|
|
9
|
+
include requirements.txt
|
|
10
|
+
|
|
11
|
+
# Exclude build artifacts
|
|
12
|
+
global-exclude *.pyc
|
|
13
|
+
global-exclude __pycache__
|
|
14
|
+
global-exclude *.so
|
|
15
|
+
global-exclude *.pyd
|
|
16
|
+
global-exclude *.dll
|
|
17
|
+
global-exclude .DS_Store
|
|
18
|
+
|
|
19
|
+
# Exclude test outputs
|
|
20
|
+
recursive-exclude tests *.txt
|
|
21
|
+
recursive-exclude tests __pycache__
|
|
22
|
+
|
|
23
|
+
# Exclude build directories
|
|
24
|
+
prune build
|
|
25
|
+
prune dist
|
|
26
|
+
prune *.egg-info
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: slick-stream-buffer-multiplexer-py
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lock-free MPMC byte stream multiplexer with C++ interoperability via shared memory
|
|
5
|
+
Home-page: https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py
|
|
6
|
+
Author: Slick Quant
|
|
7
|
+
Author-email: Slick Quant <slickquant@slickquant.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Homepage, https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py
|
|
10
|
+
Project-URL: Documentation, https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py#readme
|
|
11
|
+
Project-URL: Repository, https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py
|
|
12
|
+
Project-URL: Bug Tracker, https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py/issues
|
|
13
|
+
Keywords: stream-buffer,multiplexer,lock-free,atomic,shared-memory,ipc,multiprocessing,mpmc
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
26
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
27
|
+
Classifier: Operating System :: MacOS
|
|
28
|
+
Requires-Python: >=3.8
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Requires-Dist: slick-queue-py>=1.2.0
|
|
32
|
+
Requires-Dist: slick-stream-buffer-py>=0.1.0
|
|
33
|
+
Dynamic: author
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
Dynamic: requires-python
|
|
37
|
+
|
|
38
|
+
# slick-stream-buffer-multiplexer-py
|
|
39
|
+
|
|
40
|
+
A Python implementation of [slick-stream-buffer-multiplexer](https://github.com/SlickQuant/slick-stream-buffer-multiplexer) —
|
|
41
|
+
a lock-free multi-producer multi-consumer (MPMC) byte stream multiplexer with shared memory support.
|
|
42
|
+
|
|
43
|
+
**Maintains exact binary compatibility with the C++ version**: Python and C++ producers and
|
|
44
|
+
consumers can be mixed freely across processes, fanning into the same shared memory segments.
|
|
45
|
+
|
|
46
|
+
## Concept
|
|
47
|
+
|
|
48
|
+
Each producer owns its own [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer)
|
|
49
|
+
(an independently-sized byte ring with per-producer lap/loss detection). `consume()` additionally
|
|
50
|
+
publishes a small 16-byte `{sequence, producer_id}` record into one shared
|
|
51
|
+
[slick-queue](https://github.com/SlickQuant/slick-queue), which acts as the lock-free MPMC
|
|
52
|
+
fan-in / global ordering point. Consumers read from the shared queue and dereference each record
|
|
53
|
+
back into the matching producer's buffer.
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
Producer A ──prepare/commit/consume──▶ [StreamBuffer A] ──┐
|
|
57
|
+
Producer B ──prepare/commit/consume──▶ [StreamBuffer B] ──┼─▶ {sequence, producer_id}
|
|
58
|
+
Producer N ──prepare/commit/consume──▶ [StreamBuffer N] ──┘ 16-byte records
|
|
59
|
+
│
|
|
60
|
+
[shared SlickQueue]
|
|
61
|
+
│
|
|
62
|
+
consumers read(cursor) and dereference
|
|
63
|
+
into StreamBuffer[producer_id]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Producers and the shared queue each independently choose local memory or shared memory (IPC).
|
|
67
|
+
A cross-process consumer only registers the producer ids whose shared memory it can access;
|
|
68
|
+
records referencing other producer ids are silently skipped (not counted as loss).
|
|
69
|
+
|
|
70
|
+
## Requirements
|
|
71
|
+
|
|
72
|
+
- Python 3.8+
|
|
73
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 1.2.0
|
|
74
|
+
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 0.1.0
|
|
75
|
+
|
|
76
|
+
Both dependencies bundle a small C++ extension for `std::atomic` operations; this package itself
|
|
77
|
+
is pure Python.
|
|
78
|
+
|
|
79
|
+
## Installation
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install slick-queue-py slick-stream-buffer-py
|
|
83
|
+
pip install -e .
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Quick Start
|
|
87
|
+
|
|
88
|
+
### Producers (any process, any language)
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from slick_stream_buffer_multiplexer_py import StreamBufferMultiplexer
|
|
92
|
+
|
|
93
|
+
# create the shared record queue (the fan-in ordering point)
|
|
94
|
+
mux = StreamBufferMultiplexer(shared_queue_size=1 << 16, name="mux_records")
|
|
95
|
+
|
|
96
|
+
# each producer gets its own independently-sized stream buffer
|
|
97
|
+
feed_a = mux.add_producer(0, capacity=1 << 26, control_size=1 << 16, name="feed_a")
|
|
98
|
+
feed_b = mux.add_producer(1, capacity=1 << 20, control_size=1 << 10, name="feed_b")
|
|
99
|
+
|
|
100
|
+
mv = feed_a.prepare(64 * 1024) # contiguous writable memoryview (zero-copy)
|
|
101
|
+
n = sock.recv_into(mv) # write network bytes directly into the ring
|
|
102
|
+
feed_a.commit(n)
|
|
103
|
+
feed_a.consume(n) # publish as ONE message + fan into the shared queue
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Consumer (opens everything created elsewhere — Python or C++)
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
mux = StreamBufferMultiplexer(name="mux_records") # open the shared record queue
|
|
110
|
+
mux.add_producer(0, name="feed_a") # register the producers to follow
|
|
111
|
+
mux.add_producer(1, name="feed_b") # (geometry read from each header)
|
|
112
|
+
|
|
113
|
+
cursor = mux.initial_reading_index() # or 0 to read history
|
|
114
|
+
while True:
|
|
115
|
+
rec, cursor = mux.read(cursor)
|
|
116
|
+
if not rec:
|
|
117
|
+
continue
|
|
118
|
+
handle_message(rec.producer_id, rec.data, rec.length)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Work-stealing consumers (each message to exactly one consumer)
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from slick_stream_buffer_multiplexer_py import AtomicCursor
|
|
125
|
+
from multiprocessing.shared_memory import SharedMemory
|
|
126
|
+
|
|
127
|
+
cursor_shm = SharedMemory(name="mux_cursor", create=True, size=8)
|
|
128
|
+
shared_cursor = AtomicCursor(cursor_shm.buf, 0)
|
|
129
|
+
|
|
130
|
+
rec, idx = mux.read(shared_cursor) # atomically claims the next message
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Three Loss Counters
|
|
134
|
+
|
|
135
|
+
| Counter | Meaning |
|
|
136
|
+
|---|---|
|
|
137
|
+
| `mux.loss_count()` | shared-queue wrap loss **plus** multiplexer-level loss (registered producers lapped before dereference) |
|
|
138
|
+
| `producer.loss_count()` | that producer's own inner-ring loss |
|
|
139
|
+
| unregistered producer ids | **never counted** — silently skipped |
|
|
140
|
+
|
|
141
|
+
## C++ Interoperability
|
|
142
|
+
|
|
143
|
+
The C++ side composes the identical primitives — either language can create any segment:
|
|
144
|
+
|
|
145
|
+
```cpp
|
|
146
|
+
#include <slick/stream_buffer_multiplexer.hpp>
|
|
147
|
+
|
|
148
|
+
slick::stream_buffer_multiplexer mux("mux_records"); // open queue created by Python
|
|
149
|
+
auto feed = mux.add_producer(0, "feed_a"); // open Python's producer segment
|
|
150
|
+
|
|
151
|
+
uint64_t cursor = mux.initial_reading_index();
|
|
152
|
+
for (;;) {
|
|
153
|
+
auto rec = mux.read(cursor);
|
|
154
|
+
if (!rec) continue;
|
|
155
|
+
handle_message(rec.producer_id, rec.data, rec.length);
|
|
156
|
+
}
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
Use `get_shm_name()` on the multiplexer (queue segment) and each producer buffer to get the
|
|
160
|
+
exact names to pass to C++ (POSIX names include the required leading `/`).
|
|
161
|
+
|
|
162
|
+
### Binary layout
|
|
163
|
+
|
|
164
|
+
The multiplexer adds no shared-memory structures of its own:
|
|
165
|
+
|
|
166
|
+
- The shared queue is a standard slick-queue segment (`'SLQ1'`, `element_size=16`) whose
|
|
167
|
+
elements are `uint64 sequence | uint32 producer_id | uint32 pad0`
|
|
168
|
+
- Each producer buffer is a standard slick-stream-buffer segment (`'SSB1'`)
|
|
169
|
+
|
|
170
|
+
## Caveats (same as C++)
|
|
171
|
+
|
|
172
|
+
- `add_producer()` is single-threaded setup: call it before producer/consumer threads start.
|
|
173
|
+
- Each producer's `prepare/commit/consume/...` must be called from a single thread.
|
|
174
|
+
- Lossy semantics: slow consumers skip overwritten data (see the loss counters above).
|
|
175
|
+
- A single message (one `consume()` call) is limited to < 4 GiB.
|
|
176
|
+
|
|
177
|
+
See [API_DIFFERENCES.md](API_DIFFERENCES.md) for exact deviations from the C++ API.
|
|
178
|
+
|
|
179
|
+
## Building and Testing
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
# Pure-Python tests (sibling repos slick-queue-py / slick-stream-buffer-py are
|
|
183
|
+
# picked up automatically from ../ if not pip-installed)
|
|
184
|
+
python tests/test_multiplexer.py
|
|
185
|
+
python tests/test_multiplexer_mpmc.py
|
|
186
|
+
python tests/test_multiplexer_shm.py
|
|
187
|
+
|
|
188
|
+
# C++ interop tests (requires CMake + a C++20 compiler)
|
|
189
|
+
cmake -S . -B build
|
|
190
|
+
cmake --build build --config Debug
|
|
191
|
+
cd build && ctest -C Debug --output-on-failure
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
The interop tests fetch slick-stream-buffer-multiplexer (which pulls slick-stream-buffer,
|
|
195
|
+
slick-queue, and slick-shm) from GitHub and build real C++ producer/consumer binaries against
|
|
196
|
+
the actual header. To build against local checkouts instead (no network):
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
cmake -S . -B build \
|
|
200
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER-MULTIPLEXER=/path/to/slick-stream-buffer-multiplexer \
|
|
201
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER=/path/to/slick-stream-buffer \
|
|
202
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-QUEUE=/path/to/slick-queue \
|
|
203
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-SHM=/path/to/slick-shm
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
If a failed test run leaves segments behind: `python tests/cleanup_shm.py`
|
|
207
|
+
|
|
208
|
+
## Related Projects
|
|
209
|
+
|
|
210
|
+
- [slick-stream-buffer-multiplexer](https://github.com/SlickQuant/slick-stream-buffer-multiplexer) — the C++ implementation
|
|
211
|
+
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) / [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) — the per-producer SPMC byte stream buffer
|
|
212
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) / [slick-queue](https://github.com/SlickQuant/slick-queue) — the shared MPMC record queue
|
|
213
|
+
|
|
214
|
+
## License
|
|
215
|
+
|
|
216
|
+
MIT
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# slick-stream-buffer-multiplexer-py
|
|
2
|
+
|
|
3
|
+
A Python implementation of [slick-stream-buffer-multiplexer](https://github.com/SlickQuant/slick-stream-buffer-multiplexer) —
|
|
4
|
+
a lock-free multi-producer multi-consumer (MPMC) byte stream multiplexer with shared memory support.
|
|
5
|
+
|
|
6
|
+
**Maintains exact binary compatibility with the C++ version**: Python and C++ producers and
|
|
7
|
+
consumers can be mixed freely across processes, fanning into the same shared memory segments.
|
|
8
|
+
|
|
9
|
+
## Concept
|
|
10
|
+
|
|
11
|
+
Each producer owns its own [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer)
|
|
12
|
+
(an independently-sized byte ring with per-producer lap/loss detection). `consume()` additionally
|
|
13
|
+
publishes a small 16-byte `{sequence, producer_id}` record into one shared
|
|
14
|
+
[slick-queue](https://github.com/SlickQuant/slick-queue), which acts as the lock-free MPMC
|
|
15
|
+
fan-in / global ordering point. Consumers read from the shared queue and dereference each record
|
|
16
|
+
back into the matching producer's buffer.
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
Producer A ──prepare/commit/consume──▶ [StreamBuffer A] ──┐
|
|
20
|
+
Producer B ──prepare/commit/consume──▶ [StreamBuffer B] ──┼─▶ {sequence, producer_id}
|
|
21
|
+
Producer N ──prepare/commit/consume──▶ [StreamBuffer N] ──┘ 16-byte records
|
|
22
|
+
│
|
|
23
|
+
[shared SlickQueue]
|
|
24
|
+
│
|
|
25
|
+
consumers read(cursor) and dereference
|
|
26
|
+
into StreamBuffer[producer_id]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Producers and the shared queue each independently choose local memory or shared memory (IPC).
|
|
30
|
+
A cross-process consumer only registers the producer ids whose shared memory it can access;
|
|
31
|
+
records referencing other producer ids are silently skipped (not counted as loss).
|
|
32
|
+
|
|
33
|
+
## Requirements
|
|
34
|
+
|
|
35
|
+
- Python 3.8+
|
|
36
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) >= 1.2.0
|
|
37
|
+
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) >= 0.1.0
|
|
38
|
+
|
|
39
|
+
Both dependencies bundle a small C++ extension for `std::atomic` operations; this package itself
|
|
40
|
+
is pure Python.
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install slick-queue-py slick-stream-buffer-py
|
|
46
|
+
pip install -e .
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Quick Start
|
|
50
|
+
|
|
51
|
+
### Producers (any process, any language)
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from slick_stream_buffer_multiplexer_py import StreamBufferMultiplexer
|
|
55
|
+
|
|
56
|
+
# create the shared record queue (the fan-in ordering point)
|
|
57
|
+
mux = StreamBufferMultiplexer(shared_queue_size=1 << 16, name="mux_records")
|
|
58
|
+
|
|
59
|
+
# each producer gets its own independently-sized stream buffer
|
|
60
|
+
feed_a = mux.add_producer(0, capacity=1 << 26, control_size=1 << 16, name="feed_a")
|
|
61
|
+
feed_b = mux.add_producer(1, capacity=1 << 20, control_size=1 << 10, name="feed_b")
|
|
62
|
+
|
|
63
|
+
mv = feed_a.prepare(64 * 1024) # contiguous writable memoryview (zero-copy)
|
|
64
|
+
n = sock.recv_into(mv) # write network bytes directly into the ring
|
|
65
|
+
feed_a.commit(n)
|
|
66
|
+
feed_a.consume(n) # publish as ONE message + fan into the shared queue
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### Consumer (opens everything created elsewhere — Python or C++)
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
mux = StreamBufferMultiplexer(name="mux_records") # open the shared record queue
|
|
73
|
+
mux.add_producer(0, name="feed_a") # register the producers to follow
|
|
74
|
+
mux.add_producer(1, name="feed_b") # (geometry read from each header)
|
|
75
|
+
|
|
76
|
+
cursor = mux.initial_reading_index() # or 0 to read history
|
|
77
|
+
while True:
|
|
78
|
+
rec, cursor = mux.read(cursor)
|
|
79
|
+
if not rec:
|
|
80
|
+
continue
|
|
81
|
+
handle_message(rec.producer_id, rec.data, rec.length)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Work-stealing consumers (each message to exactly one consumer)
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from slick_stream_buffer_multiplexer_py import AtomicCursor
|
|
88
|
+
from multiprocessing.shared_memory import SharedMemory
|
|
89
|
+
|
|
90
|
+
cursor_shm = SharedMemory(name="mux_cursor", create=True, size=8)
|
|
91
|
+
shared_cursor = AtomicCursor(cursor_shm.buf, 0)
|
|
92
|
+
|
|
93
|
+
rec, idx = mux.read(shared_cursor) # atomically claims the next message
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Three Loss Counters
|
|
97
|
+
|
|
98
|
+
| Counter | Meaning |
|
|
99
|
+
|---|---|
|
|
100
|
+
| `mux.loss_count()` | shared-queue wrap loss **plus** multiplexer-level loss (registered producers lapped before dereference) |
|
|
101
|
+
| `producer.loss_count()` | that producer's own inner-ring loss |
|
|
102
|
+
| unregistered producer ids | **never counted** — silently skipped |
|
|
103
|
+
|
|
104
|
+
## C++ Interoperability
|
|
105
|
+
|
|
106
|
+
The C++ side composes the identical primitives — either language can create any segment:
|
|
107
|
+
|
|
108
|
+
```cpp
|
|
109
|
+
#include <slick/stream_buffer_multiplexer.hpp>
|
|
110
|
+
|
|
111
|
+
slick::stream_buffer_multiplexer mux("mux_records"); // open queue created by Python
|
|
112
|
+
auto feed = mux.add_producer(0, "feed_a"); // open Python's producer segment
|
|
113
|
+
|
|
114
|
+
uint64_t cursor = mux.initial_reading_index();
|
|
115
|
+
for (;;) {
|
|
116
|
+
auto rec = mux.read(cursor);
|
|
117
|
+
if (!rec) continue;
|
|
118
|
+
handle_message(rec.producer_id, rec.data, rec.length);
|
|
119
|
+
}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Use `get_shm_name()` on the multiplexer (queue segment) and each producer buffer to get the
|
|
123
|
+
exact names to pass to C++ (POSIX names include the required leading `/`).
|
|
124
|
+
|
|
125
|
+
### Binary layout
|
|
126
|
+
|
|
127
|
+
The multiplexer adds no shared-memory structures of its own:
|
|
128
|
+
|
|
129
|
+
- The shared queue is a standard slick-queue segment (`'SLQ1'`, `element_size=16`) whose
|
|
130
|
+
elements are `uint64 sequence | uint32 producer_id | uint32 pad0`
|
|
131
|
+
- Each producer buffer is a standard slick-stream-buffer segment (`'SSB1'`)
|
|
132
|
+
|
|
133
|
+
## Caveats (same as C++)
|
|
134
|
+
|
|
135
|
+
- `add_producer()` is single-threaded setup: call it before producer/consumer threads start.
|
|
136
|
+
- Each producer's `prepare/commit/consume/...` must be called from a single thread.
|
|
137
|
+
- Lossy semantics: slow consumers skip overwritten data (see the loss counters above).
|
|
138
|
+
- A single message (one `consume()` call) is limited to < 4 GiB.
|
|
139
|
+
|
|
140
|
+
See [API_DIFFERENCES.md](API_DIFFERENCES.md) for exact deviations from the C++ API.
|
|
141
|
+
|
|
142
|
+
## Building and Testing
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
# Pure-Python tests (sibling repos slick-queue-py / slick-stream-buffer-py are
|
|
146
|
+
# picked up automatically from ../ if not pip-installed)
|
|
147
|
+
python tests/test_multiplexer.py
|
|
148
|
+
python tests/test_multiplexer_mpmc.py
|
|
149
|
+
python tests/test_multiplexer_shm.py
|
|
150
|
+
|
|
151
|
+
# C++ interop tests (requires CMake + a C++20 compiler)
|
|
152
|
+
cmake -S . -B build
|
|
153
|
+
cmake --build build --config Debug
|
|
154
|
+
cd build && ctest -C Debug --output-on-failure
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The interop tests fetch slick-stream-buffer-multiplexer (which pulls slick-stream-buffer,
|
|
158
|
+
slick-queue, and slick-shm) from GitHub and build real C++ producer/consumer binaries against
|
|
159
|
+
the actual header. To build against local checkouts instead (no network):
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
cmake -S . -B build \
|
|
163
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER-MULTIPLEXER=/path/to/slick-stream-buffer-multiplexer \
|
|
164
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER=/path/to/slick-stream-buffer \
|
|
165
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-QUEUE=/path/to/slick-queue \
|
|
166
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-SHM=/path/to/slick-shm
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
If a failed test run leaves segments behind: `python tests/cleanup_shm.py`
|
|
170
|
+
|
|
171
|
+
## Related Projects
|
|
172
|
+
|
|
173
|
+
- [slick-stream-buffer-multiplexer](https://github.com/SlickQuant/slick-stream-buffer-multiplexer) — the C++ implementation
|
|
174
|
+
- [slick-stream-buffer-py](https://github.com/SlickQuant/slick-stream-buffer-py) / [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) — the per-producer SPMC byte stream buffer
|
|
175
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) / [slick-queue](https://github.com/SlickQuant/slick-queue) — the shared MPMC record queue
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
MIT
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "slick-stream-buffer-multiplexer-py"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Lock-free MPMC byte stream multiplexer with C++ interoperability via shared memory"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Slick Quant", email = "slickquant@slickquant.com"}
|
|
14
|
+
]
|
|
15
|
+
keywords = ["stream-buffer", "multiplexer", "lock-free", "atomic", "shared-memory", "ipc", "multiprocessing", "mpmc"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
20
|
+
"Topic :: System :: Distributed Computing",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.8",
|
|
24
|
+
"Programming Language :: Python :: 3.9",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Operating System :: Microsoft :: Windows",
|
|
29
|
+
"Operating System :: POSIX :: Linux",
|
|
30
|
+
"Operating System :: MacOS",
|
|
31
|
+
]
|
|
32
|
+
dependencies = [
|
|
33
|
+
"slick-queue-py>=1.2.0",
|
|
34
|
+
"slick-stream-buffer-py>=0.1.0",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py"
|
|
39
|
+
Documentation = "https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py#readme"
|
|
40
|
+
Repository = "https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py"
|
|
41
|
+
"Bug Tracker" = "https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py/issues"
|
|
42
|
+
|
|
43
|
+
[tool.setuptools]
|
|
44
|
+
py-modules = ["slick_stream_buffer_multiplexer_py"]
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.dynamic]
|
|
47
|
+
version = {attr = "slick_stream_buffer_multiplexer_py.__version__"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from setuptools import setup
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
# Read version from slick_stream_buffer_multiplexer_py.py
|
|
6
|
+
def get_version():
|
|
7
|
+
with open('slick_stream_buffer_multiplexer_py.py', 'r', encoding='utf-8') as f:
|
|
8
|
+
content = f.read()
|
|
9
|
+
match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", content, re.M)
|
|
10
|
+
if match:
|
|
11
|
+
return match.group(1)
|
|
12
|
+
raise RuntimeError("Unable to find version string.")
|
|
13
|
+
|
|
14
|
+
# Read long description from README
|
|
15
|
+
def get_long_description():
|
|
16
|
+
readme_path = Path(__file__).parent / 'README.md'
|
|
17
|
+
if readme_path.exists():
|
|
18
|
+
with open(readme_path, 'r', encoding='utf-8') as f:
|
|
19
|
+
return f.read()
|
|
20
|
+
return ''
|
|
21
|
+
|
|
22
|
+
setup(
|
|
23
|
+
name='slick-stream-buffer-multiplexer-py',
|
|
24
|
+
version=get_version(),
|
|
25
|
+
description='Lock-free MPMC byte stream multiplexer with C++ interoperability via shared memory',
|
|
26
|
+
long_description=get_long_description(),
|
|
27
|
+
long_description_content_type='text/markdown',
|
|
28
|
+
author='Slick Quant',
|
|
29
|
+
author_email='slickquant@slickquant.com',
|
|
30
|
+
url='https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py',
|
|
31
|
+
project_urls={
|
|
32
|
+
'Bug Tracker': 'https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py/issues',
|
|
33
|
+
'Documentation': 'https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py#readme',
|
|
34
|
+
'Source Code': 'https://github.com/SlickQuant/slick-stream-buffer-multiplexer-py',
|
|
35
|
+
},
|
|
36
|
+
classifiers=[
|
|
37
|
+
'Development Status :: 4 - Beta',
|
|
38
|
+
'Intended Audience :: Developers',
|
|
39
|
+
'Topic :: Software Development :: Libraries :: Python Modules',
|
|
40
|
+
'Topic :: System :: Distributed Computing',
|
|
41
|
+
'License :: OSI Approved :: MIT License',
|
|
42
|
+
'Programming Language :: Python :: 3',
|
|
43
|
+
'Programming Language :: Python :: 3.8',
|
|
44
|
+
'Programming Language :: Python :: 3.9',
|
|
45
|
+
'Programming Language :: Python :: 3.10',
|
|
46
|
+
'Programming Language :: Python :: 3.11',
|
|
47
|
+
'Programming Language :: Python :: 3.12',
|
|
48
|
+
'Operating System :: Microsoft :: Windows',
|
|
49
|
+
'Operating System :: POSIX :: Linux',
|
|
50
|
+
'Operating System :: MacOS',
|
|
51
|
+
],
|
|
52
|
+
keywords='stream-buffer, multiplexer, lock-free, atomic, shared-memory, ipc, multiprocessing, mpmc',
|
|
53
|
+
py_modules=['slick_stream_buffer_multiplexer_py'],
|
|
54
|
+
install_requires=[
|
|
55
|
+
'slick-queue-py>=1.2.0',
|
|
56
|
+
'slick-stream-buffer-py>=0.1.0',
|
|
57
|
+
],
|
|
58
|
+
python_requires='>=3.8',
|
|
59
|
+
license='MIT',
|
|
60
|
+
)
|
slick_stream_buffer_multiplexer_py-0.1.0/slick_stream_buffer_multiplexer_py.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
API_DIFFERENCES.md
|
|
2
|
+
LICENSE
|
|
3
|
+
MANIFEST.in
|
|
4
|
+
README.md
|
|
5
|
+
pyproject.toml
|
|
6
|
+
requirements.txt
|
|
7
|
+
setup.py
|
|
8
|
+
slick_stream_buffer_multiplexer_py.py
|
|
9
|
+
tests/test_interop.py
|
|
10
|
+
tests/test_multiplexer.py
|
|
11
|
+
tests/test_multiplexer_mpmc.py
|
|
12
|
+
tests/test_multiplexer_shm.py
|