slick-stream-buffer-py 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slick_stream_buffer_py-0.1.0/API_DIFFERENCES.md +94 -0
- slick_stream_buffer_py-0.1.0/LICENSE +21 -0
- slick_stream_buffer_py-0.1.0/MANIFEST.in +29 -0
- slick_stream_buffer_py-0.1.0/PKG-INFO +232 -0
- slick_stream_buffer_py-0.1.0/README.md +196 -0
- slick_stream_buffer_py-0.1.0/pyproject.toml +69 -0
- slick_stream_buffer_py-0.1.0/requirements.txt +2 -0
- slick_stream_buffer_py-0.1.0/setup.cfg +4 -0
- slick_stream_buffer_py-0.1.0/setup.py +68 -0
- slick_stream_buffer_py-0.1.0/slick_stream_buffer_py.egg-info/SOURCES.txt +14 -0
- slick_stream_buffer_py-0.1.0/slick_stream_buffer_py.py +724 -0
- slick_stream_buffer_py-0.1.0/ssb_atomic_ops.py +493 -0
- slick_stream_buffer_py-0.1.0/ssb_atomic_ops_ext.cpp +260 -0
- slick_stream_buffer_py-0.1.0/tests/test_atomic_ops.py +303 -0
- slick_stream_buffer_py-0.1.0/tests/test_interop.py +346 -0
- slick_stream_buffer_py-0.1.0/tests/test_local_mode.py +615 -0
- slick_stream_buffer_py-0.1.0/tests/test_shm_mode.py +372 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# API Differences: Python vs C++
|
|
2
|
+
|
|
3
|
+
The Python implementation is binary-compatible with the C++ `slick::SlickStreamBuffer`
|
|
4
|
+
at the shared memory level — every byte, offset, and memory ordering matches. The
|
|
5
|
+
*language-level* API differs in the following ways.
|
|
6
|
+
|
|
7
|
+
## Construction
|
|
8
|
+
|
|
9
|
+
| C++ | Python |
|
|
10
|
+
|---|---|
|
|
11
|
+
| `stream_buffer buf(capacity, control_size)` | `SlickStreamBuffer(capacity=..., control_size=...)` |
|
|
12
|
+
| `stream_buffer buf(capacity, control_size, "name")` | `SlickStreamBuffer(capacity=..., control_size=..., name="name")` |
|
|
13
|
+
| `stream_buffer buf("name")` | `SlickStreamBuffer(name="name")` |
|
|
14
|
+
| throws `std::invalid_argument` (bad sizes) | raises `ValueError` |
|
|
15
|
+
| throws `std::runtime_error` (shm failures) | raises `RuntimeError` (or `FileNotFoundError` when opening a missing segment) |
|
|
16
|
+
|
|
17
|
+
## Consumer cursor is passed by value
|
|
18
|
+
|
|
19
|
+
C++ updates the cursor through a reference; Python has no by-reference ints, so the
|
|
20
|
+
updated cursor is returned:
|
|
21
|
+
|
|
22
|
+
```cpp
|
|
23
|
+
// C++
|
|
24
|
+
uint64_t cursor = 0;
|
|
25
|
+
auto [data, length] = buf.read(cursor); // cursor updated in place
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
# Python
|
|
30
|
+
cursor = 0
|
|
31
|
+
data, length, cursor = buf.read(cursor) # updated cursor returned
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
`read()` returns `(None, 0, cursor)` where C++ returns `{nullptr, 0}`.
|
|
35
|
+
`read_last()` returns `(None, 0)` / `(bytes, length)`.
|
|
36
|
+
|
|
37
|
+
## Zero-copy vs copies
|
|
38
|
+
|
|
39
|
+
| Operation | C++ | Python |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `prepare(n)` | `uint8_t*` into the ring | writable `memoryview` into the ring (zero-copy) |
|
|
42
|
+
| `data()` | `const uint8_t*` into the ring | read-only `memoryview` into the ring (zero-copy) |
|
|
43
|
+
| `read(cursor)` | pointer into the ring | **`bytes` copy** |
|
|
44
|
+
| `read_last()` | pointer into the ring | **`bytes` copy** |
|
|
45
|
+
| `consume(n).data` | pointer into the ring | **`bytes` copy** |
|
|
46
|
+
|
|
47
|
+
Consumer-side data is copied because the ring is lossy: the producer may overwrite the
|
|
48
|
+
bytes at any moment after the validity check (the same caveat exists in C++, where the
|
|
49
|
+
caller must finish with the pointer before the producer laps). A `bytes` copy makes the
|
|
50
|
+
Python API safe by default.
|
|
51
|
+
|
|
52
|
+
`prepare()`/`data()` memoryviews are invalidated by the next `prepare()` (which may
|
|
53
|
+
relocate the region) — same semantics as C++ flat_buffer-style invalidation.
|
|
54
|
+
|
|
55
|
+
## prepare() overflow
|
|
56
|
+
|
|
57
|
+
C++ throws `std::length_error` when `size() + n > capacity()`; Python raises
|
|
58
|
+
`ValueError`. C++ `consume()` asserts messages are < 4 GiB (debug builds); Python
|
|
59
|
+
always raises `ValueError`.
|
|
60
|
+
|
|
61
|
+
## consume() return value
|
|
62
|
+
|
|
63
|
+
Both return the published record. Python's `PublishedRecord` mirrors C++
|
|
64
|
+
`published_record` (`sequence`, `data`, `length`) and is falsy when nothing was
|
|
65
|
+
published (`data is None`), matching C++ `operator bool`.
|
|
66
|
+
|
|
67
|
+
## loss_count()
|
|
68
|
+
|
|
69
|
+
C++ counts losses only when `SLICK_STREAM_BUFFER_ENABLE_LOSS_DETECTION` is enabled
|
|
70
|
+
(default: debug builds only). Python always counts. The counter is per-instance in
|
|
71
|
+
both implementations, not shared through the segment.
|
|
72
|
+
|
|
73
|
+
## Memory orderings
|
|
74
|
+
|
|
75
|
+
Python routes every shared-atomic access through C++ `std::atomic` (via the
|
|
76
|
+
`ssb_atomic_ops_ext` extension), using the same orderings on the synchronization
|
|
77
|
+
edges (record `seq` store-release / load-acquire, `next_seq_`/`reserve_end_`
|
|
78
|
+
release/acquire, `init_state` acq_rel CAS). Where the C++ side uses
|
|
79
|
+
`memory_order_relaxed` (e.g. `committed_`/`consumed_` stores), Python uses
|
|
80
|
+
release/acquire — strictly stronger, therefore still correct.
|
|
81
|
+
|
|
82
|
+
## Lifecycle
|
|
83
|
+
|
|
84
|
+
Python adds explicit lifecycle management that C++ handles via RAII:
|
|
85
|
+
|
|
86
|
+
- `close()` — detach from the segment (does not delete it)
|
|
87
|
+
- `unlink()` — delete the segment (call once, from the owner, after all users closed)
|
|
88
|
+
- context manager support: `with SlickStreamBuffer(...) as buf: ...` (closes on exit)
|
|
89
|
+
- `get_shm_name()` — the exact segment name to pass to C++ (`/`-prefixed on POSIX)
|
|
90
|
+
|
|
91
|
+
On POSIX with Python >= 3.13, attaching to an existing segment uses
|
|
92
|
+
`SharedMemory(track=False)` so Python's resource tracker never unlinks a segment the
|
|
93
|
+
process does not own. On older Pythons the tracker may log a harmless warning at exit
|
|
94
|
+
(see `tests/run_test.py`, which filters it).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Slick Quant
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Include documentation
|
|
2
|
+
include README.md
|
|
3
|
+
include LICENSE
|
|
4
|
+
include API_DIFFERENCES.md
|
|
5
|
+
|
|
6
|
+
# Include C++ source files for extension
|
|
7
|
+
include ssb_atomic_ops_ext.cpp
|
|
8
|
+
include setup.py
|
|
9
|
+
include pyproject.toml
|
|
10
|
+
|
|
11
|
+
# Include requirements
|
|
12
|
+
include requirements.txt
|
|
13
|
+
|
|
14
|
+
# Exclude build artifacts
|
|
15
|
+
global-exclude *.pyc
|
|
16
|
+
global-exclude __pycache__
|
|
17
|
+
global-exclude *.so
|
|
18
|
+
global-exclude *.pyd
|
|
19
|
+
global-exclude *.dll
|
|
20
|
+
global-exclude .DS_Store
|
|
21
|
+
|
|
22
|
+
# Exclude test outputs
|
|
23
|
+
recursive-exclude tests *.txt
|
|
24
|
+
recursive-exclude tests __pycache__
|
|
25
|
+
|
|
26
|
+
# Exclude build directories
|
|
27
|
+
prune build
|
|
28
|
+
prune dist
|
|
29
|
+
prune *.egg-info
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: slick-stream-buffer-py
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lock-free SPMC byte stream buffer with C++ interoperability via shared memory
|
|
5
|
+
Home-page: https://github.com/SlickQuant/slick-stream-buffer-py
|
|
6
|
+
Author: Slick Quant
|
|
7
|
+
Author-email: Slick Quant <slickquant@slickquant.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Homepage, https://github.com/SlickQuant/slick-stream-buffer-py
|
|
10
|
+
Project-URL: Documentation, https://github.com/SlickQuant/slick-stream-buffer-py#readme
|
|
11
|
+
Project-URL: Repository, https://github.com/SlickQuant/slick-stream-buffer-py
|
|
12
|
+
Project-URL: Bug Tracker, https://github.com/SlickQuant/slick-stream-buffer-py/issues
|
|
13
|
+
Keywords: stream-buffer,lock-free,atomic,shared-memory,ipc,multiprocessing,spmc
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Programming Language :: C++
|
|
26
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
27
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
28
|
+
Classifier: Operating System :: MacOS
|
|
29
|
+
Requires-Python: >=3.8
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Dynamic: author
|
|
33
|
+
Dynamic: home-page
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
Dynamic: requires-python
|
|
36
|
+
|
|
37
|
+
# slick-stream-buffer-py
|
|
38
|
+
|
|
39
|
+
A Python implementation of [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) —
|
|
40
|
+
a lock-free single-producer multi-consumer (SPMC) byte stream buffer with shared memory support.
|
|
41
|
+
|
|
42
|
+
**Maintains exact binary compatibility with the C++ version**: a Python producer and a C++
|
|
43
|
+
consumer (or vice versa) communicate seamlessly through the same shared memory segment, using
|
|
44
|
+
the same `std::atomic` synchronization primitives.
|
|
45
|
+
|
|
46
|
+
## Features
|
|
47
|
+
|
|
48
|
+
- **Byte stream, message records**: the producer writes raw bytes (`prepare`/`commit`) and
|
|
49
|
+
publishes them as discrete message records (`consume`), like a network receive buffer
|
|
50
|
+
feeding a framing layer
|
|
51
|
+
- **Lock-free SPMC**: one producer, any number of consumers, no locks anywhere
|
|
52
|
+
- **Binary-compatible with C++**: identical 64-byte header, 32-byte control records, and
|
|
53
|
+
acquire/release memory orderings (`slick/stream_buffer.hpp`, header magic `'SSB1'`)
|
|
54
|
+
- **Shared memory IPC**: built on `multiprocessing.shared_memory`, matching the C++
|
|
55
|
+
slick-shm naming conventions on Windows, Linux, and macOS
|
|
56
|
+
- **Lossy by design**: slow consumers skip overwritten data and count the loss instead of
|
|
57
|
+
blocking the producer
|
|
58
|
+
- **Local memory mode**: same API without shared memory for single-process use
|
|
59
|
+
- **No runtime dependencies**: Python 3.8+ standard library plus a small bundled C++
|
|
60
|
+
extension for the atomics
|
|
61
|
+
|
|
62
|
+
## Requirements
|
|
63
|
+
|
|
64
|
+
- Python 3.8+
|
|
65
|
+
- 64-bit platform (Windows x86-64, Linux x86-64/ARM64, macOS x86-64/ARM64)
|
|
66
|
+
- A C++17 compiler to build the `ssb_atomic_ops_ext` extension (MSVC 2017+, GCC 5+, Clang 3.8+)
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install -e .
|
|
72
|
+
# or just build the extension in place:
|
|
73
|
+
python setup.py build_ext --inplace
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Quick Start
|
|
77
|
+
|
|
78
|
+
### Producer (creates the shared memory segment)
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from slick_stream_buffer_py import SlickStreamBuffer
|
|
82
|
+
|
|
83
|
+
# 64 MB data ring, 65536 message records
|
|
84
|
+
stream = SlickStreamBuffer(capacity=1 << 26, control_size=1 << 16, name="market_data")
|
|
85
|
+
|
|
86
|
+
while True:
|
|
87
|
+
mv = stream.prepare(64 * 1024) # contiguous writable memoryview (zero-copy)
|
|
88
|
+
n = sock.recv_into(mv) # write network bytes directly into the ring
|
|
89
|
+
stream.commit(n)
|
|
90
|
+
|
|
91
|
+
# publish every complete package as one message record
|
|
92
|
+
while (package_size := find_complete_package(stream.data(), stream.size())):
|
|
93
|
+
stream.consume(package_size)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Consumer (opens the existing segment — can be C++ or Python)
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from slick_stream_buffer_py import SlickStreamBuffer
|
|
100
|
+
|
|
101
|
+
stream = SlickStreamBuffer(name="market_data") # geometry read from the segment header
|
|
102
|
+
|
|
103
|
+
cursor = stream.initial_reading_index() # skip history; use 0 to read from the start
|
|
104
|
+
while True:
|
|
105
|
+
data, length, cursor = stream.read(cursor)
|
|
106
|
+
if data is None:
|
|
107
|
+
continue
|
|
108
|
+
handle_package(data, length)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Local memory mode (single process)
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
buf = SlickStreamBuffer(capacity=1024, control_size=16)
|
|
115
|
+
|
|
116
|
+
mv = buf.prepare(5)
|
|
117
|
+
mv[:] = b"hello"
|
|
118
|
+
buf.commit(5)
|
|
119
|
+
buf.consume(5) # publish as one record
|
|
120
|
+
|
|
121
|
+
data, length, cursor = buf.read(0) # -> b"hello", 5, 1
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## C++ Interoperability
|
|
125
|
+
|
|
126
|
+
The C++ side uses the identical layout — either side can create the segment, the other
|
|
127
|
+
attaches to it:
|
|
128
|
+
|
|
129
|
+
```cpp
|
|
130
|
+
#include <slick/stream_buffer.hpp>
|
|
131
|
+
|
|
132
|
+
slick::stream_buffer stream("market_data"); // open segment created by Python
|
|
133
|
+
|
|
134
|
+
uint64_t cursor = stream.initial_reading_index();
|
|
135
|
+
for (;;) {
|
|
136
|
+
auto [data, length] = stream.read(cursor);
|
|
137
|
+
if (data == nullptr) continue;
|
|
138
|
+
handle_package(data, length);
|
|
139
|
+
}
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Shared memory naming: use `get_shm_name()` to obtain the exact name to pass to C++
|
|
143
|
+
(on POSIX it includes the leading `/` that `shm_open()` requires; on Windows it is the
|
|
144
|
+
raw name).
|
|
145
|
+
|
|
146
|
+
### Shared memory layout
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
[HEADER: 64 bytes]
|
|
150
|
+
0-7 atomic<uint64> committed_ - monotonic end of committed bytes
|
|
151
|
+
8-15 atomic<uint64> consumed_ - monotonic publish boundary
|
|
152
|
+
16-23 atomic<uint64> next_seq_ - next record sequence number
|
|
153
|
+
24-31 atomic<uint64> reserve_end_ - prepared-region high-water mark
|
|
154
|
+
32-39 uint64 capacity_ - data ring size in bytes (power of 2)
|
|
155
|
+
40-43 uint32 control_size_ - control ring record count (power of 2)
|
|
156
|
+
44-47 uint32 header_magic - 0x53534231 ('SSB1')
|
|
157
|
+
48-51 atomic<uint32> init_state - 0=uninit, 2=initializing, 3=ready
|
|
158
|
+
52-63 padding
|
|
159
|
+
[CONTROL RING: 32 bytes x control_size]
|
|
160
|
+
each record: atomic<uint64> seq | uint64 offset | uint32 length | 12 bytes padding
|
|
161
|
+
[DATA RING: capacity bytes]
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Total segment size: `64 + 32 * control_size + capacity`.
|
|
165
|
+
|
|
166
|
+
## API Summary
|
|
167
|
+
|
|
168
|
+
| Method | Role | Description |
|
|
169
|
+
|---|---|---|
|
|
170
|
+
| `prepare(n) -> memoryview` | producer | contiguous writable region; may relocate unconsumed bytes on wrap |
|
|
171
|
+
| `commit(n)` | producer | move n prepared bytes into the readable region (clamped) |
|
|
172
|
+
| `consume(n) -> PublishedRecord` | producer | publish the first n readable bytes as ONE record (clamped) |
|
|
173
|
+
| `discard()` | producer | drop unconsumed + prepared bytes without publishing |
|
|
174
|
+
| `data() -> memoryview` / `size()` | producer | the committed-but-unconsumed region |
|
|
175
|
+
| `read(cursor) -> (data, length, cursor)` | consumer | next record, or `(None, 0, cursor)`; skips lost records |
|
|
176
|
+
| `read_last() -> (data, length)` | consumer | newest record without a cursor |
|
|
177
|
+
| `initial_reading_index()` | consumer | cursor for a late joiner (skips history) |
|
|
178
|
+
| `loss_count()` | consumer | records skipped by this instance due to overwrite |
|
|
179
|
+
| `reset()` | owner | clear all state (not thread-safe) |
|
|
180
|
+
| `close()` / `unlink()` | lifecycle | detach / delete the segment |
|
|
181
|
+
|
|
182
|
+
See [API_DIFFERENCES.md](API_DIFFERENCES.md) for the exact deviations from the C++ API
|
|
183
|
+
(cursor passed by value, bytes copies vs pointers, exceptions).
|
|
184
|
+
|
|
185
|
+
### Caveats (same as C++)
|
|
186
|
+
|
|
187
|
+
- Producer methods (`prepare`/`commit`/`consume`/`discard`/`data`/`size`/`reset`) must be
|
|
188
|
+
called from a single thread.
|
|
189
|
+
- The buffer is lossy: if the producer outruns a consumer by more than the control ring or
|
|
190
|
+
data ring size, the consumer skips ahead and the loss is counted.
|
|
191
|
+
- `prepare()` may relocate the readable region: memoryviews previously returned by `data()`
|
|
192
|
+
or `prepare()` are invalidated.
|
|
193
|
+
- A single message (one `consume()` call) is limited to < 4 GiB.
|
|
194
|
+
|
|
195
|
+
## Building and Testing
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
# 1. Build the atomics extension
|
|
199
|
+
python setup.py build_ext --inplace
|
|
200
|
+
|
|
201
|
+
# 2. Pure-Python tests
|
|
202
|
+
python tests/test_atomic_ops.py
|
|
203
|
+
python tests/test_local_mode.py
|
|
204
|
+
python tests/test_shm_mode.py
|
|
205
|
+
|
|
206
|
+
# 3. C++ interop tests (requires CMake + a C++20 compiler)
|
|
207
|
+
cmake -S . -B build
|
|
208
|
+
cmake --build build --config Debug
|
|
209
|
+
cd build && ctest -C Debug --output-on-failure
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
The interop tests fetch [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer)
|
|
213
|
+
(which pulls [slick-shm](https://github.com/SlickQuant/slick-shm)) from GitHub and build real
|
|
214
|
+
C++ producer/consumer binaries against the actual `stream_buffer.hpp`. To build against local
|
|
215
|
+
checkouts instead (no network):
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
cmake -S . -B build \
|
|
219
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER=/path/to/slick-stream-buffer \
|
|
220
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-SHM=/path/to/slick-shm
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
If a failed test run leaves segments behind: `python tests/cleanup_shm.py`
|
|
224
|
+
|
|
225
|
+
## Related Projects
|
|
226
|
+
|
|
227
|
+
- [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) — the C++ implementation
|
|
228
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) / [slick-queue](https://github.com/SlickQuant/slick-queue) — MPMC fixed-element queue with the same interop approach
|
|
229
|
+
|
|
230
|
+
## License
|
|
231
|
+
|
|
232
|
+
MIT
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# slick-stream-buffer-py
|
|
2
|
+
|
|
3
|
+
A Python implementation of [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) —
|
|
4
|
+
a lock-free single-producer multi-consumer (SPMC) byte stream buffer with shared memory support.
|
|
5
|
+
|
|
6
|
+
**Maintains exact binary compatibility with the C++ version**: a Python producer and a C++
|
|
7
|
+
consumer (or vice versa) communicate seamlessly through the same shared memory segment, using
|
|
8
|
+
the same `std::atomic` synchronization primitives.
|
|
9
|
+
|
|
10
|
+
## Features
|
|
11
|
+
|
|
12
|
+
- **Byte stream, message records**: the producer writes raw bytes (`prepare`/`commit`) and
|
|
13
|
+
publishes them as discrete message records (`consume`), like a network receive buffer
|
|
14
|
+
feeding a framing layer
|
|
15
|
+
- **Lock-free SPMC**: one producer, any number of consumers, no locks anywhere
|
|
16
|
+
- **Binary-compatible with C++**: identical 64-byte header, 32-byte control records, and
|
|
17
|
+
acquire/release memory orderings (`slick/stream_buffer.hpp`, header magic `'SSB1'`)
|
|
18
|
+
- **Shared memory IPC**: built on `multiprocessing.shared_memory`, matching the C++
|
|
19
|
+
slick-shm naming conventions on Windows, Linux, and macOS
|
|
20
|
+
- **Lossy by design**: slow consumers skip overwritten data and count the loss instead of
|
|
21
|
+
blocking the producer
|
|
22
|
+
- **Local memory mode**: same API without shared memory for single-process use
|
|
23
|
+
- **No runtime dependencies**: Python 3.8+ standard library plus a small bundled C++
|
|
24
|
+
extension for the atomics
|
|
25
|
+
|
|
26
|
+
## Requirements
|
|
27
|
+
|
|
28
|
+
- Python 3.8+
|
|
29
|
+
- 64-bit platform (Windows x86-64, Linux x86-64/ARM64, macOS x86-64/ARM64)
|
|
30
|
+
- A C++17 compiler to build the `ssb_atomic_ops_ext` extension (MSVC 2017+, GCC 5+, Clang 3.8+)
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install -e .
|
|
36
|
+
# or just build the extension in place:
|
|
37
|
+
python setup.py build_ext --inplace
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Quick Start
|
|
41
|
+
|
|
42
|
+
### Producer (creates the shared memory segment)
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from slick_stream_buffer_py import SlickStreamBuffer
|
|
46
|
+
|
|
47
|
+
# 64 MB data ring, 65536 message records
|
|
48
|
+
stream = SlickStreamBuffer(capacity=1 << 26, control_size=1 << 16, name="market_data")
|
|
49
|
+
|
|
50
|
+
while True:
|
|
51
|
+
mv = stream.prepare(64 * 1024) # contiguous writable memoryview (zero-copy)
|
|
52
|
+
n = sock.recv_into(mv) # write network bytes directly into the ring
|
|
53
|
+
stream.commit(n)
|
|
54
|
+
|
|
55
|
+
# publish every complete package as one message record
|
|
56
|
+
while (package_size := find_complete_package(stream.data(), stream.size())):
|
|
57
|
+
stream.consume(package_size)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Consumer (opens the existing segment — can be C++ or Python)
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from slick_stream_buffer_py import SlickStreamBuffer
|
|
64
|
+
|
|
65
|
+
stream = SlickStreamBuffer(name="market_data") # geometry read from the segment header
|
|
66
|
+
|
|
67
|
+
cursor = stream.initial_reading_index() # skip history; use 0 to read from the start
|
|
68
|
+
while True:
|
|
69
|
+
data, length, cursor = stream.read(cursor)
|
|
70
|
+
if data is None:
|
|
71
|
+
continue
|
|
72
|
+
handle_package(data, length)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Local memory mode (single process)
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
buf = SlickStreamBuffer(capacity=1024, control_size=16)
|
|
79
|
+
|
|
80
|
+
mv = buf.prepare(5)
|
|
81
|
+
mv[:] = b"hello"
|
|
82
|
+
buf.commit(5)
|
|
83
|
+
buf.consume(5) # publish as one record
|
|
84
|
+
|
|
85
|
+
data, length, cursor = buf.read(0) # -> b"hello", 5, 1
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## C++ Interoperability
|
|
89
|
+
|
|
90
|
+
The C++ side uses the identical layout — either side can create the segment, the other
|
|
91
|
+
attaches to it:
|
|
92
|
+
|
|
93
|
+
```cpp
|
|
94
|
+
#include <slick/stream_buffer.hpp>
|
|
95
|
+
|
|
96
|
+
slick::stream_buffer stream("market_data"); // open segment created by Python
|
|
97
|
+
|
|
98
|
+
uint64_t cursor = stream.initial_reading_index();
|
|
99
|
+
for (;;) {
|
|
100
|
+
auto [data, length] = stream.read(cursor);
|
|
101
|
+
if (data == nullptr) continue;
|
|
102
|
+
handle_package(data, length);
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Shared memory naming: use `get_shm_name()` to obtain the exact name to pass to C++
|
|
107
|
+
(on POSIX it includes the leading `/` that `shm_open()` requires; on Windows it is the
|
|
108
|
+
raw name).
|
|
109
|
+
|
|
110
|
+
### Shared memory layout
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
[HEADER: 64 bytes]
|
|
114
|
+
0-7 atomic<uint64> committed_ - monotonic end of committed bytes
|
|
115
|
+
8-15 atomic<uint64> consumed_ - monotonic publish boundary
|
|
116
|
+
16-23 atomic<uint64> next_seq_ - next record sequence number
|
|
117
|
+
24-31 atomic<uint64> reserve_end_ - prepared-region high-water mark
|
|
118
|
+
32-39 uint64 capacity_ - data ring size in bytes (power of 2)
|
|
119
|
+
40-43 uint32 control_size_ - control ring record count (power of 2)
|
|
120
|
+
44-47 uint32 header_magic - 0x53534231 ('SSB1')
|
|
121
|
+
48-51 atomic<uint32> init_state - 0=uninit, 2=initializing, 3=ready
|
|
122
|
+
52-63 padding
|
|
123
|
+
[CONTROL RING: 32 bytes x control_size]
|
|
124
|
+
each record: atomic<uint64> seq | uint64 offset | uint32 length | 12 bytes padding
|
|
125
|
+
[DATA RING: capacity bytes]
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Total segment size: `64 + 32 * control_size + capacity`.
|
|
129
|
+
|
|
130
|
+
## API Summary
|
|
131
|
+
|
|
132
|
+
| Method | Role | Description |
|
|
133
|
+
|---|---|---|
|
|
134
|
+
| `prepare(n) -> memoryview` | producer | contiguous writable region; may relocate unconsumed bytes on wrap |
|
|
135
|
+
| `commit(n)` | producer | move n prepared bytes into the readable region (clamped) |
|
|
136
|
+
| `consume(n) -> PublishedRecord` | producer | publish the first n readable bytes as ONE record (clamped) |
|
|
137
|
+
| `discard()` | producer | drop unconsumed + prepared bytes without publishing |
|
|
138
|
+
| `data() -> memoryview` / `size()` | producer | the committed-but-unconsumed region |
|
|
139
|
+
| `read(cursor) -> (data, length, cursor)` | consumer | next record, or `(None, 0, cursor)`; skips lost records |
|
|
140
|
+
| `read_last() -> (data, length)` | consumer | newest record without a cursor |
|
|
141
|
+
| `initial_reading_index()` | consumer | cursor for a late joiner (skips history) |
|
|
142
|
+
| `loss_count()` | consumer | records skipped by this instance due to overwrite |
|
|
143
|
+
| `reset()` | owner | clear all state (not thread-safe) |
|
|
144
|
+
| `close()` / `unlink()` | lifecycle | detach / delete the segment |
|
|
145
|
+
|
|
146
|
+
See [API_DIFFERENCES.md](API_DIFFERENCES.md) for the exact deviations from the C++ API
|
|
147
|
+
(cursor passed by value, bytes copies vs pointers, exceptions).
|
|
148
|
+
|
|
149
|
+
### Caveats (same as C++)
|
|
150
|
+
|
|
151
|
+
- Producer methods (`prepare`/`commit`/`consume`/`discard`/`data`/`size`/`reset`) must be
|
|
152
|
+
called from a single thread.
|
|
153
|
+
- The buffer is lossy: if the producer outruns a consumer by more than the control ring or
|
|
154
|
+
data ring size, the consumer skips ahead and the loss is counted.
|
|
155
|
+
- `prepare()` may relocate the readable region: memoryviews previously returned by `data()`
|
|
156
|
+
or `prepare()` are invalidated.
|
|
157
|
+
- A single message (one `consume()` call) is limited to < 4 GiB.
|
|
158
|
+
|
|
159
|
+
## Building and Testing
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
# 1. Build the atomics extension
|
|
163
|
+
python setup.py build_ext --inplace
|
|
164
|
+
|
|
165
|
+
# 2. Pure-Python tests
|
|
166
|
+
python tests/test_atomic_ops.py
|
|
167
|
+
python tests/test_local_mode.py
|
|
168
|
+
python tests/test_shm_mode.py
|
|
169
|
+
|
|
170
|
+
# 3. C++ interop tests (requires CMake + a C++20 compiler)
|
|
171
|
+
cmake -S . -B build
|
|
172
|
+
cmake --build build --config Debug
|
|
173
|
+
cd build && ctest -C Debug --output-on-failure
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
The interop tests fetch [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer)
|
|
177
|
+
(which pulls [slick-shm](https://github.com/SlickQuant/slick-shm)) from GitHub and build real
|
|
178
|
+
C++ producer/consumer binaries against the actual `stream_buffer.hpp`. To build against local
|
|
179
|
+
checkouts instead (no network):
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
cmake -S . -B build \
|
|
183
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER=/path/to/slick-stream-buffer \
|
|
184
|
+
-DFETCHCONTENT_SOURCE_DIR_SLICK-SHM=/path/to/slick-shm
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
If a failed test run leaves segments behind: `python tests/cleanup_shm.py`
|
|
188
|
+
|
|
189
|
+
## Related Projects
|
|
190
|
+
|
|
191
|
+
- [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) — the C++ implementation
|
|
192
|
+
- [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) / [slick-queue](https://github.com/SlickQuant/slick-queue) — MPMC fixed-element queue with the same interop approach
|
|
193
|
+
|
|
194
|
+
## License
|
|
195
|
+
|
|
196
|
+
MIT
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "slick-stream-buffer-py"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Lock-free SPMC byte stream buffer with C++ interoperability via shared memory"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Slick Quant", email = "slickquant@slickquant.com"}
|
|
14
|
+
]
|
|
15
|
+
keywords = ["stream-buffer", "lock-free", "atomic", "shared-memory", "ipc", "multiprocessing", "spmc"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
20
|
+
"Topic :: System :: Distributed Computing",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.8",
|
|
24
|
+
"Programming Language :: Python :: 3.9",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: C++",
|
|
29
|
+
"Operating System :: Microsoft :: Windows",
|
|
30
|
+
"Operating System :: POSIX :: Linux",
|
|
31
|
+
"Operating System :: MacOS",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/SlickQuant/slick-stream-buffer-py"
|
|
36
|
+
Documentation = "https://github.com/SlickQuant/slick-stream-buffer-py#readme"
|
|
37
|
+
Repository = "https://github.com/SlickQuant/slick-stream-buffer-py"
|
|
38
|
+
"Bug Tracker" = "https://github.com/SlickQuant/slick-stream-buffer-py/issues"
|
|
39
|
+
|
|
40
|
+
[tool.setuptools]
|
|
41
|
+
py-modules = ["slick_stream_buffer_py", "ssb_atomic_ops"]
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.dynamic]
|
|
44
|
+
version = {attr = "slick_stream_buffer_py.__version__"}
|
|
45
|
+
|
|
46
|
+
[tool.cibuildwheel]
|
|
47
|
+
# Build for CPython 3.8-3.14
|
|
48
|
+
build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*"
|
|
49
|
+
# Skip musllinux builds (only build manylinux for Linux)
|
|
50
|
+
skip = "*-musllinux_*"
|
|
51
|
+
|
|
52
|
+
# Test the wheels after building
|
|
53
|
+
test-command = "python -c \"import slick_stream_buffer_py; import ssb_atomic_ops; print('Import successful')\""
|
|
54
|
+
test-requires = []
|
|
55
|
+
|
|
56
|
+
# Build settings
|
|
57
|
+
build-verbosity = 1
|
|
58
|
+
|
|
59
|
+
# Platform-specific settings
|
|
60
|
+
[tool.cibuildwheel.linux]
|
|
61
|
+
archs = ["x86_64"]
|
|
62
|
+
# Use manylinux2014 for compatibility (RHEL 7+, Ubuntu 14.04+, Debian 8+)
|
|
63
|
+
manylinux-x86_64-image = "manylinux2014"
|
|
64
|
+
|
|
65
|
+
[tool.cibuildwheel.windows]
|
|
66
|
+
archs = ["AMD64"]
|
|
67
|
+
|
|
68
|
+
[tool.cibuildwheel.macos]
|
|
69
|
+
archs = ["universal2"]
|