slick-stream-buffer-py 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,94 @@
1
+ # API Differences: Python vs C++
2
+
3
+ The Python implementation is binary-compatible with the C++ `slick::SlickStreamBuffer`
4
+ at the shared memory level — every byte, offset, and memory ordering matches. The
5
+ *language-level* API differs in the following ways.
6
+
7
+ ## Construction
8
+
9
+ | C++ | Python |
10
+ |---|---|
11
+ | `stream_buffer buf(capacity, control_size)` | `SlickStreamBuffer(capacity=..., control_size=...)` |
12
+ | `stream_buffer buf(capacity, control_size, "name")` | `SlickStreamBuffer(capacity=..., control_size=..., name="name")` |
13
+ | `stream_buffer buf("name")` | `SlickStreamBuffer(name="name")` |
14
+ | throws `std::invalid_argument` (bad sizes) | raises `ValueError` |
15
+ | throws `std::runtime_error` (shm failures) | raises `RuntimeError` (or `FileNotFoundError` when opening a missing segment) |
16
+
17
+ ## Consumer cursor is passed by value
18
+
19
+ C++ updates the cursor through a reference; Python has no by-reference ints, so the
20
+ updated cursor is returned:
21
+
22
+ ```cpp
23
+ // C++
24
+ uint64_t cursor = 0;
25
+ auto [data, length] = buf.read(cursor); // cursor updated in place
26
+ ```
27
+
28
+ ```python
29
+ # Python
30
+ cursor = 0
31
+ data, length, cursor = buf.read(cursor) # updated cursor returned
32
+ ```
33
+
34
+ `read()` returns `(None, 0, cursor)` where C++ returns `{nullptr, 0}`.
35
+ `read_last()` returns `(None, 0)` / `(bytes, length)`.
36
+
37
+ ## Zero-copy vs copies
38
+
39
+ | Operation | C++ | Python |
40
+ |---|---|---|
41
+ | `prepare(n)` | `uint8_t*` into the ring | writable `memoryview` into the ring (zero-copy) |
42
+ | `data()` | `const uint8_t*` into the ring | read-only `memoryview` into the ring (zero-copy) |
43
+ | `read(cursor)` | pointer into the ring | **`bytes` copy** |
44
+ | `read_last()` | pointer into the ring | **`bytes` copy** |
45
+ | `consume(n).data` | pointer into the ring | **`bytes` copy** |
46
+
47
+ Consumer-side data is copied because the ring is lossy: the producer may overwrite the
48
+ bytes at any moment after the validity check (the same caveat exists in C++, where the
49
+ caller must finish with the pointer before the producer laps). A `bytes` copy makes the
50
+ Python API safe by default.
51
+
52
+ `prepare()`/`data()` memoryviews are invalidated by the next `prepare()` (which may
53
+ relocate the region) — same semantics as C++ flat_buffer-style invalidation.
54
+
55
+ ## prepare() overflow
56
+
57
+ C++ throws `std::length_error` when `size() + n > capacity()`; Python raises
58
+ `ValueError`. C++ `consume()` asserts messages are < 4 GiB (debug builds); Python
59
+ always raises `ValueError`.
60
+
61
+ ## consume() return value
62
+
63
+ Both return the published record. Python's `PublishedRecord` mirrors C++
64
+ `published_record` (`sequence`, `data`, `length`) and is falsy when nothing was
65
+ published (`data is None`), matching C++ `operator bool`.
66
+
67
+ ## loss_count()
68
+
69
+ C++ counts losses only when `SLICK_STREAM_BUFFER_ENABLE_LOSS_DETECTION` is enabled
70
+ (default: debug builds only). Python always counts. The counter is per-instance in
71
+ both implementations, not shared through the segment.
72
+
73
+ ## Memory orderings
74
+
75
+ Python routes every shared-atomic access through C++ `std::atomic` (via the
76
+ `ssb_atomic_ops_ext` extension), using the same orderings on the synchronization
77
+ edges (record `seq` store-release / load-acquire, `next_seq_`/`reserve_end_`
78
+ release/acquire, `init_state` acq_rel CAS). Where the C++ side uses
79
+ `memory_order_relaxed` (e.g. `committed_`/`consumed_` stores), Python uses
80
+ release/acquire — strictly stronger, therefore still correct.
81
+
82
+ ## Lifecycle
83
+
84
+ Python adds explicit lifecycle management that C++ handles via RAII:
85
+
86
+ - `close()` — detach from the segment (does not delete it)
87
+ - `unlink()` — delete the segment (call once, from the owner, after all users closed)
88
+ - context manager support: `with SlickStreamBuffer(...) as buf: ...` (closes on exit)
89
+ - `get_shm_name()` — the exact segment name to pass to C++ (`/`-prefixed on POSIX)
90
+
91
+ On POSIX with Python >= 3.13, attaching to an existing segment uses
92
+ `SharedMemory(track=False)` so Python's resource tracker never unlinks a segment the
93
+ process does not own. On older Pythons the tracker may log a harmless warning at exit
94
+ (see `tests/run_test.py`, which filters it).
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Slick Quant
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,29 @@
1
+ # Include documentation
2
+ include README.md
3
+ include LICENSE
4
+ include API_DIFFERENCES.md
5
+
6
+ # Include C++ source files for extension
7
+ include ssb_atomic_ops_ext.cpp
8
+ include setup.py
9
+ include pyproject.toml
10
+
11
+ # Include requirements
12
+ include requirements.txt
13
+
14
+ # Exclude build artifacts
15
+ global-exclude *.pyc
16
+ global-exclude __pycache__
17
+ global-exclude *.so
18
+ global-exclude *.pyd
19
+ global-exclude *.dll
20
+ global-exclude .DS_Store
21
+
22
+ # Exclude test outputs
23
+ recursive-exclude tests *.txt
24
+ recursive-exclude tests __pycache__
25
+
26
+ # Exclude build directories
27
+ prune build
28
+ prune dist
29
+ prune *.egg-info
@@ -0,0 +1,232 @@
1
+ Metadata-Version: 2.4
2
+ Name: slick-stream-buffer-py
3
+ Version: 0.1.0
4
+ Summary: Lock-free SPMC byte stream buffer with C++ interoperability via shared memory
5
+ Home-page: https://github.com/SlickQuant/slick-stream-buffer-py
6
+ Author: Slick Quant
7
+ Author-email: Slick Quant <slickquant@slickquant.com>
8
+ License: MIT
9
+ Project-URL: Homepage, https://github.com/SlickQuant/slick-stream-buffer-py
10
+ Project-URL: Documentation, https://github.com/SlickQuant/slick-stream-buffer-py#readme
11
+ Project-URL: Repository, https://github.com/SlickQuant/slick-stream-buffer-py
12
+ Project-URL: Bug Tracker, https://github.com/SlickQuant/slick-stream-buffer-py/issues
13
+ Keywords: stream-buffer,lock-free,atomic,shared-memory,ipc,multiprocessing,spmc
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
+ Classifier: Topic :: System :: Distributed Computing
18
+ Classifier: License :: OSI Approved :: MIT License
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.8
21
+ Classifier: Programming Language :: Python :: 3.9
22
+ Classifier: Programming Language :: Python :: 3.10
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Classifier: Programming Language :: C++
26
+ Classifier: Operating System :: Microsoft :: Windows
27
+ Classifier: Operating System :: POSIX :: Linux
28
+ Classifier: Operating System :: MacOS
29
+ Requires-Python: >=3.8
30
+ Description-Content-Type: text/markdown
31
+ License-File: LICENSE
32
+ Dynamic: author
33
+ Dynamic: home-page
34
+ Dynamic: license-file
35
+ Dynamic: requires-python
36
+
37
+ # slick-stream-buffer-py
38
+
39
+ A Python implementation of [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) —
40
+ a lock-free single-producer multi-consumer (SPMC) byte stream buffer with shared memory support.
41
+
42
+ **Maintains exact binary compatibility with the C++ version**: a Python producer and a C++
43
+ consumer (or vice versa) communicate seamlessly through the same shared memory segment, using
44
+ the same `std::atomic` synchronization primitives.
45
+
46
+ ## Features
47
+
48
+ - **Byte stream, message records**: the producer writes raw bytes (`prepare`/`commit`) and
49
+ publishes them as discrete message records (`consume`), like a network receive buffer
50
+ feeding a framing layer
51
+ - **Lock-free SPMC**: one producer, any number of consumers, no locks anywhere
52
+ - **Binary-compatible with C++**: identical 64-byte header, 32-byte control records, and
53
+ acquire/release memory orderings (`slick/stream_buffer.hpp`, header magic `'SSB1'`)
54
+ - **Shared memory IPC**: built on `multiprocessing.shared_memory`, matching the C++
55
+ slick-shm naming conventions on Windows, Linux, and macOS
56
+ - **Lossy by design**: slow consumers skip overwritten data and count the loss instead of
57
+ blocking the producer
58
+ - **Local memory mode**: same API without shared memory for single-process use
59
+ - **No runtime dependencies**: Python 3.8+ standard library plus a small bundled C++
60
+ extension for the atomics
61
+
62
+ ## Requirements
63
+
64
+ - Python 3.8+
65
+ - 64-bit platform (Windows x86-64, Linux x86-64/ARM64, macOS x86-64/ARM64)
66
+ - A C++17 compiler to build the `ssb_atomic_ops_ext` extension (MSVC 2017+, GCC 5+, Clang 3.8+)
67
+
68
+ ## Installation
69
+
70
+ ```bash
71
+ pip install -e .
72
+ # or just build the extension in place:
73
+ python setup.py build_ext --inplace
74
+ ```
75
+
76
+ ## Quick Start
77
+
78
+ ### Producer (creates the shared memory segment)
79
+
80
+ ```python
81
+ from slick_stream_buffer_py import SlickStreamBuffer
82
+
83
+ # 64 MB data ring, 65536 message records
84
+ stream = SlickStreamBuffer(capacity=1 << 26, control_size=1 << 16, name="market_data")
85
+
86
+ while True:
87
+ mv = stream.prepare(64 * 1024) # contiguous writable memoryview (zero-copy)
88
+ n = sock.recv_into(mv) # write network bytes directly into the ring
89
+ stream.commit(n)
90
+
91
+ # publish every complete package as one message record
92
+ while (package_size := find_complete_package(stream.data(), stream.size())):
93
+ stream.consume(package_size)
94
+ ```
95
+
96
+ ### Consumer (opens the existing segment — can be C++ or Python)
97
+
98
+ ```python
99
+ from slick_stream_buffer_py import SlickStreamBuffer
100
+
101
+ stream = SlickStreamBuffer(name="market_data") # geometry read from the segment header
102
+
103
+ cursor = stream.initial_reading_index() # skip history; use 0 to read from the start
104
+ while True:
105
+ data, length, cursor = stream.read(cursor)
106
+ if data is None:
107
+ continue
108
+ handle_package(data, length)
109
+ ```
110
+
111
+ ### Local memory mode (single process)
112
+
113
+ ```python
114
+ buf = SlickStreamBuffer(capacity=1024, control_size=16)
115
+
116
+ mv = buf.prepare(5)
117
+ mv[:] = b"hello"
118
+ buf.commit(5)
119
+ buf.consume(5) # publish as one record
120
+
121
+ data, length, cursor = buf.read(0) # -> b"hello", 5, 1
122
+ ```
123
+
124
+ ## C++ Interoperability
125
+
126
+ The C++ side uses the identical layout — either side can create the segment, the other
127
+ attaches to it:
128
+
129
+ ```cpp
130
+ #include <slick/stream_buffer.hpp>
131
+
132
+ slick::stream_buffer stream("market_data"); // open segment created by Python
133
+
134
+ uint64_t cursor = stream.initial_reading_index();
135
+ for (;;) {
136
+ auto [data, length] = stream.read(cursor);
137
+ if (data == nullptr) continue;
138
+ handle_package(data, length);
139
+ }
140
+ ```
141
+
142
+ Shared memory naming: use `get_shm_name()` to obtain the exact name to pass to C++
143
+ (on POSIX it includes the leading `/` that `shm_open()` requires; on Windows it is the
144
+ raw name).
145
+
146
+ ### Shared memory layout
147
+
148
+ ```
149
+ [HEADER: 64 bytes]
150
+ 0-7 atomic<uint64> committed_ - monotonic end of committed bytes
151
+ 8-15 atomic<uint64> consumed_ - monotonic publish boundary
152
+ 16-23 atomic<uint64> next_seq_ - next record sequence number
153
+ 24-31 atomic<uint64> reserve_end_ - prepared-region high-water mark
154
+ 32-39 uint64 capacity_ - data ring size in bytes (power of 2)
155
+ 40-43 uint32 control_size_ - control ring record count (power of 2)
156
+ 44-47 uint32 header_magic - 0x53534231 ('SSB1')
157
+ 48-51 atomic<uint32> init_state - 0=uninit, 2=initializing, 3=ready
158
+ 52-63 padding
159
+ [CONTROL RING: 32 bytes x control_size]
160
+ each record: atomic<uint64> seq | uint64 offset | uint32 length | 12 bytes padding
161
+ [DATA RING: capacity bytes]
162
+ ```
163
+
164
+ Total segment size: `64 + 32 * control_size + capacity`.
165
+
166
+ ## API Summary
167
+
168
+ | Method | Role | Description |
169
+ |---|---|---|
170
+ | `prepare(n) -> memoryview` | producer | contiguous writable region; may relocate unconsumed bytes on wrap |
171
+ | `commit(n)` | producer | move n prepared bytes into the readable region (clamped) |
172
+ | `consume(n) -> PublishedRecord` | producer | publish the first n readable bytes as ONE record (clamped) |
173
+ | `discard()` | producer | drop unconsumed + prepared bytes without publishing |
174
+ | `data() -> memoryview` / `size()` | producer | the committed-but-unconsumed region |
175
+ | `read(cursor) -> (data, length, cursor)` | consumer | next record, or `(None, 0, cursor)`; skips lost records |
176
+ | `read_last() -> (data, length)` | consumer | newest record without a cursor |
177
+ | `initial_reading_index()` | consumer | cursor for a late joiner (skips history) |
178
+ | `loss_count()` | consumer | records skipped by this instance due to overwrite |
179
+ | `reset()` | owner | clear all state (not thread-safe) |
180
+ | `close()` / `unlink()` | lifecycle | detach / delete the segment |
181
+
182
+ See [API_DIFFERENCES.md](API_DIFFERENCES.md) for the exact deviations from the C++ API
183
+ (cursor passed by value, bytes copies vs pointers, exceptions).
184
+
185
+ ### Caveats (same as C++)
186
+
187
+ - Producer methods (`prepare`/`commit`/`consume`/`discard`/`data`/`size`/`reset`) must be
188
+ called from a single thread.
189
+ - The buffer is lossy: if the producer outruns a consumer by more than the control ring or
190
+ data ring size, the consumer skips ahead and the loss is counted.
191
+ - `prepare()` may relocate the readable region: memoryviews previously returned by `data()`
192
+ or `prepare()` are invalidated.
193
+ - A single message (one `consume()` call) is limited to < 4 GiB.
194
+
195
+ ## Building and Testing
196
+
197
+ ```bash
198
+ # 1. Build the atomics extension
199
+ python setup.py build_ext --inplace
200
+
201
+ # 2. Pure-Python tests
202
+ python tests/test_atomic_ops.py
203
+ python tests/test_local_mode.py
204
+ python tests/test_shm_mode.py
205
+
206
+ # 3. C++ interop tests (requires CMake + a C++20 compiler)
207
+ cmake -S . -B build
208
+ cmake --build build --config Debug
209
+ cd build && ctest -C Debug --output-on-failure
210
+ ```
211
+
212
+ The interop tests fetch [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer)
213
+ (which pulls [slick-shm](https://github.com/SlickQuant/slick-shm)) from GitHub and build real
214
+ C++ producer/consumer binaries against the actual `stream_buffer.hpp`. To build against local
215
+ checkouts instead (no network):
216
+
217
+ ```bash
218
+ cmake -S . -B build \
219
+ -DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER=/path/to/slick-stream-buffer \
220
+ -DFETCHCONTENT_SOURCE_DIR_SLICK-SHM=/path/to/slick-shm
221
+ ```
222
+
223
+ If a failed test run leaves segments behind: `python tests/cleanup_shm.py`
224
+
225
+ ## Related Projects
226
+
227
+ - [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) — the C++ implementation
228
+ - [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) / [slick-queue](https://github.com/SlickQuant/slick-queue) — MPMC fixed-element queue with the same interop approach
229
+
230
+ ## License
231
+
232
+ MIT
@@ -0,0 +1,196 @@
1
+ # slick-stream-buffer-py
2
+
3
+ A Python implementation of [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) —
4
+ a lock-free single-producer multi-consumer (SPMC) byte stream buffer with shared memory support.
5
+
6
+ **Maintains exact binary compatibility with the C++ version**: a Python producer and a C++
7
+ consumer (or vice versa) communicate seamlessly through the same shared memory segment, using
8
+ the same `std::atomic` synchronization primitives.
9
+
10
+ ## Features
11
+
12
+ - **Byte stream, message records**: the producer writes raw bytes (`prepare`/`commit`) and
13
+ publishes them as discrete message records (`consume`), like a network receive buffer
14
+ feeding a framing layer
15
+ - **Lock-free SPMC**: one producer, any number of consumers, no locks anywhere
16
+ - **Binary-compatible with C++**: identical 64-byte header, 32-byte control records, and
17
+ acquire/release memory orderings (`slick/stream_buffer.hpp`, header magic `'SSB1'`)
18
+ - **Shared memory IPC**: built on `multiprocessing.shared_memory`, matching the C++
19
+ slick-shm naming conventions on Windows, Linux, and macOS
20
+ - **Lossy by design**: slow consumers skip overwritten data and count the loss instead of
21
+ blocking the producer
22
+ - **Local memory mode**: same API without shared memory for single-process use
23
+ - **No runtime dependencies**: Python 3.8+ standard library plus a small bundled C++
24
+ extension for the atomics
25
+
26
+ ## Requirements
27
+
28
+ - Python 3.8+
29
+ - 64-bit platform (Windows x86-64, Linux x86-64/ARM64, macOS x86-64/ARM64)
30
+ - A C++17 compiler to build the `ssb_atomic_ops_ext` extension (MSVC 2017+, GCC 5+, Clang 3.8+)
31
+
32
+ ## Installation
33
+
34
+ ```bash
35
+ pip install -e .
36
+ # or just build the extension in place:
37
+ python setup.py build_ext --inplace
38
+ ```
39
+
40
+ ## Quick Start
41
+
42
+ ### Producer (creates the shared memory segment)
43
+
44
+ ```python
45
+ from slick_stream_buffer_py import SlickStreamBuffer
46
+
47
+ # 64 MB data ring, 65536 message records
48
+ stream = SlickStreamBuffer(capacity=1 << 26, control_size=1 << 16, name="market_data")
49
+
50
+ while True:
51
+ mv = stream.prepare(64 * 1024) # contiguous writable memoryview (zero-copy)
52
+ n = sock.recv_into(mv) # write network bytes directly into the ring
53
+ stream.commit(n)
54
+
55
+ # publish every complete package as one message record
56
+ while (package_size := find_complete_package(stream.data(), stream.size())):
57
+ stream.consume(package_size)
58
+ ```
59
+
60
+ ### Consumer (opens the existing segment — can be C++ or Python)
61
+
62
+ ```python
63
+ from slick_stream_buffer_py import SlickStreamBuffer
64
+
65
+ stream = SlickStreamBuffer(name="market_data") # geometry read from the segment header
66
+
67
+ cursor = stream.initial_reading_index() # skip history; use 0 to read from the start
68
+ while True:
69
+ data, length, cursor = stream.read(cursor)
70
+ if data is None:
71
+ continue
72
+ handle_package(data, length)
73
+ ```
74
+
75
+ ### Local memory mode (single process)
76
+
77
+ ```python
78
+ buf = SlickStreamBuffer(capacity=1024, control_size=16)
79
+
80
+ mv = buf.prepare(5)
81
+ mv[:] = b"hello"
82
+ buf.commit(5)
83
+ buf.consume(5) # publish as one record
84
+
85
+ data, length, cursor = buf.read(0) # -> b"hello", 5, 1
86
+ ```
87
+
88
+ ## C++ Interoperability
89
+
90
+ The C++ side uses the identical layout — either side can create the segment, the other
91
+ attaches to it:
92
+
93
+ ```cpp
94
+ #include <slick/stream_buffer.hpp>
95
+
96
+ slick::stream_buffer stream("market_data"); // open segment created by Python
97
+
98
+ uint64_t cursor = stream.initial_reading_index();
99
+ for (;;) {
100
+ auto [data, length] = stream.read(cursor);
101
+ if (data == nullptr) continue;
102
+ handle_package(data, length);
103
+ }
104
+ ```
105
+
106
+ Shared memory naming: use `get_shm_name()` to obtain the exact name to pass to C++
107
+ (on POSIX it includes the leading `/` that `shm_open()` requires; on Windows it is the
108
+ raw name).
109
+
110
+ ### Shared memory layout
111
+
112
+ ```
113
+ [HEADER: 64 bytes]
114
+ 0-7 atomic<uint64> committed_ - monotonic end of committed bytes
115
+ 8-15 atomic<uint64> consumed_ - monotonic publish boundary
116
+ 16-23 atomic<uint64> next_seq_ - next record sequence number
117
+ 24-31 atomic<uint64> reserve_end_ - prepared-region high-water mark
118
+ 32-39 uint64 capacity_ - data ring size in bytes (power of 2)
119
+ 40-43 uint32 control_size_ - control ring record count (power of 2)
120
+ 44-47 uint32 header_magic - 0x53534231 ('SSB1')
121
+ 48-51 atomic<uint32> init_state - 0=uninit, 2=initializing, 3=ready
122
+ 52-63 padding
123
+ [CONTROL RING: 32 bytes x control_size]
124
+ each record: atomic<uint64> seq | uint64 offset | uint32 length | 12 bytes padding
125
+ [DATA RING: capacity bytes]
126
+ ```
127
+
128
+ Total segment size: `64 + 32 * control_size + capacity`.
129
+
130
+ ## API Summary
131
+
132
+ | Method | Role | Description |
133
+ |---|---|---|
134
+ | `prepare(n) -> memoryview` | producer | contiguous writable region; may relocate unconsumed bytes on wrap |
135
+ | `commit(n)` | producer | move n prepared bytes into the readable region (clamped) |
136
+ | `consume(n) -> PublishedRecord` | producer | publish the first n readable bytes as ONE record (clamped) |
137
+ | `discard()` | producer | drop unconsumed + prepared bytes without publishing |
138
+ | `data() -> memoryview` / `size()` | producer | the committed-but-unconsumed region |
139
+ | `read(cursor) -> (data, length, cursor)` | consumer | next record, or `(None, 0, cursor)`; skips lost records |
140
+ | `read_last() -> (data, length)` | consumer | newest record without a cursor |
141
+ | `initial_reading_index()` | consumer | cursor for a late joiner (skips history) |
142
+ | `loss_count()` | consumer | records skipped by this instance due to overwrite |
143
+ | `reset()` | owner | clear all state (not thread-safe) |
144
+ | `close()` / `unlink()` | lifecycle | detach / delete the segment |
145
+
146
+ See [API_DIFFERENCES.md](API_DIFFERENCES.md) for the exact deviations from the C++ API
147
+ (cursor passed by value, bytes copies vs pointers, exceptions).
148
+
149
+ ### Caveats (same as C++)
150
+
151
+ - Producer methods (`prepare`/`commit`/`consume`/`discard`/`data`/`size`/`reset`) must be
152
+ called from a single thread.
153
+ - The buffer is lossy: if the producer outruns a consumer by more than the control ring or
154
+ data ring size, the consumer skips ahead and the loss is counted.
155
+ - `prepare()` may relocate the readable region: memoryviews previously returned by `data()`
156
+ or `prepare()` are invalidated.
157
+ - A single message (one `consume()` call) is limited to < 4 GiB.
158
+
159
+ ## Building and Testing
160
+
161
+ ```bash
162
+ # 1. Build the atomics extension
163
+ python setup.py build_ext --inplace
164
+
165
+ # 2. Pure-Python tests
166
+ python tests/test_atomic_ops.py
167
+ python tests/test_local_mode.py
168
+ python tests/test_shm_mode.py
169
+
170
+ # 3. C++ interop tests (requires CMake + a C++20 compiler)
171
+ cmake -S . -B build
172
+ cmake --build build --config Debug
173
+ cd build && ctest -C Debug --output-on-failure
174
+ ```
175
+
176
+ The interop tests fetch [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer)
177
+ (which pulls [slick-shm](https://github.com/SlickQuant/slick-shm)) from GitHub and build real
178
+ C++ producer/consumer binaries against the actual `stream_buffer.hpp`. To build against local
179
+ checkouts instead (no network):
180
+
181
+ ```bash
182
+ cmake -S . -B build \
183
+ -DFETCHCONTENT_SOURCE_DIR_SLICK-STREAM-BUFFER=/path/to/slick-stream-buffer \
184
+ -DFETCHCONTENT_SOURCE_DIR_SLICK-SHM=/path/to/slick-shm
185
+ ```
186
+
187
+ If a failed test run leaves segments behind: `python tests/cleanup_shm.py`
188
+
189
+ ## Related Projects
190
+
191
+ - [slick-stream-buffer](https://github.com/SlickQuant/slick-stream-buffer) — the C++ implementation
192
+ - [slick-queue-py](https://github.com/SlickQuant/slick-queue-py) / [slick-queue](https://github.com/SlickQuant/slick-queue) — MPMC fixed-element queue with the same interop approach
193
+
194
+ ## License
195
+
196
+ MIT
@@ -0,0 +1,69 @@
1
+ [build-system]
2
+ requires = ["setuptools>=45", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "slick-stream-buffer-py"
7
+ dynamic = ["version"]
8
+ description = "Lock-free SPMC byte stream buffer with C++ interoperability via shared memory"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = {text = "MIT"}
12
+ authors = [
13
+ {name = "Slick Quant", email = "slickquant@slickquant.com"}
14
+ ]
15
+ keywords = ["stream-buffer", "lock-free", "atomic", "shared-memory", "ipc", "multiprocessing", "spmc"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "Topic :: Software Development :: Libraries :: Python Modules",
20
+ "Topic :: System :: Distributed Computing",
21
+ "License :: OSI Approved :: MIT License",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.8",
24
+ "Programming Language :: Python :: 3.9",
25
+ "Programming Language :: Python :: 3.10",
26
+ "Programming Language :: Python :: 3.11",
27
+ "Programming Language :: Python :: 3.12",
28
+ "Programming Language :: C++",
29
+ "Operating System :: Microsoft :: Windows",
30
+ "Operating System :: POSIX :: Linux",
31
+ "Operating System :: MacOS",
32
+ ]
33
+
34
+ [project.urls]
35
+ Homepage = "https://github.com/SlickQuant/slick-stream-buffer-py"
36
+ Documentation = "https://github.com/SlickQuant/slick-stream-buffer-py#readme"
37
+ Repository = "https://github.com/SlickQuant/slick-stream-buffer-py"
38
+ "Bug Tracker" = "https://github.com/SlickQuant/slick-stream-buffer-py/issues"
39
+
40
+ [tool.setuptools]
41
+ py-modules = ["slick_stream_buffer_py", "ssb_atomic_ops"]
42
+
43
+ [tool.setuptools.dynamic]
44
+ version = {attr = "slick_stream_buffer_py.__version__"}
45
+
46
+ [tool.cibuildwheel]
47
+ # Build for CPython 3.8-3.14
48
+ build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*"
49
+ # Skip musllinux builds (only build manylinux for Linux)
50
+ skip = "*-musllinux_*"
51
+
52
+ # Test the wheels after building
53
+ test-command = "python -c \"import slick_stream_buffer_py; import ssb_atomic_ops; print('Import successful')\""
54
+ test-requires = []
55
+
56
+ # Build settings
57
+ build-verbosity = 1
58
+
59
+ # Platform-specific settings
60
+ [tool.cibuildwheel.linux]
61
+ archs = ["x86_64"]
62
+ # Use manylinux2014 for compatibility (RHEL 7+, Ubuntu 14.04+, Debian 8+)
63
+ manylinux-x86_64-image = "manylinux2014"
64
+
65
+ [tool.cibuildwheel.windows]
66
+ archs = ["AMD64"]
67
+
68
+ [tool.cibuildwheel.macos]
69
+ archs = ["universal2"]
@@ -0,0 +1,2 @@
1
+ # No runtime dependencies (Python 3.8+ standard library only)
2
+ # multiprocessing.shared_memory is included in Python 3.8+
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+