slick-queue-py 1.0.0__cp312-cp312-win_amd64.whl → 1.1.0__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atomic_ops.py +628 -609
- atomic_ops_ext.cp312-win_amd64.pyd +0 -0
- {slick_queue_py-1.0.0.dist-info → slick_queue_py-1.1.0.dist-info}/METADATA +57 -42
- slick_queue_py-1.1.0.dist-info/RECORD +8 -0
- {slick_queue_py-1.0.0.dist-info → slick_queue_py-1.1.0.dist-info}/WHEEL +1 -1
- slick_queue_py.py +723 -520
- slick_queue_py-1.0.0.dist-info/RECORD +0 -8
- {slick_queue_py-1.0.0.dist-info → slick_queue_py-1.1.0.dist-info}/licenses/LICENSE +0 -0
- {slick_queue_py-1.0.0.dist-info → slick_queue_py-1.1.0.dist-info}/top_level.txt +0 -0
|
Binary file
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 1.
|
|
2
|
+
Name: slick-queue-py
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Lock-free MPMC queue with C++ interoperability via shared memory
|
|
5
|
-
Home-page: https://github.com/SlickQuant/
|
|
5
|
+
Home-page: https://github.com/SlickQuant/slick-queue-py
|
|
6
6
|
Author: Slick Quant
|
|
7
7
|
Author-email: Slick Quant <slickquant@slickquant.com>
|
|
8
8
|
License: MIT
|
|
9
|
-
Project-URL: Homepage, https://github.com/SlickQuant/
|
|
10
|
-
Project-URL: Documentation, https://github.com/SlickQuant/
|
|
11
|
-
Project-URL: Repository, https://github.com/SlickQuant/
|
|
12
|
-
Project-URL: Bug Tracker, https://github.com/SlickQuant/
|
|
9
|
+
Project-URL: Homepage, https://github.com/SlickQuant/slick-queue-py
|
|
10
|
+
Project-URL: Documentation, https://github.com/SlickQuant/slick-queue-py#readme
|
|
11
|
+
Project-URL: Repository, https://github.com/SlickQuant/slick-queue-py
|
|
12
|
+
Project-URL: Bug Tracker, https://github.com/SlickQuant/slick-queue-py/issues
|
|
13
13
|
Keywords: queue,lock-free,atomic,shared-memory,ipc,multiprocessing,mpmc
|
|
14
14
|
Classifier: Development Status :: 4 - Beta
|
|
15
15
|
Classifier: Intended Audience :: Developers
|
|
@@ -34,11 +34,15 @@ Dynamic: home-page
|
|
|
34
34
|
Dynamic: license-file
|
|
35
35
|
Dynamic: requires-python
|
|
36
36
|
|
|
37
|
-
#
|
|
37
|
+
# slick-queue-py
|
|
38
38
|
|
|
39
39
|
Python implementation of SlickQueue - a lock-free multi-producer multi-consumer (MPMC) queue with C++ interoperability through shared memory.
|
|
40
40
|
|
|
41
|
-
This is the Python binding for the [SlickQueue C++ library](https://github.com/SlickQuant/
|
|
41
|
+
This is the Python binding for the [SlickQueue C++ library](https://github.com/SlickQuant/slick-queue). The Python implementation maintains exact binary compatibility with the C++ version, enabling seamless interprocess communication between Python and C++ applications.
|
|
42
|
+
|
|
43
|
+
[](https://opensource.org/licenses/MIT)
|
|
44
|
+
[](https://github.com/SlickQuant/slick-queue-py/actions/workflows/ci.yml)
|
|
45
|
+
[](https://github.com/SlickQuant/slick-queue-py/releases)
|
|
42
46
|
|
|
43
47
|
## Features
|
|
44
48
|
|
|
@@ -172,7 +176,7 @@ def consumer_worker(q, cursor, worker_id, results):
|
|
|
172
176
|
items_processed = 0
|
|
173
177
|
while True:
|
|
174
178
|
# Atomically claim next item (work-stealing)
|
|
175
|
-
data, size = q.read(cursor)
|
|
179
|
+
data, size, index = q.read(cursor)
|
|
176
180
|
|
|
177
181
|
if data is None:
|
|
178
182
|
break # No more data
|
|
@@ -228,7 +232,7 @@ def consumer_worker(queue_name, cursor_name, worker_id):
|
|
|
228
232
|
items_processed = 0
|
|
229
233
|
while True:
|
|
230
234
|
# Atomically claim next item (work-stealing)
|
|
231
|
-
data, size = q.read(cursor)
|
|
235
|
+
data, size, index = q.read(cursor)
|
|
232
236
|
|
|
233
237
|
if data is None:
|
|
234
238
|
break # No more data
|
|
@@ -273,7 +277,7 @@ q.unlink()
|
|
|
273
277
|
|
|
274
278
|
### C++/Python Interoperability
|
|
275
279
|
|
|
276
|
-
The Python implementation is fully compatible with the C++ [SlickQueue](https://github.com/SlickQuant/
|
|
280
|
+
The Python implementation is fully compatible with the C++ [SlickQueue](https://github.com/SlickQuant/slick-queue) library. Python and C++ processes can produce and consume from the same queue with:
|
|
277
281
|
|
|
278
282
|
- **Exact memory layout compatibility**: Binary-compatible with `slick::SlickQueue<T>`
|
|
279
283
|
- **Atomic operation compatibility**: Same 16-byte and 8-byte CAS semantics
|
|
@@ -329,10 +333,10 @@ To use the C++ SlickQueue library with your Python queues:
|
|
|
329
333
|
|
|
330
334
|
```bash
|
|
331
335
|
# Clone the C++ library
|
|
332
|
-
git clone https://github.com/SlickQuant/
|
|
336
|
+
git clone https://github.com/SlickQuant/slick-queue.git
|
|
333
337
|
|
|
334
338
|
# Build your C++ program
|
|
335
|
-
g++ -std=c++17 -I
|
|
339
|
+
g++ -std=c++17 -I slick-queue/include my_program.cpp -o my_program
|
|
336
340
|
```
|
|
337
341
|
|
|
338
342
|
Or use CMake (see [CMakeLists.txt](CMakeLists.txt) for reference):
|
|
@@ -340,14 +344,14 @@ Or use CMake (see [CMakeLists.txt](CMakeLists.txt) for reference):
|
|
|
340
344
|
```cmake
|
|
341
345
|
include(FetchContent)
|
|
342
346
|
FetchContent_Declare(
|
|
343
|
-
|
|
344
|
-
GIT_REPOSITORY https://github.com/SlickQuant/
|
|
347
|
+
slick-queue
|
|
348
|
+
GIT_REPOSITORY https://github.com/SlickQuant/slick-queue.git
|
|
345
349
|
GIT_TAG main
|
|
346
350
|
)
|
|
347
|
-
FetchContent_MakeAvailable(
|
|
351
|
+
FetchContent_MakeAvailable(slick-queue)
|
|
348
352
|
|
|
349
353
|
add_executable(my_program my_program.cpp)
|
|
350
|
-
target_link_libraries(my_program PRIVATE
|
|
354
|
+
target_link_libraries(my_program PRIVATE slick::queue)
|
|
351
355
|
```
|
|
352
356
|
|
|
353
357
|
See [tests/test_interop.py](tests/test_interop.py) and [tests/cpp_*.cpp](tests/) for comprehensive examples.
|
|
@@ -436,7 +440,7 @@ data, size, read_index = q.read(read_index)
|
|
|
436
440
|
# Python multi-consumer (atomic cursor)
|
|
437
441
|
from slick_queue_py import AtomicCursor
|
|
438
442
|
cursor = AtomicCursor(cursor_shm.buf, 0)
|
|
439
|
-
data, size = q.read(cursor) # Atomically claim next item
|
|
443
|
+
data, size, index = q.read(cursor) # Atomically claim next item
|
|
440
444
|
|
|
441
445
|
# C++ (updates by reference for both)
|
|
442
446
|
auto [data, size] = queue.read(read_index); // read_index modified in-place
|
|
@@ -463,7 +467,7 @@ cursor.store(0)
|
|
|
463
467
|
|
|
464
468
|
# Multiple threads can share this cursor
|
|
465
469
|
while True:
|
|
466
|
-
data, size = q.read(cursor) # Each thread atomically claims items
|
|
470
|
+
data, size, index = q.read(cursor) # Each thread atomically claims items
|
|
467
471
|
if data is not None:
|
|
468
472
|
process(data)
|
|
469
473
|
```
|
|
@@ -480,17 +484,26 @@ cursor.store(0)
|
|
|
480
484
|
|
|
481
485
|
# Multiple processes can share this cursor
|
|
482
486
|
while True:
|
|
483
|
-
data, size = q.read(cursor) # Each process atomically claims items
|
|
487
|
+
data, size, index = q.read(cursor) # Each process atomically claims items
|
|
484
488
|
if data is not None:
|
|
485
489
|
process(data)
|
|
486
490
|
```
|
|
487
491
|
|
|
488
|
-
#### `read_last() -> Optional[bytes]`
|
|
492
|
+
#### `read_last() -> Tuple[Optional[bytes], int]`
|
|
489
493
|
|
|
490
494
|
Read the most recently published item.
|
|
491
495
|
|
|
492
496
|
**Returns:**
|
|
493
|
-
- `Optional[bytes]`:
|
|
497
|
+
- `Tuple[Optional[bytes], int]`: Tuple of (data, size)
|
|
498
|
+
- `data`: Last published data or None if queue is empty
|
|
499
|
+
- `size`: Number of slots the item occupies (0 if queue is empty)
|
|
500
|
+
|
|
501
|
+
**Example:**
|
|
502
|
+
```python
|
|
503
|
+
data, size = q.read_last()
|
|
504
|
+
if data is not None:
|
|
505
|
+
print(f"Last item: {data[:size * element_size]}")
|
|
506
|
+
```
|
|
494
507
|
|
|
495
508
|
#### `__getitem__(index) -> memoryview`
|
|
496
509
|
|
|
@@ -596,37 +609,39 @@ Offset | Size | Content
|
|
|
596
609
|
## Platform Support
|
|
597
610
|
|
|
598
611
|
### Fully Supported (Lock-Free)
|
|
599
|
-
- **Windows x86-64**: Uses
|
|
600
|
-
- **Linux x86-64**: Uses `
|
|
601
|
-
- **macOS x86-64**: Uses `
|
|
612
|
+
- **Windows x86-64**: Uses C++ extension (`atomic_ops_ext.pyd`) with `std::atomic`
|
|
613
|
+
- **Linux x86-64**: Uses C++ extension (`atomic_ops_ext.so`) with `std::atomic`, fallback to `libatomic`
|
|
614
|
+
- **macOS x86-64**: Uses C++ extension (`atomic_ops_ext.so`) with `std::atomic`, fallback to compiler builtins
|
|
602
615
|
|
|
603
616
|
**Platform-specific atomic operation implementations:**
|
|
604
|
-
- **
|
|
605
|
-
- **Linux/macOS
|
|
617
|
+
- **All platforms**: The `atomic_ops_ext` C++ extension is now used on all platforms for the most reliable cross-process atomic operations
|
|
618
|
+
- **Fallback support**: Linux/macOS can fall back to `libatomic` or compiler builtins if the extension isn't available
|
|
606
619
|
|
|
607
|
-
### Building
|
|
620
|
+
### Building and Installation
|
|
608
621
|
|
|
609
|
-
|
|
622
|
+
The C++ extension is built automatically during installation:
|
|
610
623
|
|
|
611
624
|
```bash
|
|
612
|
-
# Install build
|
|
613
|
-
pip install
|
|
625
|
+
# Install with automatic extension build
|
|
626
|
+
pip install -e .
|
|
614
627
|
|
|
615
|
-
#
|
|
628
|
+
# Or build manually first
|
|
616
629
|
python setup.py build_ext --inplace
|
|
617
|
-
|
|
618
|
-
# Or install in development mode (builds automatically)
|
|
619
630
|
pip install -e .
|
|
620
631
|
```
|
|
621
632
|
|
|
622
|
-
**
|
|
623
|
-
- Visual Studio 2017+ or MSVC build tools
|
|
624
|
-
-
|
|
633
|
+
**Build requirements:**
|
|
634
|
+
- **Windows**: Visual Studio 2017+ or MSVC build tools
|
|
635
|
+
- **Linux**: GCC 5+ or Clang 3.8+
|
|
636
|
+
- **macOS**: Xcode command line tools (clang)
|
|
637
|
+
- **All platforms**: Python development headers (included with standard Python installation)
|
|
625
638
|
|
|
626
|
-
The extension will be built as
|
|
639
|
+
The extension will be built as:
|
|
640
|
+
- Windows: `atomic_ops_ext.cp3XX-win_amd64.pyd`
|
|
641
|
+
- Linux: `atomic_ops_ext.cpython-3XX-x86_64-linux-gnu.so`
|
|
642
|
+
- macOS: `atomic_ops_ext.cpython-3XX-darwin.so`
|
|
627
643
|
|
|
628
|
-
|
|
629
|
-
No build step required! The `libatomic` library is typically included with GCC/Clang toolchains and is automatically loaded via ctypes.
|
|
644
|
+
(where `XX` is your Python version, e.g., `312` for Python 3.12)
|
|
630
645
|
|
|
631
646
|
### Requirements for Lock-Free Operation
|
|
632
647
|
|
|
@@ -796,7 +811,7 @@ This ensures:
|
|
|
796
811
|
|
|
797
812
|
## Contributing
|
|
798
813
|
|
|
799
|
-
Issues and pull requests welcome at [SlickQuant/
|
|
814
|
+
Issues and pull requests welcome at [SlickQuant/slick-queue-py](https://github.com/SlickQuant/slick-queue-py).
|
|
800
815
|
|
|
801
816
|
## License
|
|
802
817
|
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
atomic_ops.py,sha256=IowoYaE_y0QmPjrc36gggXXnZrPQXlOk_DEO-98y0kI,22206
|
|
2
|
+
atomic_ops_ext.cp312-win_amd64.pyd,sha256=fjVTEjSU3eVs_0XFVdKE1yA3k9rxfFqiQREpZ-qzBAo,12800
|
|
3
|
+
slick_queue_py.py,sha256=w3Ut0j7RIfbQfzyLaH4nnimVnLY-Bn1KctT-Ta4GNhk,31015
|
|
4
|
+
slick_queue_py-1.1.0.dist-info/licenses/LICENSE,sha256=GsgPE6yHSgjPd--0cXVu4tmzSrApWVih5TeqOHpVeo0,1089
|
|
5
|
+
slick_queue_py-1.1.0.dist-info/METADATA,sha256=QJTwXP_-Pcwe6WvI48oUboDt3f6txPpcV2nNTzoTx-4,25965
|
|
6
|
+
slick_queue_py-1.1.0.dist-info/WHEEL,sha256=4SIlGrTWEevUMa-6zmQ9aBHcYatnnG9aOHYILIJiCXU,102
|
|
7
|
+
slick_queue_py-1.1.0.dist-info/top_level.txt,sha256=wSnL-GmRhMTtudT__dLgRw5eZUU_XHo2OZTOP3M1MwA,41
|
|
8
|
+
slick_queue_py-1.1.0.dist-info/RECORD,,
|