turbopipe 1.0.5__tar.gz → 1.2.0__tar.gz

{turbopipe-1.0.5 → turbopipe-1.2.0}/.github/workflows/release.yaml

@@ -48,8 +48,6 @@ jobs:
       - name: Install MSVC
         if: matrix.os == 'windows-latest'
         uses: bus1/cabuild/action/msdevshell@v1
-        with:
-          architecture: x64
 
       - name: deps
         run: python -m pip install cibuildwheel==2.19.2
@@ -86,3 +84,23 @@ jobs:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{secrets.PYPI_TOKEN}}
         run: twine upload package/*
+
+  tag:
+    needs: publish
+    name: Create Release Tag
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+
+      - name: Get version
+        run: echo VERSION=$(python turbopipe/version.py) >> $GITHUB_ENV
+        shell: bash
+
+      - name: Create Release Tag
+        run: |
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git tag -a v$VERSION -m "Release v$VERSION"
+          git push origin v$VERSION

{turbopipe-1.0.5 → turbopipe-1.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: turbopipe
-Version: 1.0.5
-Summary: 🌀 Faster ModernGL Buffer inter process data transfers
+Version: 1.2.0
+Summary: 🌀 Faster MemoryView inter-process data transfers for subprocesses
 Home-page: https://brokensrc.dev
 Author-Email: Tremeschin <29046864+Tremeschin@users.noreply.github.com>
 License: MIT License
@@ -35,12 +35,18 @@ Description-Content-Type: text/markdown
 
 > [!IMPORTANT]
 > <sub>Also check out [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow), where **TurboPipe** shines! 😉</sub>
-
+<!-- PyPI -->
 <div align="center">
   <a href="https://brokensrc.dev/"><img src="https://raw.githubusercontent.com/BrokenSource/TurboPipe/main/turbopipe/resources/images/turbopipe.png" width="200"></a>
   <h1>TurboPipe</h1>
+  Faster <a href="https://github.com/moderngl/moderngl"><b>ModernGL Buffers</b></a> inter-process data transfers for subprocesses
+  <br>
   <br>
-  Faster <a href="https://github.com/moderngl/moderngl"><b>ModernGL</b></a> inter-process data transfers
+  <a href="https://pypi.org/project/turbopipe/"><img src="https://img.shields.io/pypi/v/turbopipe?label=PyPI&color=blue"></a>
+  <a href="https://pypi.org/project/turbopipe/"><img src="https://img.shields.io/pypi/dw/turbopipe?label=Installs&color=blue"></a>
+  <a href="https://github.com/BrokenSource/TurboPipe"><img src="https://img.shields.io/github/v/tag/BrokenSource/TurboPipe?label=GitHub&color=orange"></a>
+  <a href="https://github.com/BrokenSource/TurboPipe/stargazers"><img src="https://img.shields.io/github/stars/BrokenSource/TurboPipe?label=Stars&style=flat&color=orange"></a>
+  <a href="https://discord.gg/KjqvcYwRHm"><img src="https://img.shields.io/discord/1184696441298485370?label=Discord&style=flat&color=purple"></a>
 </div>
 
 <br>
@@ -53,7 +59,7 @@ The **optimizations** involved are:
 
 - **Zero-copy**: Avoid unnecessary memory copies or allocation (intermediate `buffer.read()`)
 - **C++**: The core of TurboPipe is written in C++ for speed, efficiency and low-level control
-- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy
+- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy (Unix)
 - **Threaded**:
     - Doesn't block Python code execution, allows to render next frame
     - Decouples the main thread from the I/O thread for performance
@@ -68,7 +74,7 @@ It couldn't be easier! Just install the [**`turbopipe`**](https://pypi.org/proje
 
 ```bash
 # With pip (https://pip.pypa.io/)
-python -m pip install turbopipe
+pip install turbopipe
 
 # With Poetry (https://python-poetry.org/)
 poetry add turbopipe
@@ -84,7 +90,7 @@ rye add turbopipe
 
 # 🚀 Usage
 
-See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for more controlled usage, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Scene.py) usage of it!
+See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for comparisons, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Scene.py) usage of it!
 
 ```python
 import subprocess
@@ -106,7 +112,7 @@ ffmpeg = subprocess.Popen(
 for _ in range(60 * 60):
     turbopipe.pipe(buffer, ffmpeg.stdin.fileno())
 
-# Finalize writing
+# Finalize writing, encoding
 turbopipe.sync()
 ffmpeg.stdin.close()
 ffmpeg.wait()
@@ -148,9 +154,9 @@ ffmpeg.wait()
 | 🐢   | Null      |         1 |   882 fps | 2.44 GB/s   |          |
 | 🚀   | Null      |         1 |   793 fps | 2.19 GB/s   | -10.04%  |
 | 🌀   | Null      |         1 |  1911 fps | 5.28 GB/s   | 116.70%  |
-| 🐢   | Null      |         4 |   880 fps | 2.43 GB/s   |          |
-| 🚀   | Null      |         4 |   924 fps | 2.56 GB/s   | 5.05%    |
-| 🌀   | Null      |         4 |  2037 fps | 5.63 GB/s   | 131.59%  |
+| 🐢   | Null      |         4 |   857 fps | 2.37 GB/s   |          |
+| 🚀   | Null      |         4 |   891 fps | 2.47 GB/s   | 4.05%    |
+| 🌀   | Null      |         4 |  2309 fps | 6.38 GB/s   | 169.45%  |
 | 🐢   | ultrafast |         4 |   714 fps | 1.98 GB/s   |          |
 | 🚀   | ultrafast |         4 |   670 fps | 1.85 GB/s   | -6.10%   |
 | 🌀   | ultrafast |         4 |  1093 fps | 3.02 GB/s   | 53.13%   |
@@ -166,9 +172,9 @@ ffmpeg.wait()
 | 🐢    | Null      |         4 |   390 fps | 2.43 GB/s   |         |
 | 🚀    | Null      |         4 |   391 fps | 2.43 GB/s   | 0.26%   |
 | 🌀    | Null      |         4 |   756 fps | 4.71 GB/s   | 94.01%  |
-| 🐢    | ultrafast |         4 |   277 fps | 1.73 GB/s   |         |
-| 🚀    | ultrafast |         4 |   270 fps | 1.68 GB/s   | -2.40%  |
-| 🌀    | ultrafast |         4 |   402 fps | 2.50 GB/s   | 45.32%  |
+| 🐢    | ultrafast |         4 |   269 fps | 1.68 GB/s   |         |
+| 🚀    | ultrafast |         4 |   272 fps | 1.70 GB/s   | 1.48%   |
+| 🌀    | ultrafast |         4 |   409 fps | 2.55 GB/s   | 52.29%  |
 | 🐢    | slow      |         4 |   115 fps | 0.72 GB/s   |         |
 | 🚀    | slow      |         4 |   118 fps | 0.74 GB/s   | 3.40%   |
 | 🌀    | slow      |         4 |   119 fps | 0.75 GB/s   | 4.34%   |
@@ -178,9 +184,9 @@ ffmpeg.wait()
 | 🐢    | Null      |         1 |   210 fps | 2.33 GB/s   |          |
 | 🚀    | Null      |         1 |   239 fps | 2.64 GB/s   | 13.84%   |
 | 🌀    | Null      |         1 |   534 fps | 5.91 GB/s   | 154.32%  |
-| 🐢    | Null      |         4 |   233 fps | 2.58 GB/s   |          |
-| 🚀    | Null      |         4 |   232 fps | 2.57 GB/s   | -0.08%   |
-| 🌀    | Null      |         4 |   495 fps | 5.48 GB/s   | 112.64%  |
+| 🐢    | Null      |         4 |   219 fps | 2.43 GB/s   |          |
+| 🚀    | Null      |         4 |   231 fps | 2.56 GB/s   | 5.64%    |
+| 🌀    | Null      |         4 |   503 fps | 5.56 GB/s   | 129.75%  |
 | 🐢    | ultrafast |         4 |   141 fps | 1.56 GB/s   |          |
 | 🚀    | ultrafast |         4 |   150 fps | 1.67 GB/s   | 6.92%    |
 | 🌀    | ultrafast |         4 |   226 fps | 2.50 GB/s   | 60.37%   |
@@ -360,9 +366,7 @@ On realistically loads, like [**ShaderFlow**](https://github.com/BrokenSource/Sh
 
 # 📚 Future work
 
-- Add support for NumPy arrays, memoryviews, and byte-like objects
 - Disable/investigate performance degradation on Windows iGPUs
 - Improve the thread synchronization and/or use a ThreadPool
-- Stabler way for finding mglo struct offsets (moderngl.h?)
 - Maybe use `mmap` instead of chunks writing on Linux
-- Test on MacOS 🙈
+- Test on macOS 🙈

{turbopipe-1.0.5 → turbopipe-1.2.0}/Readme.md

@@ -1,11 +1,17 @@
 > [!IMPORTANT]
 > <sub>Also check out [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow), where **TurboPipe** shines! 😉</sub>
-
+<!-- PyPI -->
 <div align="center">
   <a href="https://brokensrc.dev/"><img src="https://raw.githubusercontent.com/BrokenSource/TurboPipe/main/turbopipe/resources/images/turbopipe.png" width="200"></a>
   <h1>TurboPipe</h1>
+  Faster <a href="https://github.com/moderngl/moderngl"><b>ModernGL Buffers</b></a> inter-process data transfers for subprocesses
+  <br>
   <br>
-  Faster <a href="https://github.com/moderngl/moderngl"><b>ModernGL</b></a> inter-process data transfers
+  <a href="https://pypi.org/project/turbopipe/"><img src="https://img.shields.io/pypi/v/turbopipe?label=PyPI&color=blue"></a>
+  <a href="https://pypi.org/project/turbopipe/"><img src="https://img.shields.io/pypi/dw/turbopipe?label=Installs&color=blue"></a>
+  <a href="https://github.com/BrokenSource/TurboPipe"><img src="https://img.shields.io/github/v/tag/BrokenSource/TurboPipe?label=GitHub&color=orange"></a>
+  <a href="https://github.com/BrokenSource/TurboPipe/stargazers"><img src="https://img.shields.io/github/stars/BrokenSource/TurboPipe?label=Stars&style=flat&color=orange"></a>
+  <a href="https://discord.gg/KjqvcYwRHm"><img src="https://img.shields.io/discord/1184696441298485370?label=Discord&style=flat&color=purple"></a>
 </div>
 
 <br>
@@ -18,7 +24,7 @@ The **optimizations** involved are:
 
 - **Zero-copy**: Avoid unnecessary memory copies or allocation (intermediate `buffer.read()`)
 - **C++**: The core of TurboPipe is written in C++ for speed, efficiency and low-level control
-- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy
+- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy (Unix)
 - **Threaded**:
     - Doesn't block Python code execution, allows to render next frame
     - Decouples the main thread from the I/O thread for performance
@@ -33,7 +39,7 @@ It couldn't be easier! Just install the [**`turbopipe`**](https://pypi.org/proje
 
 ```bash
 # With pip (https://pip.pypa.io/)
-python -m pip install turbopipe
+pip install turbopipe
 
 # With Poetry (https://python-poetry.org/)
 poetry add turbopipe
@@ -49,7 +55,7 @@ rye add turbopipe
 
 # 🚀 Usage
 
-See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for more controlled usage, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Scene.py) usage of it!
+See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for comparisons, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Scene.py) usage of it!
 
 ```python
 import subprocess
@@ -71,7 +77,7 @@ ffmpeg = subprocess.Popen(
 for _ in range(60 * 60):
     turbopipe.pipe(buffer, ffmpeg.stdin.fileno())
 
-# Finalize writing
+# Finalize writing, encoding
 turbopipe.sync()
 ffmpeg.stdin.close()
 ffmpeg.wait()
@@ -113,9 +119,9 @@ ffmpeg.wait()
 | 🐢   | Null      |         1 |   882 fps | 2.44 GB/s   |          |
 | 🚀   | Null      |         1 |   793 fps | 2.19 GB/s   | -10.04%  |
 | 🌀   | Null      |         1 |  1911 fps | 5.28 GB/s   | 116.70%  |
-| 🐢   | Null      |         4 |   880 fps | 2.43 GB/s   |          |
-| 🚀   | Null      |         4 |   924 fps | 2.56 GB/s   | 5.05%    |
-| 🌀   | Null      |         4 |  2037 fps | 5.63 GB/s   | 131.59%  |
+| 🐢   | Null      |         4 |   857 fps | 2.37 GB/s   |          |
+| 🚀   | Null      |         4 |   891 fps | 2.47 GB/s   | 4.05%    |
+| 🌀   | Null      |         4 |  2309 fps | 6.38 GB/s   | 169.45%  |
 | 🐢   | ultrafast |         4 |   714 fps | 1.98 GB/s   |          |
 | 🚀   | ultrafast |         4 |   670 fps | 1.85 GB/s   | -6.10%   |
 | 🌀   | ultrafast |         4 |  1093 fps | 3.02 GB/s   | 53.13%   |
@@ -131,9 +137,9 @@ ffmpeg.wait()
 | 🐢    | Null      |         4 |   390 fps | 2.43 GB/s   |         |
 | 🚀    | Null      |         4 |   391 fps | 2.43 GB/s   | 0.26%   |
 | 🌀    | Null      |         4 |   756 fps | 4.71 GB/s   | 94.01%  |
-| 🐢    | ultrafast |         4 |   277 fps | 1.73 GB/s   |         |
-| 🚀    | ultrafast |         4 |   270 fps | 1.68 GB/s   | -2.40%  |
-| 🌀    | ultrafast |         4 |   402 fps | 2.50 GB/s   | 45.32%  |
+| 🐢    | ultrafast |         4 |   269 fps | 1.68 GB/s   |         |
+| 🚀    | ultrafast |         4 |   272 fps | 1.70 GB/s   | 1.48%   |
+| 🌀    | ultrafast |         4 |   409 fps | 2.55 GB/s   | 52.29%  |
 | 🐢    | slow      |         4 |   115 fps | 0.72 GB/s   |         |
 | 🚀    | slow      |         4 |   118 fps | 0.74 GB/s   | 3.40%   |
 | 🌀    | slow      |         4 |   119 fps | 0.75 GB/s   | 4.34%   |
@@ -143,9 +149,9 @@ ffmpeg.wait()
 | 🐢    | Null      |         1 |   210 fps | 2.33 GB/s   |          |
 | 🚀    | Null      |         1 |   239 fps | 2.64 GB/s   | 13.84%   |
 | 🌀    | Null      |         1 |   534 fps | 5.91 GB/s   | 154.32%  |
-| 🐢    | Null      |         4 |   233 fps | 2.58 GB/s   |          |
-| 🚀    | Null      |         4 |   232 fps | 2.57 GB/s   | -0.08%   |
-| 🌀    | Null      |         4 |   495 fps | 5.48 GB/s   | 112.64%  |
+| 🐢    | Null      |         4 |   219 fps | 2.43 GB/s   |          |
+| 🚀    | Null      |         4 |   231 fps | 2.56 GB/s   | 5.64%    |
+| 🌀    | Null      |         4 |   503 fps | 5.56 GB/s   | 129.75%  |
 | 🐢    | ultrafast |         4 |   141 fps | 1.56 GB/s   |          |
 | 🚀    | ultrafast |         4 |   150 fps | 1.67 GB/s   | 6.92%    |
 | 🌀    | ultrafast |         4 |   226 fps | 2.50 GB/s   | 60.37%   |
@@ -325,9 +331,7 @@ On realistically loads, like [**ShaderFlow**](https://github.com/BrokenSource/Sh
 
 # 📚 Future work
 
-- Add support for NumPy arrays, memoryviews, and byte-like objects
 - Disable/investigate performance degradation on Windows iGPUs
 - Improve the thread synchronization and/or use a ThreadPool
-- Stabler way for finding mglo struct offsets (moderngl.h?)
 - Maybe use `mmap` instead of chunks writing on Linux
-- Test on MacOS 🙈
+- Test on macOS 🙈

{turbopipe-1.0.5 → turbopipe-1.2.0}/meson.build

@@ -28,7 +28,6 @@ endif
 # ----------------------------------------------|
 # Source files
 
-incdir = include_directories('turbopipe/include')
 source = files('turbopipe/_turbopipe.cpp')
 
 # ----------------------------------------------|
@@ -37,7 +36,6 @@ python = import('python').find_installation()
 
 python.extension_module(
     '_turbopipe', source,
-    include_directories: incdir,
     cpp_args: cpp_args,
     install: true,
     subdir: 'turbopipe'

{turbopipe-1.0.5 → turbopipe-1.2.0}/pyproject.toml

@@ -7,7 +7,7 @@ homepage      = "https://brokensrc.dev"
 [project]
 name        = "turbopipe"
 dynamic     = ["version"]
-description = "🌀 Faster ModernGL Buffer inter process data transfers"
+description = "🌀 Faster MemoryView inter-process data transfers for subprocesses"
 authors     = [{name="Tremeschin", email="29046864+Tremeschin@users.noreply.github.com"}]
 readme      = "Readme.md"
 license     = {file="License.md"}

turbopipe-1.2.0/turbopipe/__init__.py

@@ -0,0 +1,47 @@
+from typing import Optional, Union
+
+from moderngl import Buffer
+
+from turbopipe import _turbopipe
+
+
+def pipe(buffer: Union[Buffer, memoryview], fileno: int) -> None:
+    """
+    Pipe the content of a moderngl.Buffer or memoryview to a file descriptor, fast, threaded and
+    blocking when needed. Call `sync(buffer)` before this, and `sync()` when done for
+
+    Usage:
+        ```python
+        # Assuming `buffer = ctx.buffer(...)`
+        # Note: Use as `fbo.read_into(buffer)`
+
+        # As a open() file
+        with open("file.bin", "wb") as file:
+            turbopipe.pipe(buffer, file)
+
+        # As a subprocess
+        child = subprocess.Popen(..., stdin=subprocess.PIPE)
+        turbopipe.pipe(buffer, child.stdin.fileno())
+        ```
+    """
+    if isinstance(buffer, Buffer):
+        buffer = memoryview(buffer.mglo)
+    _turbopipe.pipe(buffer, fileno)
+    del buffer
+
+def sync(buffer: Optional[Union[Buffer, memoryview]]=None) -> None:
+    """Waits for any pending write operation on a buffer, or 'all buffers' if None, to finish"""
+    if isinstance(buffer, Buffer):
+        buffer = memoryview(buffer.mglo)
+    _turbopipe.sync(buffer)
+    del buffer
+
+def close() -> None:
+    """Syncs and deletes objects"""
+    _turbopipe.close()
+
+__all__ = [
+    "pipe",
+    "sync",
+    "close"
+]

{turbopipe-1.0.5 → turbopipe-1.2.0}/turbopipe/_turbopipe.cpp

@@ -1,6 +1,6 @@
 // ------------------------------------------------------------------------------------------------|
 //
-// TurboPipe - Faster ModernGL Buffer inter process data transfers
+// TurboPipe - Faster ModernGL Buffers inter-process data transfers for subprocesses
 //
 // (c) 2024, Tremeschin, MIT License
 //
@@ -11,7 +11,6 @@
 
 // Standard library
 #include <functional>
-#include <iostream>
 #include <chrono>
 
 // Threading
@@ -24,71 +23,16 @@
 #include <unordered_map>
 #include <deque>
 
-// Third party
-#include "gl_methods.hpp"
-
 #define dict std::unordered_map
 using namespace std;
 
-// ------------------------------------------------------------------------------------------------|
-// ModernGL Types - Courtesy of the moderngl package developers (MIT)
-
-static PyTypeObject* MGLBuffer_type = nullptr;
-
-struct MGLContext;
-struct MGLFramebuffer;
-
-struct MGLBuffer {
-    PyObject_HEAD
-    MGLContext* context;
-    int buffer;
-    Py_ssize_t size;
-    bool dynamic;
-    bool released;
-};
-
-struct MGLContext {
-    PyObject_HEAD
-    PyObject * ctx;
-    PyObject * extensions;
-    MGLFramebuffer * default_framebuffer;
-    MGLFramebuffer * bound_framebuffer;
-    PyObject * includes;
-    int version_code;
-    int max_samples;
-    int max_integer_samples;
-    int max_color_attachments;
-    int max_texture_units;
-    int default_texture_unit;
-    float max_anisotropy;
-    int enable_flags;
-    int front_face;
-    int cull_face;
-    int depth_func;
-    bool depth_clamp;
-    double depth_range[2];
-    int blend_func_src;
-    int blend_func_dst;
-    bool wireframe;
-    bool multisample;
-    int provoking_vertex;
-    float polygon_offset_factor;
-    float polygon_offset_units;
-    GLMethods gl;
-    bool released;
-};
-
 // ------------------------------------------------------------------------------------------------|
 // TurboPipe internals
 
 struct Work {
-    void* map;
+    void* data;
     int file;
     size_t size;
-
-    int hash() {
-        return std::hash<int>()(file) ^ std::hash<void*>()(map);
-    }
 };
 
 class TurboPipe {
@@ -96,22 +40,28 @@ public:
     TurboPipe(): running(true) {}
     ~TurboPipe() {close();}
 
-    void pipe(MGLBuffer* buffer, int file) {
-        const GLMethods& gl = buffer->context->gl;
+    void pipe(PyObject* memoryview, int file) {
+        Py_buffer view = *PyMemoryView_GET_BUFFER(memoryview);
+        this->_pipe(view.buf, view.len, file);
+    }
 
-        gl.BindBuffer(GL_ARRAY_BUFFER, buffer->buffer);
-        void* data = gl.MapBufferRange(GL_ARRAY_BUFFER, 0, buffer->size, GL_MAP_READ_BIT);
-        gl.UnmapBuffer(GL_ARRAY_BUFFER);
+    void sync(PyObject* memoryview=nullptr) {
+        void* data = nullptr;
 
-        this->_pipe(data, buffer->size, file);
-    }
+        if (memoryview != nullptr) {
+            Py_buffer view = *PyMemoryView_GET_BUFFER(memoryview);
+            data = view.buf;
+        }
 
-    void sync() {
-        // Wait for all queues to be empty, as they are erased when
+        // Wait for some or all queues to be empty, as they are erased when
         // each thread's writing loop is done, guaranteeing finish
         for (auto& values: queue) {
-            while (!values.second.empty()) {
-                this_thread::sleep_for(chrono::milliseconds(1));
+            while (true) {
+                if (data != nullptr && values.second.find(data) == values.second.end())
+                    break;
+                if (data == nullptr && values.second.empty())
+                    break;
+                this_thread::sleep_for(chrono::microseconds(200));
             }
         }
     }
@@ -126,8 +76,8 @@ public:
     }
 
 private:
-    dict<int, dict<int, condition_variable>> pending;
-    dict<int, unordered_set<int>> queue;
+    dict<int, dict<void*, condition_variable>> pending;
+    dict<int, unordered_set<void*>> queue;
     dict<int, deque<Work>> stream;
     dict<int, thread> threads;
     dict<int, mutex> mutexes;
@@ -136,20 +86,18 @@ private:
 
     void _pipe(void* data, size_t size, int file) {
         Work work = {data, file, size};
-        int hash = work.hash();
-
         unique_lock<mutex> lock(mutexes[file]);
 
-        // Notify this hash is queued, wait if pending
-        if (!queue[file].insert(hash).second) {
-            pending[file][hash].wait(lock, [this, file, hash] {
-                return queue[file].find(hash) == queue[file].end();
+        // Notify this memory is queued, wait if pending
+        if (!queue[file].insert(data).second) {
+            pending[file][data].wait(lock, [this, file, data] {
+                return queue[file].find(data) == queue[file].end();
             });
         }
 
         // Add another job to the queue
         stream[file].push_back(work);
-        queue[file].insert(hash);
+        queue[file].insert(data);
         this->running = true;
         lock.unlock();
 
@@ -179,13 +127,13 @@ private:
 
             #ifdef _WIN32
                 // Windows doesn't like chunked writes ??
-                write(work.file, (char*) work.map, work.size);
+                write(work.file, (char*) work.data, work.size);
             #else
                 // Optimization: Write in chunks of 4096 (RAM page size)
                 size_t tell = 0;
                 while (tell < work.size) {
                     size_t chunk = min(work.size - tell, static_cast<size_t>(4096));
-                    size_t written = write(work.file, (char*) work.map + tell, chunk);
+                    size_t written = write(work.file, (char*) work.data + tell, chunk);
                     if (written == -1) break;
                     tell += written;
                 }
@@ -193,9 +141,8 @@ private:
 
             // Signal work is done
             lock.lock();
-            int hash = work.hash();
-            pending[file][hash].notify_all();
-            queue[file].erase(hash);
+            pending[file][work.data].notify_all();
+            queue[file].erase(work.data);
             signal.notify_all();
         }
     }
@@ -211,19 +158,30 @@ static PyObject* turbopipe_pipe(
     PyObject* Py_UNUSED(self),
     PyObject* args
 ) {
-    PyObject* buffer;
+    PyObject* memoryview;
     PyObject* file;
-    if (!PyArg_ParseTuple(args, "OO", &buffer, &file))
+    if (!PyArg_ParseTuple(args, "OO", &memoryview, &file))
         return NULL;
-    turbopipe->pipe((MGLBuffer*) buffer, PyLong_AsLong(file));
+    if (!PyMemoryView_Check(memoryview)) {
+        PyErr_SetString(PyExc_TypeError, "Expected a memoryview object");
+        return NULL;
+    }
+    turbopipe->pipe(memoryview, PyLong_AsLong(file));
     Py_RETURN_NONE;
 }
 
 static PyObject* turbopipe_sync(
     PyObject* Py_UNUSED(self),
-    PyObject* Py_UNUSED(args)
+    PyObject* args
 ) {
-    turbopipe->sync();
+    PyObject* memoryview;
+    if (!PyArg_ParseTuple(args, "|O", &memoryview))
+        return NULL;
+    if (memoryview != nullptr && !PyMemoryView_Check(memoryview)) {
+        PyErr_SetString(PyExc_TypeError, "Expected a memoryview object or None");
+        return NULL;
+    }
+    turbopipe->sync(memoryview);
     Py_RETURN_NONE;
 }
 
@@ -244,7 +202,7 @@ static void turbopipe_exit() {
 
 static PyMethodDef TurboPipeMethods[] = {
     {"pipe",  (PyCFunction) turbopipe_pipe,  METH_VARARGS, ""},
-    {"sync",  (PyCFunction) turbopipe_sync,  METH_NOARGS,  ""},
+    {"sync",  (PyCFunction) turbopipe_sync,  METH_VARARGS, ""},
     {"close", (PyCFunction) turbopipe_close, METH_NOARGS,  ""},
     {NULL, NULL, 0, NULL}
 };
@@ -259,9 +217,8 @@ static struct PyModuleDef turbopipe_module = {
 
 PyMODINIT_FUNC PyInit__turbopipe(void) {
     PyObject* module = PyModule_Create(&turbopipe_module);
-    PyObject* moderngl = PyImport_ImportModule("moderngl");
-    PyObject* buffer = PyObject_GetAttrString(moderngl, "Buffer");
-    MGLBuffer_type = (PyTypeObject*) buffer;
+    if (module == NULL)
+        return NULL;
     turbopipe = new TurboPipe();
     Py_AtExit(turbopipe_exit);
     return module;

turbopipe-1.2.0/turbopipe/version.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+__version__ = "1.2.0"
+
+if __name__ == "__main__":
+    print(__version__)

turbopipe-1.0.5/.github/workflows/debug.yaml

@@ -1,42 +0,0 @@
-name: debug
-
-on:
-  workflow_dispatch:
-
-jobs:
-  wheels:
-    name: Build ${{matrix.pyver}} wheels on ${{matrix.os}}
-    runs-on: ${{matrix.os}}
-    strategy:
-      matrix:
-        os: [windows-latest]
-        pyver: [cp311]
-
-    env:
-      CIBW_BUILD: ${{matrix.pyver}}-*
-      CIBW_ARCHS_LINUX: auto
-      CIBW_ARCHS_MACOS: arm64
-      CIBW_ARCHS_WINDOWS: auto
-      CIBW_SKIP: '*musllinux* *i686* *-win32'
-
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-
-      - name: Prepare MSVC
-        if: matrix.os == 'windows-latest'
-        uses: bus1/cabuild/action/msdevshell@v1
-        with:
-          architecture: x64
-
-      - name: deps
-        run: python -m pip install build
-
-      - name: wheels
-        run: python -m build
-
-      - name: upload
-        uses: actions/upload-artifact@v3
-        with:
-          name: package
-          path: dist/*.whl

turbopipe-1.0.5/turbopipe/__init__.py

@@ -1,41 +0,0 @@
-from io import IOBase
-
-from moderngl import Buffer
-
-from turbopipe import _turbopipe
-
-
-def pipe(buffer: Buffer, file: IOBase) -> None:
-    """
-    Pipe the content of a moderngl.Buffer to a file descriptor,
-    Fast, threaded and non-blocking. Call `sync()` when done!
-
-    Usage:
-        ```python
-        # Assuming `buffer = ctx.buffer(...)`
-        # Note: Use as `fbo.read_into(buffer)`
-
-        # As a open() file
-        with open("file.bin", "wb") as file:
-            turbopipe.pipe(buffer, file)
-
-        # As a subprocess
-        child = subprocess.Popen(..., stdin=subprocess.PIPE)
-        turbopipe.pipe(buffer, child.stdin.fileno())
-        ```
-    """
-    _turbopipe.pipe(buffer.mglo, file)
-
-def sync() -> None:
-    """Waits for all jobs to finish"""
-    _turbopipe.sync()
-
-def close() -> None:
-    """Syncs and deletes objects"""
-    _turbopipe.close()
-
-__all__ = [
-    "pipe",
-    "sync",
-    "close"
-]