turbopipe 1.2.2__tar.gz → 1.2.3__tar.gz

turbopipe-1.2.2/.github/workflows/release.yaml → turbopipe-1.2.3/.github/workflows/release.yml

@@ -2,102 +2,93 @@ name: release
 
 on:
   workflow_dispatch:
-  push:
-    paths:
-      - 'turbopipe/version.py'
 
 jobs:
   sdist:
-    name: Package source
+    name: Make sdist
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-
-      - name: deps
-        run: python -m pip install -U pip wheel build
+      - uses: astral-sh/setup-uv@v6
 
-      - name: sdist
-        run: python -m build --sdist -o package
+      - name: Make sdist
+        run: uv build --sdist
 
       - name: upload
         uses: actions/upload-artifact@v4
         with:
           name: package-sdist
-          path: package/*.tar.gz
+          path: dist/*.tar.gz
 
   wheels:
-    name: Build Python wheels on ${{matrix.os}}
+    name: Make wheels for ${{matrix.os}}
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-14]
-
+        os: [
+          ubuntu-latest,
+          windows-latest,
+          macos-latest
+        ]
     env:
-      CIBW_BUILD: cp37-* cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*
-      CIBW_ARCHS_LINUX: auto
-      CIBW_ARCHS_MACOS: arm64 x86_64
-      CIBW_ARCHS_WINDOWS: auto
-      CIBW_SKIP: '*musllinux* *i686* *-win32'
-
+      CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-* cp313t-*
+      CIBW_ENABLE: cpython-freethreading
+      CIBW_BUILD_FRONTEND: "build[uv]"
+      CIBW_ARCHS_LINUX: x86_64 aarch64
+      CIBW_ARCHS_MACOS: x86_64 arm64
+      CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*"
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v6
 
       - name: Install MSVC
         if: matrix.os == 'windows-latest'
         uses: bus1/cabuild/action/msdevshell@v1
 
-      - name: deps
-        run: python -m pip install cibuildwheel==2.21.3
+      - name: Install QEMU
+        if: runner.os == 'linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
 
-      - name: wheels
-        run: python -m cibuildwheel --output-dir package
+      - name: Make wheels
+        run: uvx cibuildwheel==2.23.3 --output-dir dist
 
       - name: upload
         uses: actions/upload-artifact@v4
         with:
-          name: package-wheels-${{ matrix.os }}
-          path: package/*.whl
+          name: package-wheels-${{matrix.os}}
+          path: dist/*.whl
 
   publish:
     needs: [sdist, wheels]
     name: Publish to PyPI
     runs-on: ubuntu-latest
-
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v6
 
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
           pattern: 'package-*'
           merge-multiple: true
-          path: package
+          path: dist
 
-      - name: deps
-        run: python -m pip install -U twine
-
-      - name: publish
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{secrets.PYPI_TOKEN}}
-        run: twine upload package/*
+      - name: Publish
+        run: uv publish dist/*
 
   tag:
     needs: publish
     name: Create Release Tag
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
 
       - name: Get version
-        run: echo VERSION=$(python turbopipe/version.py) >> $GITHUB_ENV
-        shell: bash
+        run: echo VERSION=$(uv run turbopipe/version.py) >> $GITHUB_ENV
 
       - name: Create Release Tag
         run: |

turbopipe-1.2.3/.python-version

@@ -0,0 +1 @@
+3.13

{turbopipe-1.2.2 → turbopipe-1.2.3}/PKG-INFO

@@ -1,41 +1,18 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: turbopipe
-Version: 1.2.2
+Version: 1.2.3
 Summary: 🌀 Faster ModernGL Buffers inter-process data transfers for subprocesses
 Author-Email: Tremeschin <29046864+Tremeschin@users.noreply.github.com>
-License: MIT License
-         
-         Copyright (c) 2024 Gabriel Tremeschin
-         
-         Permission is hereby granted, free of charge, to any person obtaining a copy
-         of this software and associated documentation files (the "Software"), to deal
-         in the Software without restriction, including without limitation the rights
-         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-         copies of the Software, and to permit persons to whom the Software is
-         furnished to do so, subject to the following conditions:
-         
-         The above copyright notice and this permission notice shall be included in all
-         copies or substantial portions of the Software.
-         
-         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-         SOFTWARE.
-         
-Project-URL: issues, https://github.com/BrokenSource/TurboPipe/issues
-Project-URL: repository, https://github.com/BrokenSource/TurboPipe
-Project-URL: documentation, https://github.com/BrokenSource/TurboPipe
-Project-URL: homepage, https://brokensrc.dev
+License-Expression: MIT
+Project-URL: GitHub, https://github.com/BrokenSource/TurboPipe
+Project-URL: Changelog, https://brokensrc.dev/about/changelog
+Project-URL: Funding, https://brokensrc.dev/about/sponsors
+Project-URL: Contact, https://brokensrc.dev/about/contact
+Project-URL: Homepage, https://brokensrc.dev
 Requires-Python: >=3.7
 Requires-Dist: moderngl
 Description-Content-Type: text/markdown
 
-> [!IMPORTANT]
-> <sub>Also check out [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow), where **TurboPipe** shines! 😉</sub>
-<!-- PyPI -->
 <div align="center">
   <a href="https://brokensrc.dev/"><img src="https://raw.githubusercontent.com/BrokenSource/TurboPipe/main/turbopipe/resources/images/turbopipe.png" width="200"></a>
   <h1>TurboPipe</h1>
@@ -57,15 +34,17 @@ Description-Content-Type: text/markdown
 
 The **optimizations** involved are:
 
-- **Zero-copy**: Avoid unnecessary memory copies or allocation (intermediate `buffer.read()`)
+- **Zero-copy**: Avoid unnecessary memory copies or allocation (intermediate `buffer.read`)
 - **C++**: The core of TurboPipe is written in C++ for speed, efficiency and low-level control
-- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy (Unix)
 - **Threaded**:
     - Doesn't block Python code execution, allows to render next frame
     - Decouples the main thread from the I/O thread for performance
+- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy (Unix)
 
 ✅ Don't worry, there's proper **safety** in place. TurboPipe will block Python if a memory address is already queued for writing, and guarantees order of writes per file-descriptor. Just call `.sync()` when done 😉
 
+<sub>Also check out [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow), where **TurboPipe** shines! 😉</sub>
+
 <br>
 
 # 📦 Installation
@@ -90,7 +69,7 @@ rye add turbopipe
 
 # 🚀 Usage
 
-See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for comparisons, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Scene.py) usage of it!
+See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for comparisons, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Exporting.py)'s usage of it!
 
 ```python
 import subprocess
@@ -98,27 +77,53 @@ import subprocess
 import moderngl
 import turbopipe
 
-# Create ModernGL objects
+# Create ModernGL objects and proxy buffers
 ctx = moderngl.create_standalone_context()
-buffers = [ctx.buffer(reserve=1920*1080*3) for _ in range(2)]
+width, height, duration, fps = (1920, 1080, 10, 60)
+buffers = [
+    ctx.buffer(reserve=(width*height*3))
+    for _ in range(nbuffers := 2)
+]
+
+# Create your FBO, Textures, Shaders, etc.
 
 # Make sure resolution, pixel format matches!
-ffmpeg = subprocess.Popen(
-    'ffmpeg -f rawvideo -pix_fmt rgb24 -r 60 -s 1920x1080 -i - -f null -'.split(),
-    stdin=subprocess.PIPE
-)
-
-# Rendering loop of yours (eg. 1m footage)
-for frame in range(60 * 60):
-    buffer = buffers[frame % len(buffer)]
+ffmpeg = subprocess.Popen((
+    "ffmpeg",
+    "-f", "rawvideo",
+    "-pix_fmt", "rgb24",
+    "-r", str(fps),
+    "-s", f"{width}x{height}",
+    "-i", "-",
+    "-f", "null",
+    "output.mp4"
+), stdin=subprocess.PIPE)
+
+# Rendering loop of yours
+for frame in range(duration*fps):
+    buffer = buffers[frame % nbuffers]
+
+    # Wait queued writes before copying
     turbopipe.sync(buffer)
     fbo.read_into(buffer)
+
+    # Doesn't lock the GIL, writes in parallel
     turbopipe.pipe(buffer, ffmpeg.stdin.fileno())
 
-# Finalize writing, encoding
+# Wait for queued writes, clean memory
+for buffer in buffers:
+    turbopipe.sync(buffer)
+    buffer.release()
+
+# Signal stdin stream is done
 ffmpeg.stdin.close()
-turbopipe.close()
+
+# wait for encoding to finish
 ffmpeg.wait()
+
+# Warn: Albeit rare, only call close when no other data
+# write is pending, as it might skip a frame or halt
+turbopipe.close()
 ```
 
 <br>
@@ -370,5 +375,5 @@ On realistically loads, like [**ShaderFlow**](https://github.com/BrokenSource/Sh
 # 📚 Future work
 
 - Disable/investigate performance degradation on Windows iGPUs
-- Improve the thread synchronization and/or use a ThreadPool
 - Maybe use `mmap` instead of chunks writing on Linux
+- Split the code into a libturbopipe? Not sure where it would be useful 😅

{turbopipe-1.2.2 → turbopipe-1.2.3}/meson.build

@@ -8,7 +8,7 @@ project('turbopipe', 'cpp',
 
     default_options: [
         'warning_level=3',
-        'cpp_std=c++14',
+        'cpp_std=c++20',
         'buildtype=release',
         'optimization=3'
     ]

{turbopipe-1.2.2 → turbopipe-1.2.3}/pyproject.toml

@@ -1,16 +1,17 @@
 [project.urls]
-issues        = "https://github.com/BrokenSource/TurboPipe/issues"
-repository    = "https://github.com/BrokenSource/TurboPipe"
-documentation = "https://github.com/BrokenSource/TurboPipe"
-homepage      = "https://brokensrc.dev"
+GitHub    = "https://github.com/BrokenSource/TurboPipe"
+Changelog = "https://brokensrc.dev/about/changelog"
+Funding   = "https://brokensrc.dev/about/sponsors"
+Contact   = "https://brokensrc.dev/about/contact"
+Homepage  = "https://brokensrc.dev"
 
 [project]
 name        = "turbopipe"
-dynamic     = ["version"]
 description = "🌀 Faster ModernGL Buffers inter-process data transfers for subprocesses"
 authors     = [{name="Tremeschin", email="29046864+Tremeschin@users.noreply.github.com"}]
-readme      = "Readme.md"
-license     = {file="License.md"}
+dynamic     = ["version"]
+readme      = "readme.md"
+license     = "MIT"
 dependencies = ["moderngl"]
 requires-python = ">=3.7"
 
@@ -18,5 +19,8 @@ requires-python = ">=3.7"
 requires = ["meson-python", "ninja"]
 build-backend = "mesonpy"
 
+[tool.uv]
+managed = false
+
 [tool.ruff.format]
 exclude = ["*"]

turbopipe-1.2.2/Readme.md → turbopipe-1.2.3/readme.md

@@ -1,6 +1,3 @@
-> [!IMPORTANT]
-> <sub>Also check out [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow), where **TurboPipe** shines! 😉</sub>
-<!-- PyPI -->
 <div align="center">
   <a href="https://brokensrc.dev/"><img src="https://raw.githubusercontent.com/BrokenSource/TurboPipe/main/turbopipe/resources/images/turbopipe.png" width="200"></a>
   <h1>TurboPipe</h1>
@@ -22,15 +19,17 @@
 
 The **optimizations** involved are:
 
-- **Zero-copy**: Avoid unnecessary memory copies or allocation (intermediate `buffer.read()`)
+- **Zero-copy**: Avoid unnecessary memory copies or allocation (intermediate `buffer.read`)
 - **C++**: The core of TurboPipe is written in C++ for speed, efficiency and low-level control
-- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy (Unix)
 - **Threaded**:
     - Doesn't block Python code execution, allows to render next frame
     - Decouples the main thread from the I/O thread for performance
+- **Chunks**: Write in chunks of 4096 bytes (RAM page size), so the hardware is happy (Unix)
 
 ✅ Don't worry, there's proper **safety** in place. TurboPipe will block Python if a memory address is already queued for writing, and guarantees order of writes per file-descriptor. Just call `.sync()` when done 😉
 
+<sub>Also check out [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow), where **TurboPipe** shines! 😉</sub>
+
 <br>
 
 # 📦 Installation
@@ -55,7 +54,7 @@ rye add turbopipe
 
 # 🚀 Usage
 
-See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for comparisons, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Scene.py) usage of it!
+See also the [**Examples**](https://github.com/BrokenSource/TurboPipe/tree/main/examples) folder for comparisons, and [**ShaderFlow**](https://github.com/BrokenSource/ShaderFlow/blob/main/ShaderFlow/Exporting.py)'s usage of it!
 
 ```python
 import subprocess
@@ -63,27 +62,53 @@ import subprocess
 import moderngl
 import turbopipe
 
-# Create ModernGL objects
+# Create ModernGL objects and proxy buffers
 ctx = moderngl.create_standalone_context()
-buffers = [ctx.buffer(reserve=1920*1080*3) for _ in range(2)]
+width, height, duration, fps = (1920, 1080, 10, 60)
+buffers = [
+    ctx.buffer(reserve=(width*height*3))
+    for _ in range(nbuffers := 2)
+]
+
+# Create your FBO, Textures, Shaders, etc.
 
 # Make sure resolution, pixel format matches!
-ffmpeg = subprocess.Popen(
-    'ffmpeg -f rawvideo -pix_fmt rgb24 -r 60 -s 1920x1080 -i - -f null -'.split(),
-    stdin=subprocess.PIPE
-)
-
-# Rendering loop of yours (eg. 1m footage)
-for frame in range(60 * 60):
-    buffer = buffers[frame % len(buffer)]
+ffmpeg = subprocess.Popen((
+    "ffmpeg",
+    "-f", "rawvideo",
+    "-pix_fmt", "rgb24",
+    "-r", str(fps),
+    "-s", f"{width}x{height}",
+    "-i", "-",
+    "-f", "null",
+    "output.mp4"
+), stdin=subprocess.PIPE)
+
+# Rendering loop of yours
+for frame in range(duration*fps):
+    buffer = buffers[frame % nbuffers]
+
+    # Wait queued writes before copying
     turbopipe.sync(buffer)
     fbo.read_into(buffer)
+
+    # Doesn't lock the GIL, writes in parallel
     turbopipe.pipe(buffer, ffmpeg.stdin.fileno())
 
-# Finalize writing, encoding
+# Wait for queued writes, clean memory
+for buffer in buffers:
+    turbopipe.sync(buffer)
+    buffer.release()
+
+# Signal stdin stream is done
 ffmpeg.stdin.close()
-turbopipe.close()
+
+# wait for encoding to finish
 ffmpeg.wait()
+
+# Warn: Albeit rare, only call close when no other data
+# write is pending, as it might skip a frame or halt
+turbopipe.close()
 ```
 
 <br>
@@ -335,5 +360,5 @@ On realistically loads, like [**ShaderFlow**](https://github.com/BrokenSource/Sh
 # 📚 Future work
 
 - Disable/investigate performance degradation on Windows iGPUs
-- Improve the thread synchronization and/or use a ThreadPool
 - Maybe use `mmap` instead of chunks writing on Linux
+- Split the code into a libturbopipe? Not sure where it would be useful 😅

turbopipe-1.2.3/turbopipe/__init__.py

@@ -0,0 +1,29 @@
+from typing import Optional, Union
+
+from moderngl import Buffer
+
+from turbopipe import _turbopipe
+
+__all__ = [
+    "pipe",
+    "sync",
+    "close"
+]
+
+def pipe(buffer: Union[Buffer, memoryview], fileno: int) -> None:
+    """Pipe a buffer contents to a file descriptor, fast and threaded"""
+    if isinstance(buffer, Buffer):
+        buffer = memoryview(buffer.mglo)
+    _turbopipe.pipe(buffer, fileno)
+    del buffer
+
+def sync(buffer: Optional[Union[Buffer, memoryview]]=None) -> None:
+    """Wait for pending operations on a buffer to finish"""
+    if isinstance(buffer, Buffer):
+        buffer = memoryview(buffer.mglo)
+    _turbopipe.sync(buffer)
+    del buffer
+
+def close() -> None:
+    """Syncs and deletes objects"""
+    _turbopipe.close()

{turbopipe-1.2.2 → turbopipe-1.2.3}/turbopipe/_turbopipe.cpp

@@ -1,9 +1,6 @@
 // ------------------------------------------------------------------------------------------------|
-//
 // TurboPipe - Faster ModernGL Buffers inter-process data transfers for subprocesses
-//
-// (c) 2024, Tremeschin, MIT License
-//
+// (c) MIT License 2024-2025, Tremeschin
 // ------------------------------------------------------------------------------------------------|
 
 #define PY_SSIZE_T_CLEAN
@@ -29,9 +26,9 @@ using namespace std;
 // TurboPipe internals
 
 struct Work {
-    void* data;
-    int file;
+    void*  data;
     size_t size;
+    int    file;
 };
 
 class TurboPipe {
@@ -39,108 +36,110 @@ public:
     TurboPipe(): running(true) {}
     ~TurboPipe() {close();}
 
-    void pipe(PyObject* memoryview, int file) {
-        Py_buffer view = *PyMemoryView_GET_BUFFER(memoryview);
-        this->_pipe(view.buf, view.len, file);
+    void pipe(PyObject* view, int file) {
+        Py_buffer data = *PyMemoryView_GET_BUFFER(view);
+        this->_pipe(data.buf, (size_t) data.len, file);
     }
 
-    void sync(PyObject* memoryview=nullptr) {
-        void* data = nullptr;
-
-        if (memoryview != nullptr) {
-            Py_buffer view = *PyMemoryView_GET_BUFFER(memoryview);
-            data = view.buf;
-        }
-
-        // Wait for some or all queues to be empty, as they are erased when
-        // each thread's writing loop is done, guaranteeing finish
-        for (auto& values: queue) {
-            while (true) {
-                {
-                    // Prevent segfault on iteration on changing data
-                    lock_guard<mutex> lock(mutexes[values.first]);
-
-                    // Either all empty or some memory not queued (None or specific)
-                    if (data != nullptr && values.second.find(data) == values.second.end())
-                        break;
-                    if (data == nullptr && values.second.empty())
-                        break;
-                }
-                this_thread::sleep_for(chrono::microseconds(200));
-            }
-        }
+    void sync(PyObject* view=nullptr) {
+        if (view != nullptr)
+            this->_sync((*PyMemoryView_GET_BUFFER(view)).buf);
+        else
+            this->_sync(nullptr);
     }
 
     void close() {
-        sync();
-        running = false;
-        signal.notify_all();
-        for (auto& pair: threads)
+        this->_sync();
+        this->running = false;
+        for (auto& pair: this->signal)
+            pair.second.notify_all();
+        for (auto& pair: this->threads)
             pair.second.join();
-        threads.clear();
+        this->threads.clear();
     }
 
 private:
-    unordered_map<int, unordered_map<void*, condition_variable>> pending;
+    unordered_map<int, condition_variable> pending;
+    unordered_map<int, condition_variable> signal;
     unordered_map<int, unordered_set<void*>> queue;
     unordered_map<int, deque<Work>> stream;
     unordered_map<int, thread> threads;
     unordered_map<int, mutex> mutexes;
-    condition_variable signal;
     bool running;
 
     void _pipe(void* data, size_t size, int file) {
-        Work work = {data, file, size};
-        unique_lock<mutex> lock(mutexes[file]);
+        unique_lock<mutex> lock(this->mutexes[file]);
 
         /* Notify this memory is queued, wait if pending */ {
-            if (!queue[file].insert(data).second) {
-                pending[file][data].wait(lock, [this, file, data] {
-                    return queue[file].find(data) == queue[file].end();
+            if (!this->queue[file].insert(data).second) {
+                this->pending[file].wait(lock, [this, file, data] {
+                    return this->queue[file].find(data) == this->queue[file].end();
                 });
             }
         }
 
         /* Add another job to the queue */ {
-            stream[file].push_back(work);
-            queue[file].insert(data);
+            this->stream[file].push_back(Work{data, size, file});
+            this->queue[file].insert(data);
             this->running = true;
             lock.unlock();
         }
 
         // Each file descriptor has its own thread
-        if (threads.find(file) == threads.end())
-            threads[file] = thread(&TurboPipe::worker, this, file);
+        if (this->threads.find(file) == this->threads.end())
+            this->threads[file] = thread(&TurboPipe::worker, this, file);
 
-        signal.notify_all();
+        // Trigger the worker to write the data
+        this->signal[file].notify_all();
+    }
+
+    void _sync(void* data=nullptr) {
+        for (auto& values: this->queue) {
+            while (true) {
+                {
+                    // Prevent segfault on iteration on changing data
+                    lock_guard<mutex> lock(this->mutexes[values.first]);
+
+                    // Continue if specific data is not in queue
+                    if (data != nullptr)
+                        if (values.second.find(data) == values.second.end())
+                            break;
+
+                    // Continue if all queues are empty
+                    if (data == nullptr)
+                        if (values.second.empty())
+                            break;
+                }
+                this_thread::sleep_for(chrono::microseconds(200));
+            }
+        }
     }
 
     void worker(int file) {
         while (this->running) {
-            unique_lock<mutex> lock(mutexes[file]);
+            unique_lock<mutex> lock(this->mutexes[file]);
 
-            signal.wait(lock, [this, file] {
-                return (!stream[file].empty() || !this->running);
+            this->signal[file].wait(lock, [this, file] {
+                return (!this->stream[file].empty() || !this->running);
             });
 
             // Skip on false positives, exit condition
-            if (stream[file].empty()) continue;
+            if ( this->stream[file].empty()) continue;
             if (!this->running) break;
 
             // Get the next work item
-            Work work = stream[file].front();
-            stream[file].pop_front();
+            Work work = this->stream[file].front();
+            this->stream[file].pop_front();
             lock.unlock();
 
             #ifdef _WIN32
-                // Windows doesn't like chunked writes ??
-                write(work.file, (char*) work.data, work.size);
+                // Fixme: Windows doesn't like chunked writes?
+                write(work.file, (char*) work.data, static_cast<unsigned int>(work.size));
             #else
-                // Optimization: Write in chunks of 4096 (RAM page size)
                 size_t tell = 0;
                 while (tell < work.size) {
                     size_t chunk = min(work.size - tell, static_cast<size_t>(4096));
-                    size_t written = write(work.file, (char*) work.data + tell, chunk);
+                    int written = write(work.file, (char*) work.data + tell, chunk);
                     if (written == -1) break;
                     tell += written;
                 }
@@ -149,9 +148,9 @@ private:
             lock.lock();
 
             /* Signal work is done */ {
-                pending[file][work.data].notify_all();
-                queue[file].erase(work.data);
-                signal.notify_all();
+                this->pending[file].notify_all();
+                this->queue[file].erase(work.data);
+                this->signal[file].notify_all();
             }
         }
     }
@@ -167,15 +166,15 @@ static PyObject* turbopipe_pipe(
     PyObject* Py_UNUSED(self),
     PyObject* args
 ) {
-    PyObject* memoryview;
+    PyObject* view;
     PyObject* file;
-    if (!PyArg_ParseTuple(args, "OO", &memoryview, &file))
+    if (!PyArg_ParseTuple(args, "OO", &view, &file))
         return NULL;
-    if (!PyMemoryView_Check(memoryview)) {
+    if (!PyMemoryView_Check(view)) {
         PyErr_SetString(PyExc_TypeError, "Expected a memoryview object");
         return NULL;
     }
-    turbopipe->pipe(memoryview, PyLong_AsLong(file));
+    turbopipe->pipe(view, PyLong_AsLong(file));
     Py_RETURN_NONE;
 }
 
@@ -183,14 +182,14 @@ static PyObject* turbopipe_sync(
     PyObject* Py_UNUSED(self),
     PyObject* args
 ) {
-    PyObject* memoryview;
-    if (!PyArg_ParseTuple(args, "|O", &memoryview))
+    PyObject* view;
+    if (!PyArg_ParseTuple(args, "|O", &view))
         return NULL;
-    if (memoryview != nullptr && !PyMemoryView_Check(memoryview)) {
+    if (view != nullptr && !PyMemoryView_Check(view)) {
         PyErr_SetString(PyExc_TypeError, "Expected a memoryview object or None");
         return NULL;
     }
-    turbopipe->sync(memoryview);
+    turbopipe->sync(view);
     Py_RETURN_NONE;
 }
 
@@ -216,18 +215,25 @@ static PyMethodDef TurboPipeMethods[] = {
     {NULL, NULL, 0, NULL}
 };
 
-static struct PyModuleDef turbopipe_module = {
-    PyModuleDef_HEAD_INIT,
-    "_turbopipe",
-    NULL, -1,
-    TurboPipeMethods,
-    NULL, NULL, NULL, NULL
+static struct PyModuleDef TurboPipeModule = {
+    .m_base     = PyModuleDef_HEAD_INIT,
+    .m_name     = "_turbopipe",
+    .m_doc      = NULL,
+    .m_size     = -1,
+    .m_methods  = TurboPipeMethods,
+    .m_slots    = NULL,
+    .m_traverse = NULL,
+    .m_clear    = NULL,
+    .m_free     = NULL
 };
 
 PyMODINIT_FUNC PyInit__turbopipe(void) {
-    PyObject* module = PyModule_Create(&turbopipe_module);
+    PyObject* module = PyModule_Create(&TurboPipeModule);
     if (module == NULL)
         return NULL;
+    #ifdef Py_GIL_DISABLED
+        PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+    #endif
     turbopipe = new TurboPipe();
     Py_AtExit(turbopipe_exit);
     return module;

{turbopipe-1.2.2 → turbopipe-1.2.3}/turbopipe/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-__version__ = "1.2.2"
+__version__ = "1.2.3"
 
 if __name__ == "__main__":
     print(__version__)

turbopipe-1.2.2/turbopipe/__init__.py

@@ -1,47 +0,0 @@
-from typing import Optional, Union
-
-from moderngl import Buffer
-
-from turbopipe import _turbopipe
-
-
-def pipe(buffer: Union[Buffer, memoryview], fileno: int) -> None:
-    """
-    Pipe the content of a moderngl.Buffer or memoryview to a file descriptor, fast, threaded and
-    blocking when needed. Call `sync(buffer)` before this, and `sync()` when done for
-
-    Usage:
-        ```python
-        # Assuming `buffer = ctx.buffer(...)`
-        # Note: Use as `fbo.read_into(buffer)`
-
-        # As a open() file
-        with open("file.bin", "wb") as file:
-            turbopipe.pipe(buffer, file)
-
-        # As a subprocess
-        child = subprocess.Popen(..., stdin=subprocess.PIPE)
-        turbopipe.pipe(buffer, child.stdin.fileno())
-        ```
-    """
-    if isinstance(buffer, Buffer):
-        buffer = memoryview(buffer.mglo)
-    _turbopipe.pipe(buffer, fileno)
-    del buffer
-
-def sync(buffer: Optional[Union[Buffer, memoryview]]=None) -> None:
-    """Waits for any pending write operation on a buffer, or 'all buffers' if None, to finish"""
-    if isinstance(buffer, Buffer):
-        buffer = memoryview(buffer.mglo)
-    _turbopipe.sync(buffer)
-    del buffer
-
-def close() -> None:
-    """Syncs and deletes objects"""
-    _turbopipe.close()
-
-__all__ = [
-    "pipe",
-    "sync",
-    "close"
-]