turbopipe 0.0.0__tar.gz → 1.0.0__tar.gz

turbopipe-1.0.0/.gitattributes

@@ -0,0 +1,2 @@
+turbopipe/include/*.hpp linguist-vendored
+turbopipe/include/*.h linguist-vendored

turbopipe-1.0.0/.github/workflows/release.yaml

@@ -0,0 +1,83 @@
+name: release
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'turbopipe/version.py'
+
+jobs:
+  sdist:
+    name: Package source
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+
+      - name: deps
+        run: python -m pip install -U pip wheel build
+
+      - name: sdist
+        run: python -m build --sdist -o package
+
+      - name: upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: package
+          path: package/*.tar.gz
+
+  wheels:
+    name: Build ${{matrix.pyver}} wheels on ${{matrix.os}}
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        pyver: [cp39, cp310, cp311, cp312]
+
+    env:
+      CIBW_BUILD: ${{matrix.pyver}}-*
+      CIBW_ARCHS_LINUX: auto
+      CIBW_ARCHS_MACOS: arm64
+      CIBW_ARCHS_WINDOWS: auto
+      CIBW_SKIP: '*musllinux* *i686* *-win32'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+
+      - name: deps
+        run: python -m pip install cibuildwheel==2.19.2
+
+      - name: wheels
+        run: python -m cibuildwheel --output-dir package
+
+      - name: upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: package
+          path: package/*.whl
+
+  publish:
+    needs: [sdist, wheels]
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: package
+          path: package
+
+      - name: deps
+        run: python -m pip install -U twine
+
+      - name: publish
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{secrets.PYPI_TOKEN}}
+        run: twine upload package/*

turbopipe-1.0.0/.gitignore

@@ -0,0 +1,4 @@
+turbopipe.egg*
+build*
+.venv
+*.so

turbopipe-1.0.0/License.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Gabriel Tremeschin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

turbopipe-1.0.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.1
+Name: turbopipe
+Version: 1.0.0
+Summary: 🌀 Faster ModernGL Buffer inter process data transfers
+Home-page: https://brokensrc.dev
+Author-Email: Tremeschin <29046864+Tremeschin@users.noreply.github.com>
+License: MIT License
+        
+        Copyright (c) 2024 Gabriel Tremeschin
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: Issues, https://github.com/BrokenSource/TurboPipe/issues
+Project-URL: Repository, https://github.com/BrokenSource/TurboPipe
+Project-URL: Documentation, https://github.com/BrokenSource/TurboPipe
+Project-URL: Homepage, https://brokensrc.dev
+Requires-Python: >=3.7
+Requires-Dist: moderngl
+Description-Content-Type: text/markdown

turbopipe-1.0.0/examples/basic.py

@@ -0,0 +1,80 @@
+import contextlib
+import random
+import subprocess
+from typing import Generator
+
+import moderngl
+import tqdm
+import turbopipe
+
+# User constants
+WIDTH, HEIGHT = (1920, 1080)
+FRAMERATE = 60
+DURATION = 30
+
+# Calculate constants
+BYTES_PER_FRAME = (WIDTH * HEIGHT * 3)
+TOTAL_FRAMES = (DURATION * FRAMERATE)
+TOTAL_BYTES = (BYTES_PER_FRAME * TOTAL_FRAMES)
+
+# Create ModernGL objects
+ctx = moderngl.create_standalone_context()
+buffer = ctx.buffer(reserve=BYTES_PER_FRAME)
+
+# Let's play fair and avoid any OS/Python/Hardware optimizations
+buffer.write(bytearray(random.getrandbits(8) for _ in range(BYTES_PER_FRAME)))
+
+# -------------------------------------------------------------------------------------------------|
+
+@contextlib.contextmanager
+def FFmpeg() -> Generator[subprocess.Popen, None, None]:
+    try:
+        ffmpeg = subprocess.Popen([
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel", "error",
+            "-f", "rawvideo",
+            "-pix_fmt", "rgb24",
+            "-s", f"{WIDTH}x{HEIGHT}",
+            "-r", str(FRAMERATE),
+            "-i", "-",
+            "-f", "null",
+            "-", "-y"
+        ], stdin=subprocess.PIPE)
+
+        yield ffmpeg
+    finally:
+        ffmpeg.stdin.close()
+        ffmpeg.wait()
+
+# -------------------------------------------------------------------------------------------------|
+
+@contextlib.contextmanager
+def Progress():
+    with tqdm.tqdm(total=TOTAL_FRAMES, unit="Frame", smoothing=0) as frame_bar, \
+         tqdm.tqdm(total=TOTAL_BYTES,  unit="B", smoothing=0, unit_scale=True) as byte_bar:
+        def next():
+            byte_bar.update(BYTES_PER_FRAME)
+            frame_bar.update(1)
+        yield next
+
+# -------------------------------------------------------------------------------------------------|
+
+print("\n:: Traditional method\n")
+
+with Progress() as progress, FFmpeg() as ffmpeg:
+    for frame in range(TOTAL_FRAMES):
+        ffmpeg.stdin.write(buffer.read())
+        progress()
+
+print("\n:: TurboPipe method\n")
+
+with Progress() as progress, FFmpeg() as ffmpeg:
+    fileno = ffmpeg.stdin.fileno()
+
+    for frame in range(TOTAL_FRAMES):
+        turbopipe.pipe(buffer, fileno)
+        progress()
+    turbopipe.sync()
+
+turbopipe.close()

turbopipe-1.0.0/meson.build

@@ -0,0 +1,50 @@
+# ----------------------------------------------|
+
+project('turbopipe', 'cpp',
+    version: run_command(
+        ['turbopipe/version.py'],
+        check: true
+    ).stdout().strip(),
+
+    default_options: [
+        'warning_level=3',
+        'cpp_std=c++11',
+        'buildtype=release',
+        'optimization=3'
+    ]
+)
+
+# ----------------------------------------------|
+# Platform flags
+
+cpp_args = ['-fpermissive']
+
+if host_machine.system() == 'darwin'
+    cpp_args += ['-Wno-deprecated-declarations']
+endif
+
+# ----------------------------------------------|
+# Source files
+
+incdir = include_directories('turbopipe/include')
+source = files('turbopipe/_turbopipe.cpp')
+
+# ----------------------------------------------|
+
+python = import('python').find_installation()
+
+python.extension_module(
+    '_turbopipe', source,
+    include_directories: incdir,
+    cpp_args: cpp_args,
+    install: true,
+    subdir: 'turbopipe'
+)
+
+python.install_sources(
+    ['turbopipe/__init__.py'],
+    subdir: 'turbopipe',
+    pure: false
+)
+
+# ----------------------------------------------|

turbopipe-1.0.0/pyproject.toml

@@ -0,0 +1,19 @@
+[project.urls]
+issues        = "https://github.com/BrokenSource/TurboPipe/issues"
+repository    = "https://github.com/BrokenSource/TurboPipe"
+documentation = "https://github.com/BrokenSource/TurboPipe"
+homepage      = "https://brokensrc.dev"
+
+[project]
+name        = "turbopipe"
+dynamic     = ["version"]
+description = "🌀 Faster ModernGL Buffer inter process data transfers"
+authors     = [{name="Tremeschin", email="29046864+Tremeschin@users.noreply.github.com"}]
+readme      = "Readme.md"
+license     = {file="License.md"}
+dependencies = ["moderngl"]
+requires-python = ">=3.7"
+
+[build-system]
+requires = ["meson-python", "ninja"]
+build-backend = "mesonpy"

turbopipe-1.0.0/turbopipe/__init__.py

@@ -0,0 +1,41 @@
+from io import IOBase
+
+from moderngl import Buffer
+
+from turbopipe import _turbopipe
+
+
+def pipe(buffer: Buffer, file: IOBase) -> None:
+    """
+    Pipe the content of a moderngl.Buffer to a file descriptor,
+    Fast, threaded and non-blocking. Call `sync()` when done!
+
+    Usage:
+        ```python
+        # Assuming `buffer = ctx.buffer(...)`
+        # Note: Use as `fbo.read_into(buffer)`
+
+        # As a open() file
+        with open("file.bin", "wb") as file:
+            turbopipe.pipe(buffer, file)
+
+        # As a subprocess
+        child = subprocess.Popen(..., stdin=subprocess.PIPE)
+        turbopipe.pipe(buffer, child.stdin.fileno())
+        ```
+    """
+    _turbopipe.pipe(buffer.mglo, file)
+
+def sync() -> None:
+    """Waits for all jobs to finish"""
+    _turbopipe.sync()
+
+def close() -> None:
+    """Syncs and deletes objects"""
+    _turbopipe.close()
+
+__all__ = [
+    "pipe",
+    "sync",
+    "close"
+]

turbopipe-1.0.0/turbopipe/_turbopipe.cpp

@@ -0,0 +1,258 @@
+// ------------------------------------------------------------------------------------------------|
+//
+// TurboPipe - Faster ModernGL Buffer inter process data transfers
+//
+// (c) 2024, Tremeschin, MIT License
+//
+// ------------------------------------------------------------------------------------------------|
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <unistd.h>
+#include <iostream>
+#include <unordered_map>
+#include <deque>
+#include <chrono>
+#include <functional>
+#include <set>
+
+#include "gl_methods.hpp"
+
+#define dict std::unordered_map
+using namespace std;
+
+// ------------------------------------------------------------------------------------------------|
+// ModernGL Types - Courtesy of the moderngl package developers (MIT)
+
+static PyTypeObject* MGLBuffer_type = nullptr;
+
+struct MGLContext;
+struct MGLFramebuffer;
+
+struct MGLBuffer {
+    PyObject_HEAD
+    MGLContext* context;
+    int buffer;
+    Py_ssize_t size;
+    bool dynamic;
+    bool released;
+};
+
+struct MGLContext {
+    PyObject_HEAD
+    PyObject * ctx;
+    PyObject * extensions;
+    MGLFramebuffer * default_framebuffer;
+    MGLFramebuffer * bound_framebuffer;
+    PyObject * includes;
+    int version_code;
+    int max_samples;
+    int max_integer_samples;
+    int max_color_attachments;
+    int max_texture_units;
+    int default_texture_unit;
+    float max_anisotropy;
+    int enable_flags;
+    int front_face;
+    int cull_face;
+    int depth_func;
+    bool depth_clamp;
+    double depth_range[2];
+    int blend_func_src;
+    int blend_func_dst;
+    bool wireframe;
+    bool multisample;
+    int provoking_vertex;
+    float polygon_offset_factor;
+    float polygon_offset_units;
+    GLMethods gl;
+    bool released;
+};
+
+// ------------------------------------------------------------------------------------------------|
+// TurboPipe internals
+
+struct Work {
+    void* map;
+    int file;
+    size_t size;
+
+    int hash() {
+        return std::hash<int>()(file) ^ std::hash<void*>()(map);
+    }
+};
+
+class TurboPipe {
+public:
+    TurboPipe(): running(true) {}
+    ~TurboPipe() {close();}
+
+    void pipe(MGLBuffer* buffer, int file) {
+        const GLMethods& gl = buffer->context->gl;
+
+        gl.BindBuffer(GL_ARRAY_BUFFER, buffer->buffer);
+        void* data = gl.MapBufferRange(GL_ARRAY_BUFFER, 0, buffer->size, GL_MAP_READ_BIT);
+        gl.UnmapBuffer(GL_ARRAY_BUFFER);
+
+        this->_pipe(data, buffer->size, file);
+    }
+
+    void sync() {
+        // Wait for all queues to be empty, as they are erased when
+        // each thread's writing loop is done, guaranteeing finish
+        for (auto& values: queue) {
+            while (!values.second.empty()) {
+                this_thread::sleep_for(chrono::milliseconds(1));
+            }
+        }
+    }
+
+    void close() {
+        sync();
+        running = false;
+        signal.notify_all();
+        for (auto& pair: threads)
+            pair.second.join();
+        threads.clear();
+    }
+
+private:
+    dict<int, dict<int, condition_variable>> pending;
+    dict<int, deque<Work>> stream;
+    dict<int, set<int>> queue;
+    dict<int, thread> threads;
+    dict<int, mutex> mutexes;
+    condition_variable signal;
+    bool running;
+
+    void _pipe(void* data, size_t size, int file) {
+        Work work = {data, file, size};
+        int hash = work.hash();
+
+        unique_lock<mutex> lock(mutexes[file]);
+
+        // Notify this hash is queued, wait if pending
+        if (!queue[file].insert(hash).second) {
+            pending[file][hash].wait(lock, [this, file, hash] {
+                return queue[file].find(hash) == queue[file].end();
+            });
+            pending[file].erase(hash);
+            queue[file].insert(hash);
+        }
+
+        // Add another job to the queue
+        stream[file].push_back(work);
+        lock.unlock();
+
+        // Each file descriptor has its own thread
+        if (threads.find(file) == threads.end())
+            threads[file] = thread(&TurboPipe::worker, this, file);
+
+        signal.notify_all();
+    }
+
+    void worker(int file) {
+        while (this->running) {
+            unique_lock<mutex> lock(mutexes[file]);
+
+            signal.wait(lock, [this, file] {
+                return (!stream[file].empty() || !this->running);
+            });
+
+            // Skip on false positives, exit condition
+            if (stream[file].empty()) continue;
+            if (!this->running) break;
+
+            // Get the next work item
+            Work work = stream[file].front();
+            stream[file].pop_front();
+            lock.unlock();
+
+            // Optimization: Write in chunks of 4096 (RAM page size)
+            size_t tell = 0;
+            while (tell < work.size) {
+                size_t chunk = min(work.size - tell, static_cast<size_t>(4096));
+                ssize_t written = write(work.file, (char*) work.map + tell, chunk);
+                if (written == -1) break;
+                tell += written;
+            }
+
+            // Signal work is done
+            lock.lock();
+            int hash = work.hash();
+            pending[file][hash].notify_all();
+            queue[file].erase(hash);
+            signal.notify_all();
+        }
+    }
+};
+
+// The main and only instance of TurboPipe
+static TurboPipe* turbopipe = nullptr;
+
+// ------------------------------------------------------------------------------------------------|
+// End user methods
+
+static PyObject* turbopipe_pipe(
+    PyObject* Py_UNUSED(self),
+    PyObject* args
+) {
+    PyObject* buffer;
+    PyObject* file;
+    if (!PyArg_ParseTuple(args, "OO", &buffer, &file))
+        return NULL;
+    turbopipe->pipe((MGLBuffer*) buffer, PyLong_AsLong(file));
+    Py_RETURN_NONE;
+}
+
+static PyObject* turbopipe_sync(
+    PyObject* Py_UNUSED(self),
+    PyObject* Py_UNUSED(args)
+) {
+    turbopipe->sync();
+    Py_RETURN_NONE;
+}
+
+static PyObject* turbopipe_close(
+    PyObject* Py_UNUSED(self),
+    PyObject* Py_UNUSED(args)
+) {
+    turbopipe->close();
+    Py_RETURN_NONE;
+}
+
+// ------------------------------------------------------------------------------------------------|
+// Python module definition
+
+static void turbopipe_exit() {
+    turbopipe->close();
+}
+
+static PyMethodDef TurboPipeMethods[] = {
+    {"pipe",  (PyCFunction) turbopipe_pipe,  METH_VARARGS, ""},
+    {"sync",  (PyCFunction) turbopipe_sync,  METH_NOARGS,  ""},
+    {"close", (PyCFunction) turbopipe_close, METH_NOARGS,  ""},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef turbopipe_module = {
+    PyModuleDef_HEAD_INIT,
+    "_turbopipe",
+    NULL, -1,
+    TurboPipeMethods,
+    NULL, NULL, NULL, NULL
+};
+
+PyMODINIT_FUNC PyInit__turbopipe(void) {
+    PyObject* module = PyModule_Create(&turbopipe_module);
+    PyObject* moderngl = PyImport_ImportModule("moderngl");
+    PyObject* buffer = PyObject_GetAttrString(moderngl, "Buffer");
+    MGLBuffer_type = (PyTypeObject*) buffer;
+    turbopipe = new TurboPipe();
+    Py_AtExit(turbopipe_exit);
+    return module;
+}