turbopipe 1.1.0__tar.gz → 1.2.1__tar.gz

{turbopipe-1.1.0 → turbopipe-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: turbopipe
-Version: 1.1.0
+Version: 1.2.1
 Summary: 🌀 Faster MemoryView inter-process data transfers for subprocesses
 Home-page: https://brokensrc.dev
 Author-Email: Tremeschin <29046864+Tremeschin@users.noreply.github.com>
@@ -100,7 +100,7 @@ import turbopipe
 
 # Create ModernGL objects
 ctx = moderngl.create_standalone_context()
-buffer = ctx.buffer(reserve=1920*1080*3)
+buffers = [ctx.buffer(reserve=1920*1080*3) for _ in range(2)]
 
 # Make sure resolution, pixel format matches!
 ffmpeg = subprocess.Popen(
@@ -109,7 +109,10 @@ ffmpeg = subprocess.Popen(
 )
 
 # Rendering loop of yours (eg. 1m footage)
-for _ in range(60 * 60):
+for frame in range(60 * 60):
+    buffer = buffers[frame % len(buffer)]
+    turbopipe.sync(buffer)
+    fbo.read_into(buffer)
     turbopipe.pipe(buffer, ffmpeg.stdin.fileno())
 
 # Finalize writing, encoding

{turbopipe-1.1.0 → turbopipe-1.2.1}/Readme.md

@@ -65,7 +65,7 @@ import turbopipe
 
 # Create ModernGL objects
 ctx = moderngl.create_standalone_context()
-buffer = ctx.buffer(reserve=1920*1080*3)
+buffers = [ctx.buffer(reserve=1920*1080*3) for _ in range(2)]
 
 # Make sure resolution, pixel format matches!
 ffmpeg = subprocess.Popen(
@@ -74,7 +74,10 @@ ffmpeg = subprocess.Popen(
 )
 
 # Rendering loop of yours (eg. 1m footage)
-for _ in range(60 * 60):
+for frame in range(60 * 60):
+    buffer = buffers[frame % len(buffer)]
+    turbopipe.sync(buffer)
+    fbo.read_into(buffer)
     turbopipe.pipe(buffer, ffmpeg.stdin.fileno())
 
 # Finalize writing, encoding

{turbopipe-1.1.0 → turbopipe-1.2.1}/pyproject.toml

@@ -15,15 +15,8 @@ dependencies = ["moderngl"]
 requires-python = ">=3.7"
 
 [build-system]
-requires = ["meson-python", "ninja", "hatch-fancy-pypi-readme"]
+requires = ["meson-python", "ninja"]
 build-backend = "mesonpy"
 
-[tool.hatch.metadata.hooks.fancy-pypi-readme]
-content-type = "text/markdown"
-
-[[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]]
-start-after = "<!-- PyPI -->"
-path = "Readme.md"
-
 [tool.ruff.format]
 exclude = ["*"]

{turbopipe-1.1.0 → turbopipe-1.2.1}/turbopipe/__init__.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Optional, Union
 
 from moderngl import Buffer
 
@@ -7,8 +7,8 @@ from turbopipe import _turbopipe
 
 def pipe(buffer: Union[Buffer, memoryview], fileno: int) -> None:
     """
-    Pipe the content of a moderngl.Buffer or memoryview to a file descriptor,
-    Fast, threaded and non-blocking. Call `sync()` when done!
+    Pipe the content of a moderngl.Buffer or memoryview to a file descriptor, fast, threaded and
+    blocking when needed. Call `sync(buffer)` before this, and `sync()` when done for
 
     Usage:
         ```python
@@ -29,9 +29,12 @@ def pipe(buffer: Union[Buffer, memoryview], fileno: int) -> None:
     _turbopipe.pipe(buffer, fileno)
     del buffer
 
-def sync() -> None:
-    """Waits for all jobs to finish"""
-    _turbopipe.sync()
+def sync(buffer: Optional[Union[Buffer, memoryview]]=None) -> None:
+    """Waits for any pending write operation on a buffer, or 'all buffers' if None, to finish"""
+    if isinstance(buffer, Buffer):
+        buffer = memoryview(buffer.mglo)
+    _turbopipe.sync(buffer)
+    del buffer
 
 def close() -> None:
     """Syncs and deletes objects"""

{turbopipe-1.1.0 → turbopipe-1.2.1}/turbopipe/_turbopipe.cpp

@@ -23,20 +23,15 @@
 #include <unordered_map>
 #include <deque>
 
-#define dict std::unordered_map
 using namespace std;
 
 // ------------------------------------------------------------------------------------------------|
 // TurboPipe internals
 
 struct Work {
-    void* map;
+    void* data;
     int file;
     size_t size;
-
-    int hash() {
-        return std::hash<int>()(file) ^ std::hash<void*>()(map);
-    }
 };
 
 class TurboPipe {
@@ -49,12 +44,29 @@ public:
         this->_pipe(view.buf, view.len, file);
     }
 
-    void sync() {
-        // Wait for all queues to be empty, as they are erased when
+    void sync(PyObject* memoryview=nullptr) {
+        void* data = nullptr;
+
+        if (memoryview != nullptr) {
+            Py_buffer view = *PyMemoryView_GET_BUFFER(memoryview);
+            data = view.buf;
+        }
+
+        // Wait for some or all queues to be empty, as they are erased when
         // each thread's writing loop is done, guaranteeing finish
         for (auto& values: queue) {
-            while (!values.second.empty()) {
-                this_thread::sleep_for(chrono::milliseconds(1));
+            while (true) {
+                {
+                    // Prevent segfault on iteration on changing data
+                    lock_guard<mutex> lock(mutexes[values.first]);
+
+                    // Either all empty or some memory not queued (None or specific)
+                    if (data != nullptr && values.second.find(data) == values.second.end())
+                        break;
+                    if (data == nullptr && values.second.empty())
+                        break;
+                }
+                this_thread::sleep_for(chrono::microseconds(200));
             }
         }
     }
@@ -69,32 +81,32 @@ public:
     }
 
 private:
-    dict<int, dict<int, condition_variable>> pending;
-    dict<int, unordered_set<int>> queue;
-    dict<int, deque<Work>> stream;
-    dict<int, thread> threads;
-    dict<int, mutex> mutexes;
+    unordered_map<int, unordered_map<void*, condition_variable>> pending;
+    unordered_map<int, unordered_set<void*>> queue;
+    unordered_map<int, deque<Work>> stream;
+    unordered_map<int, thread> threads;
+    unordered_map<int, mutex> mutexes;
     condition_variable signal;
     bool running;
 
     void _pipe(void* data, size_t size, int file) {
         Work work = {data, file, size};
-        int hash = work.hash();
-
         unique_lock<mutex> lock(mutexes[file]);
 
-        // Notify this hash is queued, wait if pending
-        if (!queue[file].insert(hash).second) {
-            pending[file][hash].wait(lock, [this, file, hash] {
-                return queue[file].find(hash) == queue[file].end();
-            });
+        /* Notify this memory is queued, wait if pending */ {
+            if (!queue[file].insert(data).second) {
+                pending[file][data].wait(lock, [this, file, data] {
+                    return queue[file].find(data) == queue[file].end();
+                });
+            }
         }
 
-        // Add another job to the queue
-        stream[file].push_back(work);
-        queue[file].insert(hash);
-        this->running = true;
-        lock.unlock();
+        /* Add another job to the queue */ {
+            stream[file].push_back(work);
+            queue[file].insert(data);
+            this->running = true;
+            lock.unlock();
+        }
 
         // Each file descriptor has its own thread
         if (threads.find(file) == threads.end())
@@ -122,24 +134,25 @@ private:
 
             #ifdef _WIN32
                 // Windows doesn't like chunked writes ??
-                write(work.file, (char*) work.map, work.size);
+                write(work.file, (char*) work.data, work.size);
             #else
                 // Optimization: Write in chunks of 4096 (RAM page size)
                 size_t tell = 0;
                 while (tell < work.size) {
                     size_t chunk = min(work.size - tell, static_cast<size_t>(4096));
-                    size_t written = write(work.file, (char*) work.map + tell, chunk);
+                    size_t written = write(work.file, (char*) work.data + tell, chunk);
                     if (written == -1) break;
                     tell += written;
                 }
             #endif
 
-            // Signal work is done
             lock.lock();
-            int hash = work.hash();
-            pending[file][hash].notify_all();
-            queue[file].erase(hash);
-            signal.notify_all();
+
+            /* Signal work is done */ {
+                pending[file][work.data].notify_all();
+                queue[file].erase(work.data);
+                signal.notify_all();
+            }
         }
     }
 };
@@ -168,9 +181,16 @@ static PyObject* turbopipe_pipe(
 
 static PyObject* turbopipe_sync(
     PyObject* Py_UNUSED(self),
-    PyObject* Py_UNUSED(args)
+    PyObject* args
 ) {
-    turbopipe->sync();
+    PyObject* memoryview;
+    if (!PyArg_ParseTuple(args, "|O", &memoryview))
+        return NULL;
+    if (memoryview != nullptr && !PyMemoryView_Check(memoryview)) {
+        PyErr_SetString(PyExc_TypeError, "Expected a memoryview object or None");
+        return NULL;
+    }
+    turbopipe->sync(memoryview);
     Py_RETURN_NONE;
 }
 
@@ -191,7 +211,7 @@ static void turbopipe_exit() {
 
 static PyMethodDef TurboPipeMethods[] = {
     {"pipe",  (PyCFunction) turbopipe_pipe,  METH_VARARGS, ""},
-    {"sync",  (PyCFunction) turbopipe_sync,  METH_NOARGS,  ""},
+    {"sync",  (PyCFunction) turbopipe_sync,  METH_VARARGS, ""},
     {"close", (PyCFunction) turbopipe_close, METH_NOARGS,  ""},
     {NULL, NULL, 0, NULL}
 };

{turbopipe-1.1.0 → turbopipe-1.2.1}/turbopipe/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-__version__ = "1.1.0"
+__version__ = "1.2.1"
 
 if __name__ == "__main__":
     print(__version__)