py-sharedmemory 1.0.1__tar.gz → 2.0.0__tar.gz

{py_sharedmemory-1.0.1 → py_sharedmemory-2.0.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 товіаѕ
+Copyright (c) 2025 товіаѕ
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{py_sharedmemory-1.0.1/py_sharedmemory.egg-info → py_sharedmemory-2.0.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: py-sharedmemory
-Version: 1.0.1
-Summary: A replacement for Python multiprocessing.Queue that moves data objects into shared memory as raw bytes and sends only minimal metadata between processes — reducing I/O bottlenecks and improving throughput for larger data objects (>1MB).
+Version: 2.0.0
+Summary: Optimized Python IPC: Uses shared memory to bypass multiprocessing queue I/O bottlenecks, ideal for large data (1MB+) in scientific computing, RL, etc. Reduces system load and improves latency consistency
 Author: Tobias Würth
 License-Expression: MIT
 Project-URL: Repository, https://github.com/tobiaswuerth/python_shared_memory_queue
@@ -17,15 +17,17 @@ Dynamic: license-file
 
 # py-sharedmemory: High-Performance Multiprocessing Queue Alternative for Large Data
 
-Python's standard `multiprocessing.Queue` pickles data for inter-process communication. This often creates performance bottlenecks in demanding applications like Deep Reinforcement Learning (DeepRL) training, scientific computing, or distributed systems that handle large data objects (e.g., NumPy arrays, tensors >1MB) or involve high I/O volumes between many processes (like actors, replay buffers, and trainers).
-
-`py-sharedmemory` offers a high-performance replacement. It bypasses pickling by converting data into bytes and moving them into shared memory. Only minimal metadata is sent via the standard queue mechanism. This significantly reduces serialization overhead and inter-process I/O, drastically improving data transfer throughput for data-intensive Python multiprocessing tasks. If your application struggles with slow `multiprocessing.Queue` performance due to large object transfers, `py-sharedmemory` provides a faster alternative.
+[![PyPI version](https://badge.fury.io/py/py-sharedmemory.svg)](https://badge.fury.io/py/py-sharedmemory)
 
 Install using:
 ```
 pip install py-sharedmemory
 ```
 
+Python's standard `multiprocessing.Queue` pickles data for inter-process communication. This often creates performance bottlenecks in demanding applications like Deep Reinforcement Learning (DeepRL) training, scientific computing, or distributed systems that handle large data objects (e.g., NumPy arrays, tensors >1MB) or involve high I/O volumes between many processes (like actors, replay buffers, and trainers).
+
+`py-sharedmemory` offers a high-performance replacement. It bypasses pickling by converting data into bytes and moving them into shared memory. Only minimal metadata is sent via the standard queue mechanism. This significantly reduces serialization overhead and inter-process I/O, drastically improving data transfer throughput for data-intensive Python multiprocessing tasks. If your application struggles with slow `multiprocessing.Queue` performance due to large object transfers, `py-sharedmemory` provides a faster alternative.
+
 ---
 
 # Usage example

{py_sharedmemory-1.0.1 → py_sharedmemory-2.0.0}/README.md

@@ -1,14 +1,16 @@
 # py-sharedmemory: High-Performance Multiprocessing Queue Alternative for Large Data
 
-Python's standard `multiprocessing.Queue` pickles data for inter-process communication. This often creates performance bottlenecks in demanding applications like Deep Reinforcement Learning (DeepRL) training, scientific computing, or distributed systems that handle large data objects (e.g., NumPy arrays, tensors >1MB) or involve high I/O volumes between many processes (like actors, replay buffers, and trainers).
-
-`py-sharedmemory` offers a high-performance replacement. It bypasses pickling by converting data into bytes and moving them into shared memory. Only minimal metadata is sent via the standard queue mechanism. This significantly reduces serialization overhead and inter-process I/O, drastically improving data transfer throughput for data-intensive Python multiprocessing tasks. If your application struggles with slow `multiprocessing.Queue` performance due to large object transfers, `py-sharedmemory` provides a faster alternative.
+[![PyPI version](https://badge.fury.io/py/py-sharedmemory.svg)](https://badge.fury.io/py/py-sharedmemory)
 
 Install using:
 ```
 pip install py-sharedmemory
 ```
 
+Python's standard `multiprocessing.Queue` pickles data for inter-process communication. This often creates performance bottlenecks in demanding applications like Deep Reinforcement Learning (DeepRL) training, scientific computing, or distributed systems that handle large data objects (e.g., NumPy arrays, tensors >1MB) or involve high I/O volumes between many processes (like actors, replay buffers, and trainers).
+
+`py-sharedmemory` offers a high-performance replacement. It bypasses pickling by converting data into bytes and moving them into shared memory. Only minimal metadata is sent via the standard queue mechanism. This significantly reduces serialization overhead and inter-process I/O, drastically improving data transfer throughput for data-intensive Python multiprocessing tasks. If your application struggles with slow `multiprocessing.Queue` performance due to large object transfers, `py-sharedmemory` provides a faster alternative.
+
 ---
 
 # Usage example

py_sharedmemory-2.0.0/memory/__init__.py

@@ -0,0 +1,3 @@
+from .factory import create_shared_memory_pair
+from .sender import SharedMemorySender
+from .receiver import SharedMemoryReceiver

py_sharedmemory-2.0.0/memory/convert.py

@@ -0,0 +1,53 @@
+from typing import NamedTuple
+from multiprocessing.shared_memory import SharedMemory
+import pickle
+from pickle import PickleBuffer
+
+
+class SMInfo(NamedTuple):
+    smh_name: str
+    total_bytes: int
+    buffer_lengths: list[int]
+
+
+def data_to_smh(data: any) -> tuple[SharedMemory, SMInfo]:
+    buffers: list[PickleBuffer] = []
+    buffer_lengths: list[int] = []
+
+    def _append_buffer(buffer: PickleBuffer):
+        memview = buffer.raw()
+        buffers.append(memview)
+        buffer_lengths.append(len(memview))
+
+    main = pickle.dumps(data, pickle.HIGHEST_PROTOCOL, buffer_callback=_append_buffer)
+
+    len_main = len(main)
+    total_bytes = len_main + sum(buffer_lengths)
+    buffer_lengths.insert(0, len_main)
+    buffers.insert(0, main)
+
+    shm: SharedMemory = SharedMemory(create=True, size=total_bytes)
+    offset = 0
+    for length, buffer in zip(buffer_lengths, buffers):
+        shm.buf[offset : offset + length] = buffer
+        offset += length
+
+    info: SMInfo = SMInfo(shm.name, total_bytes, buffer_lengths)
+    return shm, info
+
+
+def data_from_smh(info: SMInfo) -> any:
+    shm: SharedMemory = SharedMemory(name=info.smh_name, size=info.total_bytes)
+    local_buffers: bytes = bytes(shm.buf[: info.total_bytes])
+    shm.close()
+
+    # unpack data
+    buffers = []
+    offset = 0
+    for length in info.buffer_lengths:
+        buffer = local_buffers[offset : offset + length]
+        buffers.append(buffer)
+        offset += length
+
+    main = buffers.pop(0)
+    return pickle.loads(main, buffers=buffers)

py_sharedmemory-2.0.0/memory/factory.py

@@ -0,0 +1,13 @@
+import multiprocessing as mp
+
+from .receiver import SharedMemoryReceiver
+from .sender import SharedMemorySender
+
+
+def create_shared_memory_pair(capacity):
+    data_queue = mp.Queue()
+    ack_queue = mp.Queue()
+
+    sender = SharedMemorySender(capacity, data_queue, ack_queue)
+    receiver = SharedMemoryReceiver(data_queue, ack_queue)
+    return sender, receiver

py_sharedmemory-2.0.0/memory/receiver.py

@@ -0,0 +1,23 @@
+from multiprocessing.queues import Queue
+
+from .convert import SMInfo, data_from_smh
+
+
+class SharedMemoryReceiver:
+    def __init__(self, q_data_in: Queue, q_ack_out: Queue):
+        self.q_data_in: Queue = q_data_in
+        self.q_ack_out: Queue = q_ack_out
+
+    def _process_info(self, info: SMInfo):
+        assert info, "No info received"
+        data = data_from_smh(info)
+        self.q_ack_out.put(info.smh_name)
+        del info
+        return data
+
+    def get_nowait(self):
+        return self.get(block=False)
+
+    def get(self, block: bool = True, timeout: float = None):
+        info: SMInfo = self.q_data_in.get(block, timeout)
+        return self._process_info(info)

py_sharedmemory-2.0.0/memory/sender.py

@@ -0,0 +1,111 @@
+import multiprocessing as mp
+import threading as th
+import atexit
+
+from .convert import data_to_smh
+
+
+class SharedMemorySender:
+    def __init__(self, capacity: int, q_data_out: mp.Queue, q_ack_in: mp.Queue):
+        self.q_data_out: mp.Queue = q_data_out
+        self.q_ack_in: mp.Queue = q_ack_in
+        self.capacity: int = capacity
+
+        self.is_closed = False
+        self.is_initialized = False
+
+        self.open_handles = {}
+        self.has_capacity = None
+        self.is_empty = None
+        self.thread_ack_running = True
+        self.thread_ack = None
+
+    def _initialize(self):
+        if self.is_initialized:
+            return
+
+        self.is_initialized = True
+        atexit.register(self._cleanup)
+
+        self.has_capacity = th.Semaphore(self.capacity or 0)
+        self.is_empty = th.Event()
+        self.is_empty.set()
+
+        self.thread_ack_running = True
+        self.thread_ack = th.Thread(target=self._handle_acks, daemon=True)
+        self.thread_ack.start()
+
+    def _cleanup(self):
+        if self.is_closed is not None:
+            self.is_closed = True
+
+        if self.thread_ack_running is not None:
+            self.thread_ack_running = False
+        if self.thread_ack is not None and self.thread_ack.is_alive():
+            self.thread_ack.join(timeout=1.0)
+
+        if self.open_handles is not None:
+            for shm_name in list(self.open_handles.keys()):
+                try:
+                    self._close_handle(shm_name)
+                except:
+                    pass
+            self.open_handles = None
+
+    def __del__(self):
+        self._cleanup()
+
+    def _close_handle(self, shm_name: str):
+        smh = self.open_handles.pop(shm_name, None)
+        if smh is None:
+            return
+        if len(self.open_handles) == 0:
+            self.is_empty.set()
+
+        smh.close()
+        smh.unlink()
+        if self.capacity:
+            self.has_capacity.release()
+
+    def _handle_acks(self):
+        while self.thread_ack_running:
+            try:
+                ack_smh_name = self.q_ack_in.get(timeout=0.1)
+                self._close_handle(ack_smh_name)
+            except mp.queues.Empty:
+                continue
+            except Exception as e:
+                if not self.is_closed:
+                    print(f"SharedMemorySender._handle_acks error: {e}")
+                    self._cleanup()
+                raise e
+
+    def put_nowait(self, data):
+        return self.put(data, block=False)
+
+    def put(self, data, block: bool = True, timeout: float = None):
+        if self.is_closed:
+            raise BrokenPipeError("Sender is closed.")
+        self._initialize()
+
+        if self.capacity:
+            if not self.has_capacity.acquire(blocking=block, timeout=timeout):
+                raise mp.queues.Full
+
+        try:
+            smh, info = data_to_smh(data)
+            self.is_empty.clear()
+            self.q_data_out.put_nowait(info)
+            self.open_handles[smh.name] = smh
+        except Exception as e:
+            if not self.is_closed:
+                print(f"SharedMemorySender.put error: {e}")
+                self._cleanup()
+            raise e
+
+    def wait_for_all_ack(self):
+        if self.is_closed:
+            raise BrokenPipeError("Sender is closed.")
+        if not self.is_initialized:
+            return
+        self.is_empty.wait()

{py_sharedmemory-1.0.1 → py_sharedmemory-2.0.0/py_sharedmemory.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: py-sharedmemory
-Version: 1.0.1
-Summary: A replacement for Python multiprocessing.Queue that moves data objects into shared memory as raw bytes and sends only minimal metadata between processes — reducing I/O bottlenecks and improving throughput for larger data objects (>1MB).
+Version: 2.0.0
+Summary: Optimized Python IPC: Uses shared memory to bypass multiprocessing queue I/O bottlenecks, ideal for large data (1MB+) in scientific computing, RL, etc. Reduces system load and improves latency consistency
 Author: Tobias Würth
 License-Expression: MIT
 Project-URL: Repository, https://github.com/tobiaswuerth/python_shared_memory_queue
@@ -17,15 +17,17 @@ Dynamic: license-file
 
 # py-sharedmemory: High-Performance Multiprocessing Queue Alternative for Large Data
 
-Python's standard `multiprocessing.Queue` pickles data for inter-process communication. This often creates performance bottlenecks in demanding applications like Deep Reinforcement Learning (DeepRL) training, scientific computing, or distributed systems that handle large data objects (e.g., NumPy arrays, tensors >1MB) or involve high I/O volumes between many processes (like actors, replay buffers, and trainers).
-
-`py-sharedmemory` offers a high-performance replacement. It bypasses pickling by converting data into bytes and moving them into shared memory. Only minimal metadata is sent via the standard queue mechanism. This significantly reduces serialization overhead and inter-process I/O, drastically improving data transfer throughput for data-intensive Python multiprocessing tasks. If your application struggles with slow `multiprocessing.Queue` performance due to large object transfers, `py-sharedmemory` provides a faster alternative.
+[![PyPI version](https://badge.fury.io/py/py-sharedmemory.svg)](https://badge.fury.io/py/py-sharedmemory)
 
 Install using:
 ```
 pip install py-sharedmemory
 ```
 
+Python's standard `multiprocessing.Queue` pickles data for inter-process communication. This often creates performance bottlenecks in demanding applications like Deep Reinforcement Learning (DeepRL) training, scientific computing, or distributed systems that handle large data objects (e.g., NumPy arrays, tensors >1MB) or involve high I/O volumes between many processes (like actors, replay buffers, and trainers).
+
+`py-sharedmemory` offers a high-performance replacement. It bypasses pickling by converting data into bytes and moving them into shared memory. Only minimal metadata is sent via the standard queue mechanism. This significantly reduces serialization overhead and inter-process I/O, drastically improving data transfer throughput for data-intensive Python multiprocessing tasks. If your application struggles with slow `multiprocessing.Queue` performance due to large object transfers, `py-sharedmemory` provides a faster alternative.
+
 ---
 
 # Usage example

{py_sharedmemory-1.0.1 → py_sharedmemory-2.0.0}/py_sharedmemory.egg-info/SOURCES.txt

@@ -2,7 +2,7 @@ LICENSE
 README.md
 pyproject.toml
 memory/__init__.py
-memory/converter.py
+memory/convert.py
 memory/factory.py
 memory/receiver.py
 memory/sender.py
@@ -10,5 +10,4 @@ py_sharedmemory.egg-info/PKG-INFO
 py_sharedmemory.egg-info/SOURCES.txt
 py_sharedmemory.egg-info/dependency_links.txt
 py_sharedmemory.egg-info/requires.txt
-py_sharedmemory.egg-info/top_level.txt
-tests/test_shared_memory.py
+py_sharedmemory.egg-info/top_level.txt

{py_sharedmemory-1.0.1 → py_sharedmemory-2.0.0}/pyproject.toml

@@ -8,13 +8,13 @@ packages = ['memory']
 [project]
 name = "py-sharedmemory"
 authors = [{name = "Tobias Würth"}]
-version = "1.0.1"
+version = "2.0.0"
 requires-python = ">=3.9"
 dependencies = [
     "numpy>=1.26"
 ]
 
-description = "A replacement for Python multiprocessing.Queue that moves data objects into shared memory as raw bytes and sends only minimal metadata between processes — reducing I/O bottlenecks and improving throughput for larger data objects (>1MB)."
+description = "Optimized Python IPC: Uses shared memory to bypass multiprocessing queue I/O bottlenecks, ideal for large data (1MB+) in scientific computing, RL, etc. Reduces system load and improves latency consistency"
 keywords = ["python", "performance", "queue", "multiprocessing", "numpy", "zero-copy", "data-transfer", "low-latency", "inter-process-communication", "shared-memory", "high-throughput", "parallel-processing", "large-data", "memoryview"]
 readme = {file = "README.md", content-type = "text/markdown"}
 license = "MIT"

py_sharedmemory-1.0.1/memory/__init__.py

@@ -1 +0,0 @@
-from .factory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver

py_sharedmemory-1.0.1/memory/converter.py

@@ -1,106 +0,0 @@
-import numpy as np
-
-def _encode_int(item:int):
-    size = item.bit_length() // 8 + 1
-    signed = item < 0
-    bytes_data = item.to_bytes(size, 'little', signed=signed)
-    return ('i', size, signed), memoryview(bytes_data)
-
-def _decode_int(byte_data:memoryview, info:tuple):
-    return int.from_bytes(byte_data, 'little', signed=info[2])
-
-def _encode_float(item:float):
-    byte_data = np.float64(item).tobytes()
-    return ('f', 8), memoryview(byte_data)
-
-def _decode_float(byte_data:memoryview, info:tuple):
-    return np.frombuffer(byte_data, dtype=np.float64)[0]
-
-def _encode_bool(item:bool):
-    byte_data = item.to_bytes(1, 'little')
-    return ('b', 1), memoryview(byte_data)
-
-def _decode_bool(byte_data:memoryview, info:tuple):
-    return bool.from_bytes(byte_data, 'little')
-
-def _encode_bytes(item:bytes):
-    return ('r', len(item)), memoryview(item)
-
-def _decode_bytes(byte_data:memoryview, info:tuple):
-    if byte_data is None:
-        return b''
-    return bytes(byte_data)
-
-def _encode_str(item:str):
-    bytes_data = str.encode(item)
-    return ('s', len(bytes_data)), memoryview(bytes_data)
-
-def _decode_str(byte_data:memoryview, info:tuple):
-    if byte_data is None:
-        return ''
-    return bytes(byte_data).decode()
-
-def _encode_ndarray(item:np.ndarray):
-    byte_data = item.tobytes()
-    return ('n', len(byte_data), item.dtype, item.shape), memoryview(byte_data)
-
-def _decode_ndarray(byte_data:memoryview, info:tuple):
-    return np.frombuffer(byte_data, dtype=info[2]).copy().reshape(info[3])
-
-def _encode_ndtype(item:np.dtype):
-    byte_data = str.encode(str(item))
-    return ('t', len(byte_data)), memoryview(byte_data)
-
-def _decode_ndtype(byte_data:memoryview, info:tuple):
-    return np.dtype(bytes(byte_data).decode())
-
-iterables = {
-    tuple: 'p',
-    list: 'l',
-    dict: 'd',
-    set: 'u',
-}
-
-encoder = {
-    bytes: _encode_bytes,
-    int: _encode_int,
-    float: _encode_float,
-    bool: _encode_bool,
-    str: _encode_str,
-    np.ndarray: _encode_ndarray,
-}
-decoder = {
-    'r': _decode_bytes,
-    'i': _decode_int,
-    'f': _decode_float,
-    'b': _decode_bool,
-    's': _decode_str,
-    'n': _decode_ndarray,
-}
-
-def encode(item):
-    if item is None:
-        return (None, 0), b''
-    
-    datatype = type(item)
-    enc = encoder.get(datatype, None)
-    if enc is not None:
-        return enc(item)
-    
-    if isinstance(item, np.dtype):
-        return _encode_ndtype(item)
-
-    raise ValueError(f"Unsupported data type: {type(item)}")
-
-def decode(bytes_data:memoryview, info:tuple):
-        if info[0] is None:
-            return None
-        
-        dec = decoder.get(info[0], None)
-        if dec is not None:
-            return dec(bytes_data, info)
-        
-        if info[0] == 't':
-            return _decode_ndtype(bytes_data, info)
-        
-        raise ValueError(f"Unsupported data type: {info[0]}")

py_sharedmemory-1.0.1/memory/factory.py

@@ -1,12 +0,0 @@
-import multiprocessing as mp
-
-from .receiver import SharedMemoryReceiver
-from .sender import SharedMemorySender
-
-
-def create_shared_memory_pair(capacity):
-    q_data = mp.Queue(capacity)
-    q_ack = mp.Queue()
-    sender = SharedMemorySender(capacity, q_data, q_ack)
-    receiver = SharedMemoryReceiver(q_data, q_ack)
-    return sender, receiver

py_sharedmemory-1.0.1/memory/receiver.py

@@ -1,75 +0,0 @@
-import multiprocessing as mp
-import queue
-from multiprocessing.shared_memory import SharedMemory
-
-from .converter import decode, iterables
-
-iterable_keys = set(iterables.values())
-non_dict_iterables = set(iterables.keys()) - {dict}
-
-class SharedMemoryReceiver:
-    def __init__(self, queue_data_in:mp.Queue, queue_ack_out:mp.Queue):
-        self.queue_data_in = queue_data_in
-        self.queue_ack_out = queue_ack_out
-
-    def _recover_data_recursive(self, shm:SharedMemory, info:tuple, byte_idx=0):
-        if info[0] not in iterable_keys:
-            if info[1] == 0: # size
-                return decode(None, info)
-            return decode(shm.buf[byte_idx:byte_idx+info[1]], info)
-
-        if info[0] == 'd': # dict
-            data = {}
-            for info_child, key in zip(info[3], info[4]):
-                item = self._recover_data_recursive(shm, info_child, byte_idx)
-                byte_idx += info_child[1]
-                data[key] = item
-            return data
-
-        # tuple, list, set
-        data = []
-        for info_child in info[3]:
-            item = self._recover_data_recursive(shm, info_child, byte_idx)
-            byte_idx += info_child[1]
-            data.append(item)
-        
-        if info[2] in non_dict_iterables:
-            return info[2](data)
-        
-        # NamedTuple needs to be handled separately
-        return info[2](*data)
-
-    def _recover_data(self, info:tuple):
-        if info[2] == 0:
-            return self._recover_data_recursive(None, info[1:])
-
-        name = info[0]
-        info = info[1:]
-        shm = SharedMemory(name=name, size=info[1])
-        data = self._recover_data_recursive(shm, info)
-        shm.close()
-        return data
-
-    def get_nowait(self):
-        if self.queue_data_in.empty():
-            return None
-        
-        try:
-            info = self.queue_data_in.get_nowait()
-        except queue.Empty:
-            return None
-        
-        if info is None:
-            return None
-        
-        item = self._recover_data(info)
-        self.queue_ack_out.put_nowait(info[0])
-        del info
-        return item
-
-    def get(self, timeout=None):
-        info = self.queue_data_in.get(timeout=timeout)
-        item = self._recover_data(info)
-        self.queue_ack_out.put_nowait(info[0])
-        del info
-        return item

py_sharedmemory-1.0.1/memory/sender.py

@@ -1,149 +0,0 @@
-import multiprocessing as mp
-import queue
-from multiprocessing.shared_memory import SharedMemory
-
-from .converter import encode, iterables
-
-class _Separator:
-    def __init__(self, obj):
-        self.obj = obj
-
-def _iterable_handler_list(stack, parents, entry):
-    stack.append(_Separator((type(entry), entry.__class__,)))
-    parents.append(([], []))
-    for field in reversed(entry): # reverse to maintain order in final binary
-        stack.append(field)
-
-def _iterable_handler_set(stack, parents, entry):
-    stack.append(_Separator((set,entry.__class__,)))
-    parents.append(([], []))
-    for field in entry:
-        stack.append(field)
-
-def _iterable_handler_dict(stack, parents, entry):
-    keys = list(entry.keys())
-    stack.append(_Separator((dict,entry.__class__,keys)))
-    parents.append(([], []))
-    for key in reversed(keys): # reverse to maintain order in final binary
-        stack.append(entry[key])
-
-iterable_handlers = {
-    list: _iterable_handler_list,
-    tuple: _iterable_handler_list,
-    set: _iterable_handler_set,
-    dict: _iterable_handler_dict,
-}
-
-tuple_key = iterables[tuple]
-
-class SharedMemorySender:
-    def __init__(self, capacity, queue_data_out:mp.Queue, queue_ack_in:mp.Queue):
-        self.open_handles = {}
-        self.queue_data_out = queue_data_out
-        self.queue_ack_in = queue_ack_in
-        self.capacity = capacity
-
-    def _prepare_binary(self, data):
-        parents:list[tuple[list[memoryview], list[tuple]]] = []
-        stack = [data]
-        while stack:
-            entry = stack.pop()
-            
-            if isinstance(entry, _Separator):
-                # children processed, gather the results
-                bytes_data, child_infos = parents.pop()
-                entry_type = entry.obj[0]
-                key = iterables.get(entry_type, tuple_key)
-                size = sum(info[1] for info in child_infos)
-                clazz = entry.obj[1]
-                if entry_type == dict:
-                    info = (key, size, clazz, child_infos, entry.obj[2])
-                else:   
-                    info = (key, size, clazz, child_infos)
-
-                if not parents:
-                    # root entry was tuple, return the result
-                    del parents, stack
-                    return info, bytes_data
-
-                # append to parent binary
-                parents[-1][0].extend(bytes_data)
-                parents[-1][1].append(info)
-                continue
-
-            entry_type = type(entry)
-            if entry_type in iterables:
-                iterable_handlers[entry_type](stack, parents, entry)
-                continue
-            if isinstance(entry, tuple):
-                # NamedTuple
-                _iterable_handler_list(stack, parents, entry)
-                continue
-
-            info, bytes_data = encode(entry)
-            if not parents:
-                # root entry was single data, return the result
-                del parents, stack
-                return info, [bytes_data]
-
-            parents[-1][0].append(bytes_data)
-            parents[-1][1].append(info)
-
-    def put(self, data):
-        self.process_ack()
-        while self.capacity and len(self.open_handles) >= self.capacity:
-            self.wait_for_ack()
-
-        info, bytes_datas = self._prepare_binary(data)
-        size = info[1]
-        if size == 0:
-            info = (None, *info)
-            self.queue_data_out.put(info)
-            return
-
-        shm = SharedMemory(create=True, size=size)
-        info = (shm.name, *info)
-
-        offset = 0
-        for item in bytes_datas:
-            size = len(item)
-            shm.buf[offset:offset + size] = item
-            offset += size
-
-        self.queue_data_out.put(info)
-        self.open_handles[shm.name] = shm
-
-    def close_handle(self, shm_name:str):
-        shm = self.open_handles[shm_name]
-        shm.close()
-        shm.unlink()
-        del self.open_handles[shm_name]
-
-    def has_space(self):
-        self.process_ack()
-        return not self.capacity or len(self.open_handles) < self.capacity
-
-    def process_ack(self):
-        try:
-            while True:
-                retour = self.queue_ack_in.get_nowait()
-                if retour is None:
-                    return
-                self.close_handle(retour)
-        except queue.Empty:
-            pass
-
-    def close_all(self):
-        for shm_name in list(self.open_handles.keys()):
-            self.close_handle(shm_name)
-
-    def wait_for_ack(self):
-        shm_name = self.queue_ack_in.get()
-        self.close_handle(shm_name)
-
-    def wait_for_all_ack(self):
-        while len(self.open_handles) > 0:
-            self.wait_for_ack()
-
-    def __del__(self):
-        self.close_all()

py_sharedmemory-1.0.1/tests/test_shared_memory.py

@@ -1,339 +0,0 @@
-import unittest
-import numpy as np
-
-import multiprocessing as mp
-mp.log_to_stderr()
-
-from typing import NamedTuple
-
-from memory import create_shared_memory_pair
-
-class MyTuple(NamedTuple):
-    a:int
-    b:float
-    c:str
-
-class TestSharedMemory(unittest.TestCase):
-    def test_None(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = None
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_None_tuple(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = (None,None)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_None_list(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [None,None]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_None_dict(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {None:None}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_None_set(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {None}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_int_positive_small(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = 37
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_int_negative_small(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = -37
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_int_positive_big(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = 2147483640
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_int_negative_big(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = -2147483640
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_float1(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = 21474.83640
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_float2(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = -21474.83640
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_bool1(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = True
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_bool2(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = True
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_bytes(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = b"Hello World!"
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_bytes_empty(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = b''
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_str(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = "Hello World!"
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_str_empty(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = ""
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_str_long(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = """Zarathustra went down the mountain alone, no one meeting him. When he entered the forest, however, there suddenly stood before him an old man, who had left his holy cot to seek roots. And thus spake the old man to Zarathustra:
-“No stranger to me is this wanderer: many years ago passed he by. Zarathustra he was called; but he hath altered.
-Then thou carriedst thine ashes into the mountains: wilt thou now carry thy fire into the valleys? Fearest thou not the incendiary’s doom?
-Yea, I recognise Zarathustra. Pure is his eye, and no loathing lurketh about his mouth. Goeth he not along like a dancer?
-Altered is Zarathustra; a child hath Zarathustra become; an awakened one is Zarathustra: what wilt thou do in the land of the sleepers?
-As in the sea hast thou lived in solitude, and it hath borne thee up. Alas, wilt thou now go ashore? Alas, wilt thou again drag thy body thyself?”
-Zarathustra answered: “I love mankind.”
-“Why,” said the saint, “did I go into the forest and the desert? Was it not because I loved men far too well?
-Now I love God: men, I do not love. Man is a thing too imperfect for me. Love to man would be fatal to me.”
-Zarathustra answered: “What spake I of love! I am bringing gifts unto men.”
-“Give them nothing,” said the saint. “Take rather part of their load, and carry it along with them—that will be most agreeable unto them: if only it be agreeable unto thee!
-If, however, thou wilt give unto them, give them no more than an alms, and let them also beg for it!”
-“No,” replied Zarathustra, “I give no alms. I am not poor enough for that.”
-The saint laughed at Zarathustra, and spake thus: “Then see to it that they accept thy treasures! They are distrustful of anchorites, and do not believe that we come with gifts.
-The fall of our footsteps ringeth too hollow through their streets. And just as at night, when they are in bed and hear a man abroad long before sunrise, so they ask themselves concerning us: Where goeth the thief?
-Go not to men, but stay in the forest! Go rather to the animals! Why not be like me—a bear amongst bears, a bird amongst birds?”
-“And what doeth the saint in the forest?” asked Zarathustra.
-The saint answered: “I make hymns and sing them; and in making hymns I laugh and weep and mumble: thus do I praise God.
-With singing, weeping, laughing, and mumbling do I praise the God who is my God. But what dost thou bring us as a gift?”
-When Zarathustra had heard these words, he bowed to the saint and said: “What should I have to give thee! Let me rather hurry hence lest I take aught away from thee!”—And thus they parted from one another, the old man and Zarathustra, laughing like schoolboys.
-When Zarathustra was alone, however, he said to his heart: “Could it be possible! This old saint in the forest hath not yet heard of it, that God is dead!”"""
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_ndarray1(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = np.array([1,2,3,4,5], dtype=np.uint32)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        np.testing.assert_array_equal(item, data, f"Expected {data}, got {item}")
-    def test_ndarray2(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = np.array([[1.2,2.3,3.4],[-53.3,-3535,1]], dtype=np.float32)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        np.testing.assert_array_equal(item, data, f"Expected {data}, got {item}")
-    def test_ndarray_big(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = np.random.rand(64, 30, 6, 84, 84).astype(np.float32)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        np.testing.assert_array_equal(item, data, f"Expected {data}, got {item}")
-
-    def test_ndtype1(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = np.dtype(np.float32)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_ndtype2(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = np.dtype(np.bool_)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_list(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [1,2,3,4,5]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [[1,2,3],[4,5,6]]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_mixed_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [1,2,[3,4],5]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_nested_tuple(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [1,2,(3,4),5]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_mixed(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [1,'a',3.4,5, None, b'Hello World!']
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_empty(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = []
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_empty_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [[],[]]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_list_empty_nested_tuple(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = [(),()]
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_tuple(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = (1,2,3,4,5)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_tuple_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = ((1,2,3),(4,5,6))
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_tuple_mixed_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = (1,2,[3,4],5)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_tuple_nested_list(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = (1,2,[3,4],5)
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_tuple_mixed(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = (1,'a',3.4,5, None, b'Hello World!')
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_tuple_empty(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = ()
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_tuple_empty_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = ((),())
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    
-    def test_namedtuple(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = MyTuple(1, 2.3, 'Hello World!')
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-    def test_dict(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1: 'a', 'b': 2, 3: 4}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_dict_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1: {2: 3, 4: 5}, 'a': {'b': 'c', 'd': 'e'}}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_dict_mixed_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1: {2: 3, 4: 5}, 'a': ['b', 'c', 'd']}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_dict_mixed(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1: 'a', 'b': 2, 3: 4, 'c': None, 'd': b'Hello World!'}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_dict_empty(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_dict_empty_nested(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1: {}, 'a': {}}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    
-    def test_set(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1,2,3,4,5}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_set_mixed(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = {1,2,'a',None,4}
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-    def test_set_empty(self):
-        sender, receiver = create_shared_memory_pair(capacity=1)
-        data = set()
-        sender.put(data)
-        item = receiver.get(timeout=2)
-        self.assertEqual(item, data, f"Expected {data}, got {item}")
-
-if __name__ == '__main__':
-    unittest.main()