py-sharedmemory 1.0.0__py3-none-any.whl

memory/__init__.py

@@ -0,0 +1 @@
+from .factory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver

memory/converter.py

@@ -0,0 +1,106 @@
+import numpy as np
+
+def _encode_int(item:int):
+    size = item.bit_length() // 8 + 1
+    signed = item < 0
+    bytes_data = item.to_bytes(size, 'little', signed=signed)
+    return ('i', size, signed), memoryview(bytes_data)
+
+def _decode_int(byte_data:memoryview, info:tuple):
+    return int.from_bytes(byte_data, 'little', signed=info[2])
+
+def _encode_float(item:float):
+    byte_data = np.float64(item).tobytes()
+    return ('f', 8), memoryview(byte_data)
+
+def _decode_float(byte_data:memoryview, info:tuple):
+    return np.frombuffer(byte_data, dtype=np.float64)[0]
+
+def _encode_bool(item:bool):
+    byte_data = item.to_bytes(1, 'little')
+    return ('b', 1), memoryview(byte_data)
+
+def _decode_bool(byte_data:memoryview, info:tuple):
+    return bool.from_bytes(byte_data, 'little')
+
+def _encode_bytes(item:bytes):
+    return ('r', len(item)), memoryview(item)
+
+def _decode_bytes(byte_data:memoryview, info:tuple):
+    if byte_data is None:
+        return b''
+    return bytes(byte_data)
+
+def _encode_str(item:str):
+    bytes_data = str.encode(item)
+    return ('s', len(bytes_data)), memoryview(bytes_data)
+
+def _decode_str(byte_data:memoryview, info:tuple):
+    if byte_data is None:
+        return ''
+    return bytes(byte_data).decode()
+
+def _encode_ndarray(item:np.ndarray):
+    byte_data = item.tobytes()
+    return ('n', len(byte_data), item.dtype, item.shape), memoryview(byte_data)
+
+def _decode_ndarray(byte_data:memoryview, info:tuple):
+    return np.frombuffer(byte_data, dtype=info[2]).copy().reshape(info[3])
+
+def _encode_ndtype(item:np.dtype):
+    byte_data = str.encode(str(item))
+    return ('t', len(byte_data)), memoryview(byte_data)
+
+def _decode_ndtype(byte_data:memoryview, info:tuple):
+    return np.dtype(bytes(byte_data).decode())
+
+iterables = {
+    tuple: 'p',
+    list: 'l',
+    dict: 'd',
+    set: 'u',
+}
+
+encoder = {
+    bytes: _encode_bytes,
+    int: _encode_int,
+    float: _encode_float,
+    bool: _encode_bool,
+    str: _encode_str,
+    np.ndarray: _encode_ndarray,
+}
+decoder = {
+    'r': _decode_bytes,
+    'i': _decode_int,
+    'f': _decode_float,
+    'b': _decode_bool,
+    's': _decode_str,
+    'n': _decode_ndarray,
+}
+
+def encode(item):
+    if item is None:
+        return (None, 0), b''
+    
+    datatype = type(item)
+    enc = encoder.get(datatype, None)
+    if enc is not None:
+        return enc(item)
+    
+    if isinstance(item, np.dtype):
+        return _encode_ndtype(item)
+
+    raise ValueError(f"Unsupported data type: {type(item)}")
+
+def decode(bytes_data:memoryview, info:tuple):
+        if info[0] is None:
+            return None
+        
+        dec = decoder.get(info[0], None)
+        if dec is not None:
+            return dec(bytes_data, info)
+        
+        if info[0] == 't':
+            return _decode_ndtype(bytes_data, info)
+        
+        raise ValueError(f"Unsupported data type: {info[0]}")

memory/factory.py

@@ -0,0 +1,12 @@
+import multiprocessing as mp
+
+from .receiver import SharedMemoryReceiver
+from .sender import SharedMemorySender
+
+
+def create_shared_memory_pair(capacity):
+    q_data = mp.Queue(capacity)
+    q_ack = mp.Queue()
+    sender = SharedMemorySender(capacity, q_data, q_ack)
+    receiver = SharedMemoryReceiver(q_data, q_ack)
+    return sender, receiver

memory/receiver.py

@@ -0,0 +1,75 @@
+import multiprocessing as mp
+import queue
+from multiprocessing.shared_memory import SharedMemory
+
+from .converter import decode, iterables
+
+iterable_keys = set(iterables.values())
+non_dict_iterables = set(iterables.keys()) - {dict}
+
+class SharedMemoryReceiver:
+    def __init__(self, queue_data_in:mp.Queue, queue_ack_out:mp.Queue):
+        self.queue_data_in = queue_data_in
+        self.queue_ack_out = queue_ack_out
+
+    def _recover_data_recursive(self, shm:SharedMemory, info:tuple, byte_idx=0):
+        if info[0] not in iterable_keys:
+            if info[1] == 0: # size
+                return decode(None, info)
+            return decode(shm.buf[byte_idx:byte_idx+info[1]], info)
+
+        if info[0] == 'd': # dict
+            data = {}
+            for info_child, key in zip(info[3], info[4]):
+                item = self._recover_data_recursive(shm, info_child, byte_idx)
+                byte_idx += info_child[1]
+                data[key] = item
+            return data
+
+        # tuple, list, set
+        data = []
+        for info_child in info[3]:
+            item = self._recover_data_recursive(shm, info_child, byte_idx)
+            byte_idx += info_child[1]
+            data.append(item)
+        
+        if info[2] in non_dict_iterables:
+            return info[2](data)
+        
+        # NamedTuple needs to be handled separately
+        return info[2](*data)
+
+    def _recover_data(self, info:tuple):
+        if info[2] == 0:
+            return self._recover_data_recursive(None, info[1:])
+
+        name = info[0]
+        info = info[1:]
+        shm = SharedMemory(name=name, size=info[1])
+        data = self._recover_data_recursive(shm, info)
+        shm.close()
+        return data
+
+    def get_nowait(self):
+        if self.queue_data_in.empty():
+            return None
+        
+        try:
+            info = self.queue_data_in.get_nowait()
+        except queue.Empty:
+            return None
+        
+        if info is None:
+            return None
+        
+        item = self._recover_data(info)
+        self.queue_ack_out.put_nowait(info[0])
+        del info
+        return item
+
+    def get(self, timeout=None):
+        info = self.queue_data_in.get(timeout=timeout)
+        item = self._recover_data(info)
+        self.queue_ack_out.put_nowait(info[0])
+        del info
+        return item

memory/sender.py

@@ -0,0 +1,149 @@
+import multiprocessing as mp
+import queue
+from multiprocessing.shared_memory import SharedMemory
+
+from .converter import encode, iterables
+
+class _Separator:
+    def __init__(self, obj):
+        self.obj = obj
+
+def _iterable_handler_list(stack, parents, entry):
+    stack.append(_Separator((type(entry), entry.__class__,)))
+    parents.append(([], []))
+    for field in reversed(entry): # reverse to maintain order in final binary
+        stack.append(field)
+
+def _iterable_handler_set(stack, parents, entry):
+    stack.append(_Separator((set,entry.__class__,)))
+    parents.append(([], []))
+    for field in entry:
+        stack.append(field)
+
+def _iterable_handler_dict(stack, parents, entry):
+    keys = list(entry.keys())
+    stack.append(_Separator((dict,entry.__class__,keys)))
+    parents.append(([], []))
+    for key in reversed(keys): # reverse to maintain order in final binary
+        stack.append(entry[key])
+
+iterable_handlers = {
+    list: _iterable_handler_list,
+    tuple: _iterable_handler_list,
+    set: _iterable_handler_set,
+    dict: _iterable_handler_dict,
+}
+
+tuple_key = iterables[tuple]
+
+class SharedMemorySender:
+    def __init__(self, capacity, queue_data_out:mp.Queue, queue_ack_in:mp.Queue):
+        self.open_handles = {}
+        self.queue_data_out = queue_data_out
+        self.queue_ack_in = queue_ack_in
+        self.capacity = capacity
+
+    def _prepare_binary(self, data):
+        parents:list[tuple[list[memoryview], list[tuple]]] = []
+        stack = [data]
+        while stack:
+            entry = stack.pop()
+            
+            if isinstance(entry, _Separator):
+                # children processed, gather the results
+                bytes_data, child_infos = parents.pop()
+                entry_type = entry.obj[0]
+                key = iterables.get(entry_type, tuple_key)
+                size = sum(info[1] for info in child_infos)
+                clazz = entry.obj[1]
+                if entry_type == dict:
+                    info = (key, size, clazz, child_infos, entry.obj[2])
+                else:   
+                    info = (key, size, clazz, child_infos)
+
+                if not parents:
+                    # root entry was tuple, return the result
+                    del parents, stack
+                    return info, bytes_data
+
+                # append to parent binary
+                parents[-1][0].extend(bytes_data)
+                parents[-1][1].append(info)
+                continue
+
+            entry_type = type(entry)
+            if entry_type in iterables:
+                iterable_handlers[entry_type](stack, parents, entry)
+                continue
+            if isinstance(entry, tuple):
+                # NamedTuple
+                _iterable_handler_list(stack, parents, entry)
+                continue
+
+            info, bytes_data = encode(entry)
+            if not parents:
+                # root entry was single data, return the result
+                del parents, stack
+                return info, [bytes_data]
+
+            parents[-1][0].append(bytes_data)
+            parents[-1][1].append(info)
+
+    def put(self, data):
+        self.process_ack()
+        while len(self.open_handles) >= self.capacity:
+            self.wait_for_ack()
+
+        info, bytes_datas = self._prepare_binary(data)
+        size = info[1]
+        if size == 0:
+            info = (None, *info)
+            self.queue_data_out.put(info)
+            return
+
+        shm = SharedMemory(create=True, size=size)
+        info = (shm.name, *info)
+
+        offset = 0
+        for item in bytes_datas:
+            size = len(item)
+            shm.buf[offset:offset + size] = item
+            offset += size
+
+        self.queue_data_out.put(info)
+        self.open_handles[shm.name] = shm
+
+    def close_handle(self, shm_name:str):
+        shm = self.open_handles[shm_name]
+        shm.close()
+        shm.unlink()
+        del self.open_handles[shm_name]
+
+    def has_space(self):
+        self.process_ack()
+        return len(self.open_handles) < self.capacity
+
+    def process_ack(self):
+        try:
+            while True:
+                retour = self.queue_ack_in.get_nowait()
+                if retour is None:
+                    return
+                self.close_handle(retour)
+        except queue.Empty:
+            pass
+
+    def close_all(self):
+        for shm_name in list(self.open_handles.keys()):
+            self.close_handle(shm_name)
+
+    def wait_for_ack(self):
+        shm_name = self.queue_ack_in.get()
+        self.close_handle(shm_name)
+
+    def wait_for_all_ack(self):
+        while len(self.open_handles) > 0:
+            self.wait_for_ack()
+
+    def __del__(self):
+        self.close_all()

py_sharedmemory-1.0.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 товіаѕ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

py_sharedmemory-1.0.0.dist-info/METADATA

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: py-sharedmemory
+Version: 1.0.0
+Summary: Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+Author: Tobias Würth
+License: MIT License
+        
+        Copyright (c) 2024 товіаѕ
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Repository, https://github.com/tobiaswuerth/python_shared_memory_queue
+Project-URL: Bug Tracker, https://github.com/tobiaswuerth/python_shared_memory_queue/issues
+Keywords: python,python3,multiprocessing,sharedmemory,queue,pipe
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy >=1.26
+
+# Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+
+Since [multiprocessing](https://docs.python.org/3/library/multiprocessing.html) [Queue](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue)s pickles the data, there is a certain performance decrease for larger data objects and huge amounts of I/O.
+
+This is the reason why I created a substitute which instead converts data objects into bytes (if needed), puts the bytes into shared memory and only transfers meta-data about the data structure via Queues to the receiving end.
+
+The following data types are currently supported:
+```
+- bytes
+- int
+- float
+- bool
+- str
+- np.ndarray
+- np.dtype
+```
+
+As well as any nested structures for the given types:
+```
+- tuple/NamedTuple
+- list
+- dict
+- set
+```
+
+# Considerations
+There is a certain overhead to the conversion process which is especially noticable for smaller objects and more complex data structures.
+Use the following approach depending on the size of the data you're handling:
+
+||10B|100B|1KB|10KB|100KB|1MB|10MB|100MB|1GB|10GB|
+|---|---|---|---|---|---|---|---|---|---|---|
+|``mp.Queue()``|✅|✅|✅|✅|✅|❌|❌|❌|❌|❌|
+|``SharedMemory``|❌|❌|❌|❌|❌|✅|✅|✅|✅|✅|
+
+Actual performance heavily relies on your memory speed.
+
+# Usage example
+```python
+import multiprocessing as mp
+
+from memory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver
+
+def producer_sm(sender:SharedMemorySender):
+    your_data = "your data"
+
+    # ...
+    sender.put(your_data) # blocks until there is space
+
+    # or
+    if sender.has_space():
+        sender.put(your_data)
+    else:
+        # your else case
+    
+    # ...
+    sender.wait_for_all_ack() # wait for all data to be received before closing the sender
+    # exit
+
+def consumer_sm(receiver:SharedMemoryReceiver, loops, size):
+    # ...
+    data = receiver.get() # blocks
+    
+    # or
+    data = receiver.get(timeout=3) # raises queue.Empty exception after 3s
+
+    # or
+    data = receiver.get_nowait()
+    if data is None:
+        # handle empty case
+
+    # ...
+
+if __name__ == '__main__':
+    sender, receiver = create_shared_memory_pair(capacity=5)
+
+    mp.Process(target=producer_sm, args=(sender,)).start()
+    mp.Process(target=consumer_sm, args=(receiver,)).start()
+
+```
+
+# Performance Testing
+Note that in this testing example Producer and Consumer where dependent upon each other (due to Queue capacity), that's why they take a similar amount of time. Your actual performance may vary depending on the data type, structure and overall system performance.
+
+Since there is no inter-process data transfer, I/O drops accordingly. E.g. in my case -2.4GB/s
+
+```
+--------------------
+Bytes: 100 (100 B), loops: 100000, queue capacity: 1000
+Producer done in     0.78797s  @ 12.10 MB/s
+Consumer done in     0.78698s  @ 12.12 MB/s
+Producer done in SM  5.37207s  @ 1.78 MB/s
+Consumer done in SM  5.37207s  @ 1.78 MB/s   ❌
+--------------------
+Bytes: 1000 (1000 B), loops: 100000, queue capacity: 1000
+Producer done in     0.92900s  @ 102.66 MB/s
+Consumer done in     0.91599s  @ 104.11 MB/s
+Producer done in SM  5.45683s  @ 17.48 MB/s
+Consumer done in SM  5.45582s  @ 17.48 MB/s   ❌
+--------------------
+Bytes: 10000 (9.77 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.16847s  @ 439.79 MB/s
+Consumer done in     2.16147s  @ 441.22 MB/s
+Producer done in SM  5.64625s  @ 168.90 MB/s 
+Consumer done in SM  5.64625s  @ 168.90 MB/s   ❌
+--------------------
+Bytes: 100000 (97.66 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.80213s  @ 3.32 GB/s
+Consumer done in     2.80013s  @ 3.33 GB/s
+Producer done in SM  8.24400s  @ 1.13 GB/s
+Consumer done in SM  8.24200s  @ 1.13 GB/s   ❌
+--------------------
+Bytes: 1000000 (976.56 KB), loops: 10000, queue capacity: 1000
+Producer done in     4.87300s  @ 1.91 GB/s
+Consumer done in     4.87300s  @ 1.91 GB/s
+Producer done in SM  4.01900s  @ 2.32 GB/s 
+Consumer done in SM  4.01800s  @ 2.32 GB/s   ✅
+--------------------
+Bytes: 10000000 (9.54 MB), loops: 1000, queue capacity: 1000
+Producer done in     6.25722s  @ 1.49 GB/s
+Consumer done in     6.28221s  @ 1.48 GB/s
+Producer done in SM  3.79851s  @ 2.45 GB/s 
+Consumer done in SM  3.80359s  @ 2.45 GB/s   ✅
+--------------------
+Bytes: 100000000 (95.37 MB), loops: 1000, queue capacity: 100
+Producer done in     64.91876s  @ 1.43 GB/s
+Consumer done in     65.20476s  @ 1.43 GB/s
+Producer done in SM  38.08093s  @ 2.45 GB/s
+Consumer done in SM  38.17893s  @ 2.44 GB/s   ✅
+--------------------
+Bytes: 1000000000 (953.67 MB), loops: 100, queue capacity: 10
+Producer done in     63.22359s  @ 1.47 GB/s
+Consumer done in     66.07801s  @ 1.41 GB/s
+Producer done in SM  36.39488s  @ 2.56 GB/s
+Consumer done in SM  37.61108s  @ 2.48 GB/s   ✅
+--------------------
+Bytes: 10000000000 (9.31 GB), loops: 10, queue capacity: 10
+Producer done in     CRASHED 
+Consumer done in     CRASHED 
+Producer done in SM  28.21499s  @ 3.30 GB/s
+Consumer done in SM  34.32684s  @ 2.71 GB/s   ✅✅
+```

py_sharedmemory-1.0.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+memory/__init__.py,sha256=3FFTy2frl1KxNej5rvximE5z6oCVlORC2Ia8Ac9jTkQ,88
+memory/converter.py,sha256=S_zqjuaJpWzK2eGmBLUCPip32kSkZY82M7AVg_7Ryi0,2966
+memory/factory.py,sha256=PD1zdr9srMbbNYhkI8dRz_7e8Wq420uTX07WfExs4sQ,356
+memory/receiver.py,sha256=ZUFuP6GS1E-oAQmGpCF--vWjn_5Jb2dHi4aL2fb-8k0,2398
+memory/sender.py,sha256=IO-KyWu5tqG4avd1sAOLiVy__g2sdMRmfYUamgxn44w,4820
+py_sharedmemory-1.0.0.dist-info/LICENSE,sha256=h0J2LgUZdY3fXKnyBle1HiJoBrUk8dIZpXfVeBvgIe0,1090
+py_sharedmemory-1.0.0.dist-info/METADATA,sha256=uMhsgDJO3PUAFPcXmcXtuY2A0KR8135Mf7LaIC4cfoA,7158
+py_sharedmemory-1.0.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+py_sharedmemory-1.0.0.dist-info/top_level.txt,sha256=plUqRjD5ukqJEMYsc-bhdIrLgYnGmx3mALaoMw-3E8o,7
+py_sharedmemory-1.0.0.dist-info/RECORD,,

py_sharedmemory-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

py_sharedmemory-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+memory