py-sharedmemory 1.0.0__tar.gz

py_sharedmemory-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 товіаѕ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

py_sharedmemory-1.0.0/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: py-sharedmemory
+Version: 1.0.0
+Summary: Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+Author: Tobias Würth
+License: MIT License
+        
+        Copyright (c) 2024 товіаѕ
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Repository, https://github.com/tobiaswuerth/python_shared_memory_queue
+Project-URL: Bug Tracker, https://github.com/tobiaswuerth/python_shared_memory_queue/issues
+Keywords: python,python3,multiprocessing,sharedmemory,queue,pipe
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+
+# Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+
+Since [multiprocessing](https://docs.python.org/3/library/multiprocessing.html) [Queue](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue)s pickles the data, there is a certain performance decrease for larger data objects and huge amounts of I/O.
+
+This is the reason why I created a substitute which instead converts data objects into bytes (if needed), puts the bytes into shared memory and only transfers meta-data about the data structure via Queues to the receiving end.
+
+The following data types are currently supported:
+```
+- bytes
+- int
+- float
+- bool
+- str
+- np.ndarray
+- np.dtype
+```
+
+As well as any nested structures for the given types:
+```
+- tuple/NamedTuple
+- list
+- dict
+- set
+```
+
+# Considerations
+There is a certain overhead to the conversion process which is especially noticable for smaller objects and more complex data structures.
+Use the following approach depending on the size of the data you're handling:
+
+||10B|100B|1KB|10KB|100KB|1MB|10MB|100MB|1GB|10GB|
+|---|---|---|---|---|---|---|---|---|---|---|
+|``mp.Queue()``|✅|✅|✅|✅|✅|❌|❌|❌|❌|❌|
+|``SharedMemory``|❌|❌|❌|❌|❌|✅|✅|✅|✅|✅|
+
+Actual performance heavily relies on your memory speed.
+
+# Usage example
+```python
+import multiprocessing as mp
+
+from memory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver
+
+def producer_sm(sender:SharedMemorySender):
+    your_data = "your data"
+
+    # ...
+    sender.put(your_data) # blocks until there is space
+
+    # or
+    if sender.has_space():
+        sender.put(your_data)
+    else:
+        # your else case
+    
+    # ...
+    sender.wait_for_all_ack() # wait for all data to be received before closing the sender
+    # exit
+
+def consumer_sm(receiver:SharedMemoryReceiver, loops, size):
+    # ...
+    data = receiver.get() # blocks
+    
+    # or
+    data = receiver.get(timeout=3) # raises queue.Empty exception after 3s
+
+    # or
+    data = receiver.get_nowait()
+    if data is None:
+        # handle empty case
+
+    # ...
+
+if __name__ == '__main__':
+    sender, receiver = create_shared_memory_pair(capacity=5)
+
+    mp.Process(target=producer_sm, args=(sender,)).start()
+    mp.Process(target=consumer_sm, args=(receiver,)).start()
+
+```
+
+# Performance Testing
+Note that in this testing example Producer and Consumer where dependent upon each other (due to Queue capacity), that's why they take a similar amount of time. Your actual performance may vary depending on the data type, structure and overall system performance.
+
+Since there is no inter-process data transfer, I/O drops accordingly. E.g. in my case -2.4GB/s
+
+```
+--------------------
+Bytes: 100 (100 B), loops: 100000, queue capacity: 1000
+Producer done in     0.78797s  @ 12.10 MB/s
+Consumer done in     0.78698s  @ 12.12 MB/s
+Producer done in SM  5.37207s  @ 1.78 MB/s
+Consumer done in SM  5.37207s  @ 1.78 MB/s   ❌
+--------------------
+Bytes: 1000 (1000 B), loops: 100000, queue capacity: 1000
+Producer done in     0.92900s  @ 102.66 MB/s
+Consumer done in     0.91599s  @ 104.11 MB/s
+Producer done in SM  5.45683s  @ 17.48 MB/s
+Consumer done in SM  5.45582s  @ 17.48 MB/s   ❌
+--------------------
+Bytes: 10000 (9.77 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.16847s  @ 439.79 MB/s
+Consumer done in     2.16147s  @ 441.22 MB/s
+Producer done in SM  5.64625s  @ 168.90 MB/s 
+Consumer done in SM  5.64625s  @ 168.90 MB/s   ❌
+--------------------
+Bytes: 100000 (97.66 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.80213s  @ 3.32 GB/s
+Consumer done in     2.80013s  @ 3.33 GB/s
+Producer done in SM  8.24400s  @ 1.13 GB/s
+Consumer done in SM  8.24200s  @ 1.13 GB/s   ❌
+--------------------
+Bytes: 1000000 (976.56 KB), loops: 10000, queue capacity: 1000
+Producer done in     4.87300s  @ 1.91 GB/s
+Consumer done in     4.87300s  @ 1.91 GB/s
+Producer done in SM  4.01900s  @ 2.32 GB/s 
+Consumer done in SM  4.01800s  @ 2.32 GB/s   ✅
+--------------------
+Bytes: 10000000 (9.54 MB), loops: 1000, queue capacity: 1000
+Producer done in     6.25722s  @ 1.49 GB/s
+Consumer done in     6.28221s  @ 1.48 GB/s
+Producer done in SM  3.79851s  @ 2.45 GB/s 
+Consumer done in SM  3.80359s  @ 2.45 GB/s   ✅
+--------------------
+Bytes: 100000000 (95.37 MB), loops: 1000, queue capacity: 100
+Producer done in     64.91876s  @ 1.43 GB/s
+Consumer done in     65.20476s  @ 1.43 GB/s
+Producer done in SM  38.08093s  @ 2.45 GB/s
+Consumer done in SM  38.17893s  @ 2.44 GB/s   ✅
+--------------------
+Bytes: 1000000000 (953.67 MB), loops: 100, queue capacity: 10
+Producer done in     63.22359s  @ 1.47 GB/s
+Consumer done in     66.07801s  @ 1.41 GB/s
+Producer done in SM  36.39488s  @ 2.56 GB/s
+Consumer done in SM  37.61108s  @ 2.48 GB/s   ✅
+--------------------
+Bytes: 10000000000 (9.31 GB), loops: 10, queue capacity: 10
+Producer done in     CRASHED 
+Consumer done in     CRASHED 
+Producer done in SM  28.21499s  @ 3.30 GB/s
+Consumer done in SM  34.32684s  @ 2.71 GB/s   ✅✅
+```

py_sharedmemory-1.0.0/README.md

@@ -0,0 +1,141 @@
+# Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+
+Since [multiprocessing](https://docs.python.org/3/library/multiprocessing.html) [Queue](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue)s pickles the data, there is a certain performance decrease for larger data objects and huge amounts of I/O.
+
+This is the reason why I created a substitute which instead converts data objects into bytes (if needed), puts the bytes into shared memory and only transfers meta-data about the data structure via Queues to the receiving end.
+
+The following data types are currently supported:
+```
+- bytes
+- int
+- float
+- bool
+- str
+- np.ndarray
+- np.dtype
+```
+
+As well as any nested structures for the given types:
+```
+- tuple/NamedTuple
+- list
+- dict
+- set
+```
+
+# Considerations
+There is a certain overhead to the conversion process which is especially noticable for smaller objects and more complex data structures.
+Use the following approach depending on the size of the data you're handling:
+
+||10B|100B|1KB|10KB|100KB|1MB|10MB|100MB|1GB|10GB|
+|---|---|---|---|---|---|---|---|---|---|---|
+|``mp.Queue()``|✅|✅|✅|✅|✅|❌|❌|❌|❌|❌|
+|``SharedMemory``|❌|❌|❌|❌|❌|✅|✅|✅|✅|✅|
+
+Actual performance heavily relies on your memory speed.
+
+# Usage example
+```python
+import multiprocessing as mp
+
+from memory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver
+
+def producer_sm(sender:SharedMemorySender):
+    your_data = "your data"
+
+    # ...
+    sender.put(your_data) # blocks until there is space
+
+    # or
+    if sender.has_space():
+        sender.put(your_data)
+    else:
+        # your else case
+    
+    # ...
+    sender.wait_for_all_ack() # wait for all data to be received before closing the sender
+    # exit
+
+def consumer_sm(receiver:SharedMemoryReceiver, loops, size):
+    # ...
+    data = receiver.get() # blocks
+    
+    # or
+    data = receiver.get(timeout=3) # raises queue.Empty exception after 3s
+
+    # or
+    data = receiver.get_nowait()
+    if data is None:
+        # handle empty case
+
+    # ...
+
+if __name__ == '__main__':
+    sender, receiver = create_shared_memory_pair(capacity=5)
+
+    mp.Process(target=producer_sm, args=(sender,)).start()
+    mp.Process(target=consumer_sm, args=(receiver,)).start()
+
+```
+
+# Performance Testing
+Note that in this testing example Producer and Consumer where dependent upon each other (due to Queue capacity), that's why they take a similar amount of time. Your actual performance may vary depending on the data type, structure and overall system performance.
+
+Since there is no inter-process data transfer, I/O drops accordingly. E.g. in my case -2.4GB/s
+
+```
+--------------------
+Bytes: 100 (100 B), loops: 100000, queue capacity: 1000
+Producer done in     0.78797s  @ 12.10 MB/s
+Consumer done in     0.78698s  @ 12.12 MB/s
+Producer done in SM  5.37207s  @ 1.78 MB/s
+Consumer done in SM  5.37207s  @ 1.78 MB/s   ❌
+--------------------
+Bytes: 1000 (1000 B), loops: 100000, queue capacity: 1000
+Producer done in     0.92900s  @ 102.66 MB/s
+Consumer done in     0.91599s  @ 104.11 MB/s
+Producer done in SM  5.45683s  @ 17.48 MB/s
+Consumer done in SM  5.45582s  @ 17.48 MB/s   ❌
+--------------------
+Bytes: 10000 (9.77 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.16847s  @ 439.79 MB/s
+Consumer done in     2.16147s  @ 441.22 MB/s
+Producer done in SM  5.64625s  @ 168.90 MB/s 
+Consumer done in SM  5.64625s  @ 168.90 MB/s   ❌
+--------------------
+Bytes: 100000 (97.66 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.80213s  @ 3.32 GB/s
+Consumer done in     2.80013s  @ 3.33 GB/s
+Producer done in SM  8.24400s  @ 1.13 GB/s
+Consumer done in SM  8.24200s  @ 1.13 GB/s   ❌
+--------------------
+Bytes: 1000000 (976.56 KB), loops: 10000, queue capacity: 1000
+Producer done in     4.87300s  @ 1.91 GB/s
+Consumer done in     4.87300s  @ 1.91 GB/s
+Producer done in SM  4.01900s  @ 2.32 GB/s 
+Consumer done in SM  4.01800s  @ 2.32 GB/s   ✅
+--------------------
+Bytes: 10000000 (9.54 MB), loops: 1000, queue capacity: 1000
+Producer done in     6.25722s  @ 1.49 GB/s
+Consumer done in     6.28221s  @ 1.48 GB/s
+Producer done in SM  3.79851s  @ 2.45 GB/s 
+Consumer done in SM  3.80359s  @ 2.45 GB/s   ✅
+--------------------
+Bytes: 100000000 (95.37 MB), loops: 1000, queue capacity: 100
+Producer done in     64.91876s  @ 1.43 GB/s
+Consumer done in     65.20476s  @ 1.43 GB/s
+Producer done in SM  38.08093s  @ 2.45 GB/s
+Consumer done in SM  38.17893s  @ 2.44 GB/s   ✅
+--------------------
+Bytes: 1000000000 (953.67 MB), loops: 100, queue capacity: 10
+Producer done in     63.22359s  @ 1.47 GB/s
+Consumer done in     66.07801s  @ 1.41 GB/s
+Producer done in SM  36.39488s  @ 2.56 GB/s
+Consumer done in SM  37.61108s  @ 2.48 GB/s   ✅
+--------------------
+Bytes: 10000000000 (9.31 GB), loops: 10, queue capacity: 10
+Producer done in     CRASHED 
+Consumer done in     CRASHED 
+Producer done in SM  28.21499s  @ 3.30 GB/s
+Consumer done in SM  34.32684s  @ 2.71 GB/s   ✅✅
+```

py_sharedmemory-1.0.0/memory/__init__.py

@@ -0,0 +1 @@
+from .factory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver

py_sharedmemory-1.0.0/memory/converter.py

@@ -0,0 +1,106 @@
+import numpy as np
+
+def _encode_int(item:int):
+    size = item.bit_length() // 8 + 1
+    signed = item < 0
+    bytes_data = item.to_bytes(size, 'little', signed=signed)
+    return ('i', size, signed), memoryview(bytes_data)
+
+def _decode_int(byte_data:memoryview, info:tuple):
+    return int.from_bytes(byte_data, 'little', signed=info[2])
+
+def _encode_float(item:float):
+    byte_data = np.float64(item).tobytes()
+    return ('f', 8), memoryview(byte_data)
+
+def _decode_float(byte_data:memoryview, info:tuple):
+    return np.frombuffer(byte_data, dtype=np.float64)[0]
+
+def _encode_bool(item:bool):
+    byte_data = item.to_bytes(1, 'little')
+    return ('b', 1), memoryview(byte_data)
+
+def _decode_bool(byte_data:memoryview, info:tuple):
+    return bool.from_bytes(byte_data, 'little')
+
+def _encode_bytes(item:bytes):
+    return ('r', len(item)), memoryview(item)
+
+def _decode_bytes(byte_data:memoryview, info:tuple):
+    if byte_data is None:
+        return b''
+    return bytes(byte_data)
+
+def _encode_str(item:str):
+    bytes_data = str.encode(item)
+    return ('s', len(bytes_data)), memoryview(bytes_data)
+
+def _decode_str(byte_data:memoryview, info:tuple):
+    if byte_data is None:
+        return ''
+    return bytes(byte_data).decode()
+
+def _encode_ndarray(item:np.ndarray):
+    byte_data = item.tobytes()
+    return ('n', len(byte_data), item.dtype, item.shape), memoryview(byte_data)
+
+def _decode_ndarray(byte_data:memoryview, info:tuple):
+    return np.frombuffer(byte_data, dtype=info[2]).copy().reshape(info[3])
+
+def _encode_ndtype(item:np.dtype):
+    byte_data = str.encode(str(item))
+    return ('t', len(byte_data)), memoryview(byte_data)
+
+def _decode_ndtype(byte_data:memoryview, info:tuple):
+    return np.dtype(bytes(byte_data).decode())
+
+iterables = {
+    tuple: 'p',
+    list: 'l',
+    dict: 'd',
+    set: 'u',
+}
+
+encoder = {
+    bytes: _encode_bytes,
+    int: _encode_int,
+    float: _encode_float,
+    bool: _encode_bool,
+    str: _encode_str,
+    np.ndarray: _encode_ndarray,
+}
+decoder = {
+    'r': _decode_bytes,
+    'i': _decode_int,
+    'f': _decode_float,
+    'b': _decode_bool,
+    's': _decode_str,
+    'n': _decode_ndarray,
+}
+
+def encode(item):
+    if item is None:
+        return (None, 0), b''
+    
+    datatype = type(item)
+    enc = encoder.get(datatype, None)
+    if enc is not None:
+        return enc(item)
+    
+    if isinstance(item, np.dtype):
+        return _encode_ndtype(item)
+
+    raise ValueError(f"Unsupported data type: {type(item)}")
+
+def decode(bytes_data:memoryview, info:tuple):
+        if info[0] is None:
+            return None
+        
+        dec = decoder.get(info[0], None)
+        if dec is not None:
+            return dec(bytes_data, info)
+        
+        if info[0] == 't':
+            return _decode_ndtype(bytes_data, info)
+        
+        raise ValueError(f"Unsupported data type: {info[0]}")

py_sharedmemory-1.0.0/memory/factory.py

@@ -0,0 +1,12 @@
+import multiprocessing as mp
+
+from .receiver import SharedMemoryReceiver
+from .sender import SharedMemorySender
+
+
+def create_shared_memory_pair(capacity):
+    q_data = mp.Queue(capacity)
+    q_ack = mp.Queue()
+    sender = SharedMemorySender(capacity, q_data, q_ack)
+    receiver = SharedMemoryReceiver(q_data, q_ack)
+    return sender, receiver

py_sharedmemory-1.0.0/memory/receiver.py

@@ -0,0 +1,75 @@
+import multiprocessing as mp
+import queue
+from multiprocessing.shared_memory import SharedMemory
+
+from .converter import decode, iterables
+
+iterable_keys = set(iterables.values())
+non_dict_iterables = set(iterables.keys()) - {dict}
+
+class SharedMemoryReceiver:
+    def __init__(self, queue_data_in:mp.Queue, queue_ack_out:mp.Queue):
+        self.queue_data_in = queue_data_in
+        self.queue_ack_out = queue_ack_out
+
+    def _recover_data_recursive(self, shm:SharedMemory, info:tuple, byte_idx=0):
+        if info[0] not in iterable_keys:
+            if info[1] == 0: # size
+                return decode(None, info)
+            return decode(shm.buf[byte_idx:byte_idx+info[1]], info)
+
+        if info[0] == 'd': # dict
+            data = {}
+            for info_child, key in zip(info[3], info[4]):
+                item = self._recover_data_recursive(shm, info_child, byte_idx)
+                byte_idx += info_child[1]
+                data[key] = item
+            return data
+
+        # tuple, list, set
+        data = []
+        for info_child in info[3]:
+            item = self._recover_data_recursive(shm, info_child, byte_idx)
+            byte_idx += info_child[1]
+            data.append(item)
+        
+        if info[2] in non_dict_iterables:
+            return info[2](data)
+        
+        # NamedTuple needs to be handled separately
+        return info[2](*data)
+
+    def _recover_data(self, info:tuple):
+        if info[2] == 0:
+            return self._recover_data_recursive(None, info[1:])
+
+        name = info[0]
+        info = info[1:]
+        shm = SharedMemory(name=name, size=info[1])
+        data = self._recover_data_recursive(shm, info)
+        shm.close()
+        return data
+
+    def get_nowait(self):
+        if self.queue_data_in.empty():
+            return None
+        
+        try:
+            info = self.queue_data_in.get_nowait()
+        except queue.Empty:
+            return None
+        
+        if info is None:
+            return None
+        
+        item = self._recover_data(info)
+        self.queue_ack_out.put_nowait(info[0])
+        del info
+        return item
+
+    def get(self, timeout=None):
+        info = self.queue_data_in.get(timeout=timeout)
+        item = self._recover_data(info)
+        self.queue_ack_out.put_nowait(info[0])
+        del info
+        return item

py_sharedmemory-1.0.0/memory/sender.py

@@ -0,0 +1,149 @@
+import multiprocessing as mp
+import queue
+from multiprocessing.shared_memory import SharedMemory
+
+from .converter import encode, iterables
+
+class _Separator:
+    def __init__(self, obj):
+        self.obj = obj
+
+def _iterable_handler_list(stack, parents, entry):
+    stack.append(_Separator((type(entry), entry.__class__,)))
+    parents.append(([], []))
+    for field in reversed(entry): # reverse to maintain order in final binary
+        stack.append(field)
+
+def _iterable_handler_set(stack, parents, entry):
+    stack.append(_Separator((set,entry.__class__,)))
+    parents.append(([], []))
+    for field in entry:
+        stack.append(field)
+
+def _iterable_handler_dict(stack, parents, entry):
+    keys = list(entry.keys())
+    stack.append(_Separator((dict,entry.__class__,keys)))
+    parents.append(([], []))
+    for key in reversed(keys): # reverse to maintain order in final binary
+        stack.append(entry[key])
+
+iterable_handlers = {
+    list: _iterable_handler_list,
+    tuple: _iterable_handler_list,
+    set: _iterable_handler_set,
+    dict: _iterable_handler_dict,
+}
+
+tuple_key = iterables[tuple]
+
+class SharedMemorySender:
+    def __init__(self, capacity, queue_data_out:mp.Queue, queue_ack_in:mp.Queue):
+        self.open_handles = {}
+        self.queue_data_out = queue_data_out
+        self.queue_ack_in = queue_ack_in
+        self.capacity = capacity
+
+    def _prepare_binary(self, data):
+        parents:list[tuple[list[memoryview], list[tuple]]] = []
+        stack = [data]
+        while stack:
+            entry = stack.pop()
+            
+            if isinstance(entry, _Separator):
+                # children processed, gather the results
+                bytes_data, child_infos = parents.pop()
+                entry_type = entry.obj[0]
+                key = iterables.get(entry_type, tuple_key)
+                size = sum(info[1] for info in child_infos)
+                clazz = entry.obj[1]
+                if entry_type == dict:
+                    info = (key, size, clazz, child_infos, entry.obj[2])
+                else:   
+                    info = (key, size, clazz, child_infos)
+
+                if not parents:
+                    # root entry was tuple, return the result
+                    del parents, stack
+                    return info, bytes_data
+
+                # append to parent binary
+                parents[-1][0].extend(bytes_data)
+                parents[-1][1].append(info)
+                continue
+
+            entry_type = type(entry)
+            if entry_type in iterables:
+                iterable_handlers[entry_type](stack, parents, entry)
+                continue
+            if isinstance(entry, tuple):
+                # NamedTuple
+                _iterable_handler_list(stack, parents, entry)
+                continue
+
+            info, bytes_data = encode(entry)
+            if not parents:
+                # root entry was single data, return the result
+                del parents, stack
+                return info, [bytes_data]
+
+            parents[-1][0].append(bytes_data)
+            parents[-1][1].append(info)
+
+    def put(self, data):
+        self.process_ack()
+        while len(self.open_handles) >= self.capacity:
+            self.wait_for_ack()
+
+        info, bytes_datas = self._prepare_binary(data)
+        size = info[1]
+        if size == 0:
+            info = (None, *info)
+            self.queue_data_out.put(info)
+            return
+
+        shm = SharedMemory(create=True, size=size)
+        info = (shm.name, *info)
+
+        offset = 0
+        for item in bytes_datas:
+            size = len(item)
+            shm.buf[offset:offset + size] = item
+            offset += size
+
+        self.queue_data_out.put(info)
+        self.open_handles[shm.name] = shm
+
+    def close_handle(self, shm_name:str):
+        shm = self.open_handles[shm_name]
+        shm.close()
+        shm.unlink()
+        del self.open_handles[shm_name]
+
+    def has_space(self):
+        self.process_ack()
+        return len(self.open_handles) < self.capacity
+
+    def process_ack(self):
+        try:
+            while True:
+                retour = self.queue_ack_in.get_nowait()
+                if retour is None:
+                    return
+                self.close_handle(retour)
+        except queue.Empty:
+            pass
+
+    def close_all(self):
+        for shm_name in list(self.open_handles.keys()):
+            self.close_handle(shm_name)
+
+    def wait_for_ack(self):
+        shm_name = self.queue_ack_in.get()
+        self.close_handle(shm_name)
+
+    def wait_for_all_ack(self):
+        while len(self.open_handles) > 0:
+            self.wait_for_ack()
+
+    def __del__(self):
+        self.close_all()

py_sharedmemory-1.0.0/py_sharedmemory.egg-info/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: py-sharedmemory
+Version: 1.0.0
+Summary: Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+Author: Tobias Würth
+License: MIT License
+        
+        Copyright (c) 2024 товіаѕ
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Repository, https://github.com/tobiaswuerth/python_shared_memory_queue
+Project-URL: Bug Tracker, https://github.com/tobiaswuerth/python_shared_memory_queue/issues
+Keywords: python,python3,multiprocessing,sharedmemory,queue,pipe
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+
+# Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)
+
+Since [multiprocessing](https://docs.python.org/3/library/multiprocessing.html) [Queue](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue)s pickles the data, there is a certain performance decrease for larger data objects and huge amounts of I/O.
+
+This is the reason why I created a substitute which instead converts data objects into bytes (if needed), puts the bytes into shared memory and only transfers meta-data about the data structure via Queues to the receiving end.
+
+The following data types are currently supported:
+```
+- bytes
+- int
+- float
+- bool
+- str
+- np.ndarray
+- np.dtype
+```
+
+As well as any nested structures for the given types:
+```
+- tuple/NamedTuple
+- list
+- dict
+- set
+```
+
+# Considerations
+There is a certain overhead to the conversion process which is especially noticable for smaller objects and more complex data structures.
+Use the following approach depending on the size of the data you're handling:
+
+||10B|100B|1KB|10KB|100KB|1MB|10MB|100MB|1GB|10GB|
+|---|---|---|---|---|---|---|---|---|---|---|
+|``mp.Queue()``|✅|✅|✅|✅|✅|❌|❌|❌|❌|❌|
+|``SharedMemory``|❌|❌|❌|❌|❌|✅|✅|✅|✅|✅|
+
+Actual performance heavily relies on your memory speed.
+
+# Usage example
+```python
+import multiprocessing as mp
+
+from memory import create_shared_memory_pair, SharedMemorySender, SharedMemoryReceiver
+
+def producer_sm(sender:SharedMemorySender):
+    your_data = "your data"
+
+    # ...
+    sender.put(your_data) # blocks until there is space
+
+    # or
+    if sender.has_space():
+        sender.put(your_data)
+    else:
+        # your else case
+    
+    # ...
+    sender.wait_for_all_ack() # wait for all data to be received before closing the sender
+    # exit
+
+def consumer_sm(receiver:SharedMemoryReceiver, loops, size):
+    # ...
+    data = receiver.get() # blocks
+    
+    # or
+    data = receiver.get(timeout=3) # raises queue.Empty exception after 3s
+
+    # or
+    data = receiver.get_nowait()
+    if data is None:
+        # handle empty case
+
+    # ...
+
+if __name__ == '__main__':
+    sender, receiver = create_shared_memory_pair(capacity=5)
+
+    mp.Process(target=producer_sm, args=(sender,)).start()
+    mp.Process(target=consumer_sm, args=(receiver,)).start()
+
+```
+
+# Performance Testing
+Note that in this testing example Producer and Consumer where dependent upon each other (due to Queue capacity), that's why they take a similar amount of time. Your actual performance may vary depending on the data type, structure and overall system performance.
+
+Since there is no inter-process data transfer, I/O drops accordingly. E.g. in my case -2.4GB/s
+
+```
+--------------------
+Bytes: 100 (100 B), loops: 100000, queue capacity: 1000
+Producer done in     0.78797s  @ 12.10 MB/s
+Consumer done in     0.78698s  @ 12.12 MB/s
+Producer done in SM  5.37207s  @ 1.78 MB/s
+Consumer done in SM  5.37207s  @ 1.78 MB/s   ❌
+--------------------
+Bytes: 1000 (1000 B), loops: 100000, queue capacity: 1000
+Producer done in     0.92900s  @ 102.66 MB/s
+Consumer done in     0.91599s  @ 104.11 MB/s
+Producer done in SM  5.45683s  @ 17.48 MB/s
+Consumer done in SM  5.45582s  @ 17.48 MB/s   ❌
+--------------------
+Bytes: 10000 (9.77 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.16847s  @ 439.79 MB/s
+Consumer done in     2.16147s  @ 441.22 MB/s
+Producer done in SM  5.64625s  @ 168.90 MB/s 
+Consumer done in SM  5.64625s  @ 168.90 MB/s   ❌
+--------------------
+Bytes: 100000 (97.66 KB), loops: 100000, queue capacity: 1000
+Producer done in     2.80213s  @ 3.32 GB/s
+Consumer done in     2.80013s  @ 3.33 GB/s
+Producer done in SM  8.24400s  @ 1.13 GB/s
+Consumer done in SM  8.24200s  @ 1.13 GB/s   ❌
+--------------------
+Bytes: 1000000 (976.56 KB), loops: 10000, queue capacity: 1000
+Producer done in     4.87300s  @ 1.91 GB/s
+Consumer done in     4.87300s  @ 1.91 GB/s
+Producer done in SM  4.01900s  @ 2.32 GB/s 
+Consumer done in SM  4.01800s  @ 2.32 GB/s   ✅
+--------------------
+Bytes: 10000000 (9.54 MB), loops: 1000, queue capacity: 1000
+Producer done in     6.25722s  @ 1.49 GB/s
+Consumer done in     6.28221s  @ 1.48 GB/s
+Producer done in SM  3.79851s  @ 2.45 GB/s 
+Consumer done in SM  3.80359s  @ 2.45 GB/s   ✅
+--------------------
+Bytes: 100000000 (95.37 MB), loops: 1000, queue capacity: 100
+Producer done in     64.91876s  @ 1.43 GB/s
+Consumer done in     65.20476s  @ 1.43 GB/s
+Producer done in SM  38.08093s  @ 2.45 GB/s
+Consumer done in SM  38.17893s  @ 2.44 GB/s   ✅
+--------------------
+Bytes: 1000000000 (953.67 MB), loops: 100, queue capacity: 10
+Producer done in     63.22359s  @ 1.47 GB/s
+Consumer done in     66.07801s  @ 1.41 GB/s
+Producer done in SM  36.39488s  @ 2.56 GB/s
+Consumer done in SM  37.61108s  @ 2.48 GB/s   ✅
+--------------------
+Bytes: 10000000000 (9.31 GB), loops: 10, queue capacity: 10
+Producer done in     CRASHED 
+Consumer done in     CRASHED 
+Producer done in SM  28.21499s  @ 3.30 GB/s
+Consumer done in SM  34.32684s  @ 2.71 GB/s   ✅✅
+```

py_sharedmemory-1.0.0/py_sharedmemory.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+memory/__init__.py
+memory/converter.py
+memory/factory.py
+memory/receiver.py
+memory/sender.py
+py_sharedmemory.egg-info/PKG-INFO
+py_sharedmemory.egg-info/SOURCES.txt
+py_sharedmemory.egg-info/dependency_links.txt
+py_sharedmemory.egg-info/requires.txt
+py_sharedmemory.egg-info/top_level.txt
+tests/test_shared_memory.py

py_sharedmemory-1.0.0/py_sharedmemory.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

py_sharedmemory-1.0.0/py_sharedmemory.egg-info/requires.txt

@@ -0,0 +1 @@
+numpy>=1.26

py_sharedmemory-1.0.0/py_sharedmemory.egg-info/top_level.txt

@@ -0,0 +1 @@
+memory

py_sharedmemory-1.0.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ['setuptools>=61.0', 'numpy>=1.26']
+build-backend = 'setuptools.build_meta'
+
+[tool.setuptools]
+packages = ['memory']
+
+[project]
+name = "py-sharedmemory"
+authors = [{name = "Tobias Würth"}]
+version = "1.0.0"
+requires-python = ">=3.9"
+dependencies = [
+    "numpy>=1.26"
+]
+
+description = "Shared Memory Pipe for Fast Multiprocessing Data Sharing of Large Objects (>1MB)"
+keywords = ["python", "python3", "multiprocessing", "sharedmemory", "queue", "pipe"]
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {file = "LICENSE"}
+
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Repository = "https://github.com/tobiaswuerth/python_shared_memory_queue"
+"Bug Tracker" = "https://github.com/tobiaswuerth/python_shared_memory_queue/issues"

py_sharedmemory-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

py_sharedmemory-1.0.0/tests/test_shared_memory.py

@@ -0,0 +1,339 @@
+import unittest
+import numpy as np
+
+import multiprocessing as mp
+mp.log_to_stderr()
+
+from typing import NamedTuple
+
+from memory import create_shared_memory_pair
+
+class MyTuple(NamedTuple):
+    a:int
+    b:float
+    c:str
+
+class TestSharedMemory(unittest.TestCase):
+    def test_None(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = None
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_None_tuple(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = (None,None)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_None_list(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [None,None]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_None_dict(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {None:None}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_None_set(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {None}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_int_positive_small(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = 37
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_int_negative_small(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = -37
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_int_positive_big(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = 2147483640
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_int_negative_big(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = -2147483640
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_float1(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = 21474.83640
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_float2(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = -21474.83640
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_bool1(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = True
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_bool2(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = True
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_bytes(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = b"Hello World!"
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_bytes_empty(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = b''
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_str(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = "Hello World!"
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_str_empty(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = ""
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_str_long(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = """Zarathustra went down the mountain alone, no one meeting him. When he entered the forest, however, there suddenly stood before him an old man, who had left his holy cot to seek roots. And thus spake the old man to Zarathustra:
+“No stranger to me is this wanderer: many years ago passed he by. Zarathustra he was called; but he hath altered.
+Then thou carriedst thine ashes into the mountains: wilt thou now carry thy fire into the valleys? Fearest thou not the incendiary’s doom?
+Yea, I recognise Zarathustra. Pure is his eye, and no loathing lurketh about his mouth. Goeth he not along like a dancer?
+Altered is Zarathustra; a child hath Zarathustra become; an awakened one is Zarathustra: what wilt thou do in the land of the sleepers?
+As in the sea hast thou lived in solitude, and it hath borne thee up. Alas, wilt thou now go ashore? Alas, wilt thou again drag thy body thyself?”
+Zarathustra answered: “I love mankind.”
+“Why,” said the saint, “did I go into the forest and the desert? Was it not because I loved men far too well?
+Now I love God: men, I do not love. Man is a thing too imperfect for me. Love to man would be fatal to me.”
+Zarathustra answered: “What spake I of love! I am bringing gifts unto men.”
+“Give them nothing,” said the saint. “Take rather part of their load, and carry it along with them—that will be most agreeable unto them: if only it be agreeable unto thee!
+If, however, thou wilt give unto them, give them no more than an alms, and let them also beg for it!”
+“No,” replied Zarathustra, “I give no alms. I am not poor enough for that.”
+The saint laughed at Zarathustra, and spake thus: “Then see to it that they accept thy treasures! They are distrustful of anchorites, and do not believe that we come with gifts.
+The fall of our footsteps ringeth too hollow through their streets. And just as at night, when they are in bed and hear a man abroad long before sunrise, so they ask themselves concerning us: Where goeth the thief?
+Go not to men, but stay in the forest! Go rather to the animals! Why not be like me—a bear amongst bears, a bird amongst birds?”
+“And what doeth the saint in the forest?” asked Zarathustra.
+The saint answered: “I make hymns and sing them; and in making hymns I laugh and weep and mumble: thus do I praise God.
+With singing, weeping, laughing, and mumbling do I praise the God who is my God. But what dost thou bring us as a gift?”
+When Zarathustra had heard these words, he bowed to the saint and said: “What should I have to give thee! Let me rather hurry hence lest I take aught away from thee!”—And thus they parted from one another, the old man and Zarathustra, laughing like schoolboys.
+When Zarathustra was alone, however, he said to his heart: “Could it be possible! This old saint in the forest hath not yet heard of it, that God is dead!”"""
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_ndarray1(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = np.array([1,2,3,4,5], dtype=np.uint32)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        np.testing.assert_array_equal(item, data, f"Expected {data}, got {item}")
+    def test_ndarray2(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = np.array([[1.2,2.3,3.4],[-53.3,-3535,1]], dtype=np.float32)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        np.testing.assert_array_equal(item, data, f"Expected {data}, got {item}")
+    def test_ndarray_big(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = np.random.rand(64, 30, 6, 84, 84).astype(np.float32)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        np.testing.assert_array_equal(item, data, f"Expected {data}, got {item}")
+
+    def test_ndtype1(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = np.dtype(np.float32)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_ndtype2(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = np.dtype(np.bool_)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_list(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [1,2,3,4,5]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [[1,2,3],[4,5,6]]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_mixed_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [1,2,[3,4],5]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_nested_tuple(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [1,2,(3,4),5]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_mixed(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [1,'a',3.4,5, None, b'Hello World!']
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_empty(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = []
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_empty_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [[],[]]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_list_empty_nested_tuple(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = [(),()]
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_tuple(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = (1,2,3,4,5)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_tuple_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = ((1,2,3),(4,5,6))
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_tuple_mixed_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = (1,2,[3,4],5)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_tuple_nested_list(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = (1,2,[3,4],5)
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_tuple_mixed(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = (1,'a',3.4,5, None, b'Hello World!')
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_tuple_empty(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = ()
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_tuple_empty_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = ((),())
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    
+    def test_namedtuple(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = MyTuple(1, 2.3, 'Hello World!')
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+    def test_dict(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1: 'a', 'b': 2, 3: 4}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_dict_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1: {2: 3, 4: 5}, 'a': {'b': 'c', 'd': 'e'}}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_dict_mixed_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1: {2: 3, 4: 5}, 'a': ['b', 'c', 'd']}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_dict_mixed(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1: 'a', 'b': 2, 3: 4, 'c': None, 'd': b'Hello World!'}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_dict_empty(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_dict_empty_nested(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1: {}, 'a': {}}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    
+    def test_set(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1,2,3,4,5}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_set_mixed(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = {1,2,'a',None,4}
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+    def test_set_empty(self):
+        sender, receiver = create_shared_memory_pair(capacity=1)
+        data = set()
+        sender.put(data)
+        item = receiver.get(timeout=2)
+        self.assertEqual(item, data, f"Expected {data}, got {item}")
+
+if __name__ == '__main__':
+    unittest.main()