py-sharedmemory 4.0.0__tar.gz → 4.1.0__tar.gz

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 4.1.0
+
+- Warn with a `RuntimeWarning` when leaving a `with sender:` block or exiting the
+  producing process discards unacknowledged messages, so a forgotten
+  `wait_for_all_ack()` no longer loses data silently. Explicit `close()` calls and
+  exits caused by exceptions stay silent, and closing still never blocks.
+
 ## 4.0.0
 
 - **Breaking:** the import package is renamed from `memory` to `py_sharedmemory` to match the

{py_sharedmemory-4.0.0/py_sharedmemory.egg-info → py_sharedmemory-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-sharedmemory
-Version: 4.0.0
+Version: 4.1.0
 Summary: Fast multiprocessing.Queue alternative that moves large Python objects and NumPy arrays through shared memory
 Author: Tobias Würth
 License-Expression: MIT
@@ -230,6 +230,11 @@ with sender:
         raise TimeoutError("Consumer did not receive everything")
 ```
 
+Closing never blocks. If a `with sender:` block or the producing process ends normally
+while messages are still unacknowledged, they are discarded and the sender emits a
+`RuntimeWarning`, so forgetting to drain does not lose data silently. An explicit
+`close()` or an exception leaving the `with` block discards without a warning.
+
 When both endpoints live in one process, drain before closing either of them. A sentinel
 such as `None` is a simple way to tell consumers that the stream has ended.
 
@@ -249,6 +254,7 @@ copy; other views, such as strided slices, arrive as contiguous copies.
 | `BrokenPipeError` | The endpoint is closed, or the producer's acknowledgement channel failed. |
 | `RuntimeError` | A second process tries to produce, or a used sender is passed to another process. |
 | Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still freed. |
+| `RuntimeWarning` (warning) | A `with sender:` block or the producing process ended normally and discarded unacknowledged messages. |
 
 ## API reference
 
@@ -265,9 +271,9 @@ for the underlying queues; it defaults to `multiprocessing.get_context()`.
 | `put(obj, block=True, timeout=None)` | Copy `obj` into a new shared-memory block and send it. |
 | `put_nowait(obj)` | `put(obj, block=False)`. |
 | `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and freed; `False` on timeout. |
-| `close()` | Free unread messages and release resources. Runs automatically at normal process exit. |
+| `close()` | Free unread messages silently and release resources. Runs automatically at normal process exit. |
 | `closed` | Whether the sender is closed. |
-| `with sender:` | Closes the sender on exit. |
+| `with sender:` | Closes the sender on exit; warns if a clean exit discards unacknowledged messages. |
 
 ### `SharedMemoryReceiver`
 

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/README.md

@@ -180,6 +180,11 @@ with sender:
         raise TimeoutError("Consumer did not receive everything")
 ```
 
+Closing never blocks. If a `with sender:` block or the producing process ends normally
+while messages are still unacknowledged, they are discarded and the sender emits a
+`RuntimeWarning`, so forgetting to drain does not lose data silently. An explicit
+`close()` or an exception leaving the `with` block discards without a warning.
+
 When both endpoints live in one process, drain before closing either of them. A sentinel
 such as `None` is a simple way to tell consumers that the stream has ended.
 
@@ -199,6 +204,7 @@ copy; other views, such as strided slices, arrive as contiguous copies.
 | `BrokenPipeError` | The endpoint is closed, or the producer's acknowledgement channel failed. |
 | `RuntimeError` | A second process tries to produce, or a used sender is passed to another process. |
 | Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still freed. |
+| `RuntimeWarning` (warning) | A `with sender:` block or the producing process ended normally and discarded unacknowledged messages. |
 
 ## API reference
 
@@ -215,9 +221,9 @@ for the underlying queues; it defaults to `multiprocessing.get_context()`.
 | `put(obj, block=True, timeout=None)` | Copy `obj` into a new shared-memory block and send it. |
 | `put_nowait(obj)` | `put(obj, block=False)`. |
 | `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and freed; `False` on timeout. |
-| `close()` | Free unread messages and release resources. Runs automatically at normal process exit. |
+| `close()` | Free unread messages silently and release resources. Runs automatically at normal process exit. |
 | `closed` | Whether the sender is closed. |
-| `with sender:` | Closes the sender on exit. |
+| `with sender:` | Closes the sender on exit; warns if a clean exit discards unacknowledged messages. |
 
 ### `SharedMemoryReceiver`
 

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/py_sharedmemory/__init__.py

@@ -25,5 +25,5 @@ from .factory import create_shared_memory_pair
 from .receiver import SharedMemoryReceiver
 from .sender import SharedMemorySender
 
-__version__ = "4.0.0"
+__version__ = "4.1.0"
 __all__ = ["SharedMemoryReceiver", "SharedMemorySender", "__version__", "create_shared_memory_pair"]

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/py_sharedmemory/sender.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import os
 import threading
 import time
+import warnings
 from multiprocessing.queues import Queue
 from multiprocessing.shared_memory import SharedMemory
 from multiprocessing.util import Finalize
@@ -33,7 +34,9 @@ class SharedMemorySender:
     Each `put` copies one object into a new shared-memory block that this sender owns
     until a receiver acknowledges it. The sender is thread-safe, but only one process
     may produce: pass it to that process as a `Process` argument before the first
-    `put`. Call `wait_for_all_ack` before `close`, because closing frees unread blocks.
+    `put`. Call `wait_for_all_ack` before closing, because closing frees unread blocks.
+    Leaving a `with` block or exiting the process normally while messages are still
+    unacknowledged emits a `RuntimeWarning`.
     """
 
     def __init__(
@@ -112,8 +115,9 @@ class SharedMemorySender:
     def close(self) -> None:
         """Free unread messages, stop the acknowledgement thread, and close the queues.
 
-        Idempotent. Unread messages are lost, so drain with `wait_for_all_ack` first.
-        Also runs automatically when the producing process exits normally.
+        Idempotent. Unread messages are discarded silently, so drain with
+        `wait_for_all_ack` first. Also runs automatically when the producing process
+        exits normally, warning if that discards messages.
         """
         self._check_process()
         errors: list[Exception] = []
@@ -157,7 +161,12 @@ class SharedMemorySender:
         exc: BaseException | None,
         traceback: TracebackType | None,
     ) -> None:
-        self.close()
+        try:
+            # An exception already signals the failure; only warn on a clean exit.
+            if exc_type is None:
+                self._warn_if_discarding()
+        finally:
+            self.close()
 
     def __getstate__(self) -> dict[str, Any]:
         if self._process_id is not None:
@@ -174,6 +183,26 @@ class SharedMemorySender:
         if self._process_id is not None and self._process_id != os.getpid():
             raise RuntimeError("An initialized sender cannot be used in another process")
 
+    def _close_at_exit(self) -> None:
+        try:
+            self._warn_if_discarding()
+        finally:
+            self.close()
+
+    def _warn_if_discarding(self) -> None:
+        # Explicit close() discards silently; implicit closes make the data loss visible.
+        self._check_process()
+        with self._condition:
+            unacknowledged = 0 if self._closed else len(self._blocks)
+        if unacknowledged:
+            noun = "message" if unacknowledged == 1 else "messages"
+            warnings.warn(
+                f"Closing the sender discards {unacknowledged} unacknowledged {noun} that "
+                "receivers may not have read; call wait_for_all_ack() first",
+                RuntimeWarning,
+                stacklevel=3,
+            )
+
     def _ensure_open(self) -> None:
         if self._closed:
             raise BrokenPipeError("Sender is closed") from self._ack_error
@@ -243,7 +272,7 @@ class SharedMemorySender:
             self._ack_thread = threading.Thread(
                 target=self._process_acknowledgements, name="shared-memory-acks", daemon=True
             )
-            self._finalizer = Finalize(self, self.close, exitpriority=10)
+            self._finalizer = Finalize(self, self._close_at_exit, exitpriority=10)
             self._ack_thread.start()
         except BaseException:
             if self._finalizer is not None:

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0/py_sharedmemory.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-sharedmemory
-Version: 4.0.0
+Version: 4.1.0
 Summary: Fast multiprocessing.Queue alternative that moves large Python objects and NumPy arrays through shared memory
 Author: Tobias Würth
 License-Expression: MIT
@@ -230,6 +230,11 @@ with sender:
         raise TimeoutError("Consumer did not receive everything")
 ```
 
+Closing never blocks. If a `with sender:` block or the producing process ends normally
+while messages are still unacknowledged, they are discarded and the sender emits a
+`RuntimeWarning`, so forgetting to drain does not lose data silently. An explicit
+`close()` or an exception leaving the `with` block discards without a warning.
+
 When both endpoints live in one process, drain before closing either of them. A sentinel
 such as `None` is a simple way to tell consumers that the stream has ended.
 
@@ -249,6 +254,7 @@ copy; other views, such as strided slices, arrive as contiguous copies.
 | `BrokenPipeError` | The endpoint is closed, or the producer's acknowledgement channel failed. |
 | `RuntimeError` | A second process tries to produce, or a used sender is passed to another process. |
 | Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still freed. |
+| `RuntimeWarning` (warning) | A `with sender:` block or the producing process ended normally and discarded unacknowledged messages. |
 
 ## API reference
 
@@ -265,9 +271,9 @@ for the underlying queues; it defaults to `multiprocessing.get_context()`.
 | `put(obj, block=True, timeout=None)` | Copy `obj` into a new shared-memory block and send it. |
 | `put_nowait(obj)` | `put(obj, block=False)`. |
 | `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and freed; `False` on timeout. |
-| `close()` | Free unread messages and release resources. Runs automatically at normal process exit. |
+| `close()` | Free unread messages silently and release resources. Runs automatically at normal process exit. |
 | `closed` | Whether the sender is closed. |
-| `with sender:` | Closes the sender on exit. |
+| `with sender:` | Closes the sender on exit; warns if a clean exit discards unacknowledged messages. |
 
 ### `SharedMemoryReceiver`
 

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/tests/conftest.py

@@ -1,3 +1,4 @@
+import gc
 import multiprocessing as mp
 
 import pytest
@@ -5,6 +6,15 @@ import pytest
 from py_sharedmemory import create_shared_memory_pair
 
 
+@pytest.fixture(autouse=True)
+def collect_garbage():
+    # Queues left in reference cycles finalize their semaphores whenever the collector
+    # runs. On Python 3.12 that raises ReentrantCallError if it interrupts this thread
+    # inside the resource tracker, so collect at a safe point after every test.
+    yield
+    gc.collect()
+
+
 @pytest.fixture
 def make_pair():
     pairs = []

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/tests/helpers.py

@@ -1,4 +1,5 @@
 import multiprocessing.shared_memory as sm
+import warnings
 from queue import Queue
 
 import pytest
@@ -53,7 +54,34 @@ def consume(receiver, count, results):
         results.put(values)
 
 
+PAYLOAD_BYTES = 1_000_000
+
+
+def make_payload(index):
+    """Alternate in-band bytes and out-of-band NumPy buffers with index-specific content."""
+    if index % 2:
+        return bytes([index % 251]) * PAYLOAD_BYTES
+    import numpy as np
+
+    return np.full(PAYLOAD_BYTES // 4, index, dtype=np.int32)
+
+
+def payload_intact(index, payload):
+    if isinstance(payload, bytes):
+        return len(payload) == PAYLOAD_BYTES and payload.count(index % 251) == PAYLOAD_BYTES
+    return payload.shape == (PAYLOAD_BYTES // 4,) and bool((payload == index).all())
+
+
+def verify_payloads(receiver, results):
+    with receiver:
+        while (message := receiver.get(timeout=10)) is not None:
+            index, payload = message
+            results.put((index, payload_intact(index, payload)))
+
+
 def exit_with_unreceived_data(sender, ready_to_exit):
+    # test_process_exit_with_unread_messages_warns covers the exit warning itself.
+    warnings.filterwarnings("ignore", "Closing the sender discards", RuntimeWarning)
     sender.put(bytearray(65_536))
     if not ready_to_exit.wait(10):
         raise TimeoutError("Parent did not read the metadata")
@@ -61,11 +89,15 @@ def exit_with_unreceived_data(sender, ready_to_exit):
 
 
 def abandon_metadata(sender, result_pipe):
-    with sender:
-        for _ in range(1_000):
-            sender.put(b"x")
-        result_pipe.send(list(sender._blocks))
-        result_pipe.close()
+    def abandon():
+        with sender:
+            for _ in range(1_000):
+                sender.put(b"x")
+            result_pipe.send(list(sender._blocks))
+            result_pipe.close()
+
+    with pytest.warns(RuntimeWarning, match="discards 1000 unacknowledged"):
+        abandon()
 
 
 def try_foreign_sender(sender, result_pipe):

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/tests/test_processes.py

@@ -11,8 +11,10 @@ from .helpers import (
     consume,
     exit_with_unreceived_data,
     join_process,
+    make_payload,
     produce,
     try_foreign_sender,
+    verify_payloads,
 )
 
 
@@ -94,6 +96,32 @@ def test_multiple_consumer_processes(make_pair, ctx):
         results.join_thread()
 
 
+def test_large_messages_arrive_intact_across_consumer_processes(make_pair, ctx):
+    sender, receiver = make_pair(4, ctx=ctx)
+    results = ctx.Queue()
+    consumers = [ctx.Process(target=verify_payloads, args=(receiver, results)) for _ in range(3)]
+    for process in consumers:
+        process.start()
+    try:
+        for index in range(60):
+            sender.put((index, make_payload(index)), timeout=10)
+        for _ in consumers:
+            sender.put(None, timeout=10)
+        received = dict(results.get(timeout=15) for _ in range(60))
+        assert sender.wait_for_all_ack(timeout=10)
+        # Every byte of every payload is checked in the consumer that received it.
+        assert received == dict.fromkeys(range(60), True)
+        for process in consumers:
+            join_process(process)
+    finally:
+        for process in consumers:
+            if process.is_alive():
+                process.terminate()
+                process.join(5)
+        results.close()
+        results.join_thread()
+
+
 def test_normal_process_exit_cleans_unreceived_shared_memory(make_pair, ctx):
     sender, receiver = make_pair(1, ctx=ctx)
     ready_to_exit = ctx.Event()
@@ -165,6 +193,26 @@ raise SystemExit(process.exitcode)
 """
 
 
+UNREAD_AT_EXIT = """
+from py_sharedmemory import create_shared_memory_pair
+
+sender, receiver = create_shared_memory_pair(1)
+sender.put(b"never received")
+"""
+
+
+def test_process_exit_with_unread_messages_warns():
+    result = subprocess.run(
+        [sys.executable, "-c", UNREAD_AT_EXIT],
+        cwd=Path(__file__).resolve().parents[1],
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert result.returncode == 0, result.stderr
+    assert "RuntimeWarning: Closing the sender discards 1 unacknowledged message" in result.stderr
+
+
 @pytest.mark.skipif("fork" not in mp.get_all_start_methods(), reason="Requires fork")
 def test_forked_producer_leaves_no_resource_tracker_entries():
     # A fresh interpreter has no resource tracker running before the fork.

{py_sharedmemory-4.0.0 → py_sharedmemory-4.1.0}/tests/test_queue.py

@@ -3,6 +3,7 @@ import os
 import pickle
 import threading
 import time
+import warnings
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import suppress
 from queue import Empty, Full
@@ -269,6 +270,69 @@ def test_context_managers_close_unused_endpoints(make_pair):
     assert receiver.closed
 
 
+def send_then_leave(sender, count, *, fail=False):
+    """Leave a `with sender:` block without draining; return the abandoned block names."""
+    with sender:
+        for index in range(count):
+            sender.put(index)
+        if fail:
+            raise ValueError("producer failed")
+        return list(sender._blocks)
+
+
+@pytest.mark.parametrize(
+    ("count", "text"),
+    [(1, "1 unacknowledged message that"), (2, "2 unacknowledged messages that")],
+)
+def test_clean_with_exit_warns_about_discarded_messages(make_pair, count, text):
+    sender, _ = make_pair(0)
+    with pytest.warns(RuntimeWarning, match=f"discards {text}") as caught:
+        names = send_then_leave(sender, count)
+    assert caught[0].filename == __file__  # Points at the user's `with` statement
+    assert sender.closed
+    for name in names:
+        assert_unlinked(name)
+
+
+@pytest.mark.parametrize("ending", ["drained", "exception", "close"])
+def test_drained_failed_and_explicit_closes_do_not_warn(make_pair, ending):
+    sender, receiver = make_pair()
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        if ending == "drained":
+            with sender:
+                sender.put(1)
+                assert receiver.get(timeout=3) == 1
+                assert sender.wait_for_all_ack(timeout=3)
+        elif ending == "exception":
+            with pytest.raises(ValueError, match="producer failed"):
+                send_then_leave(sender, 1, fail=True)
+        else:
+            sender.put(1)
+            sender.close()
+    assert sender.closed
+
+
+def test_process_exit_cleanup_warns_about_discarded_messages(make_pair):
+    sender, _ = make_pair()
+    sender.put(1)
+    with pytest.warns(RuntimeWarning, match="discards 1 unacknowledged message"):
+        sender._finalizer()
+    assert sender.closed
+
+
+def test_warning_as_error_still_closes_the_sender(make_pair):
+    sender, _ = make_pair()
+    sender.put(1)
+    name = next(iter(sender._blocks))
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        with pytest.raises(RuntimeWarning, match="unacknowledged"), sender:
+            pass
+    assert sender.closed
+    assert_unlinked(name)
+
+
 def test_ack_channel_failure_closes_without_joining_its_own_thread():
     class BrokenAckQueue(LocalQueue):
         def get(self, **kwargs):