nwp500-python 8.1.2__tar.gz → 8.1.3__tar.gz

{nwp500_python-8.1.2 → nwp500_python-8.1.3}/CHANGELOG.rst

@@ -5,9 +5,26 @@ Changelog
 Unreleased
 ==========
 
-Version 8.1.2 (2026-05-25)
+Version 8.1.3 (2026-06-15)
 ==========================
 
+Bug Fixes
+---------
+- **Fix MQTT reconnection storm caused by non-thread-safe Task.cancel()**: The
+  ``on_connection_resumed`` callback is invoked from an AWS IoT SDK background
+  thread.  It was calling ``asyncio.Task.cancel()`` directly on that thread, which
+  is not thread-safe.  When the event loop was busy at the moment of cancellation
+  (e.g. the sleeping task's timer callback had already been enqueued), the
+  cancellation was silently dropped.  The stale ``_reconnect_with_backoff`` task
+  would then complete its sleep, call ``_reconnect_func``, and tear down an
+  otherwise healthy connection — restarting the entire disconnect → reconnect →
+  ``AWS_ERROR_MQTT_UNEXPECTED_HANGUP`` cycle.  Fixed by replacing the direct
+  ``task.cancel()`` call with a ``_cancel_pending_reconnect()`` coroutine scheduled
+  via ``_schedule_coroutine``, so the cancellation runs on the event loop where
+  asyncio operations are safe.  The method also uses an identity check before
+  clearing ``_reconnect_task`` to avoid wiping a newer task created during the
+  await, and clears stale references to already-done tasks.
+
 Version 8.1.0 (2026-05-16)
 ==========================
 

{nwp500_python-8.1.2/src/nwp500_python.egg-info → nwp500_python-8.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nwp500-python
-Version: 8.1.2
+Version: 8.1.3
 Summary: A library for controlling Navien NWP500 Water Heaters via NaviLink
 Home-page: https://github.com/eman/nwp500-python
 Author: Emmanuel Levijarvi
@@ -89,8 +89,6 @@ Quick Example
 Documentation
 =============
 
-The documentation follows the `Diátaxis <https://diataxis.fr/>`_ framework:
-
 * `Tutorials <https://nwp500-python.readthedocs.io/en/latest/tutorials/getting-started.html>`_: Start here if you're new to the library.
 * `How-to Guides <https://nwp500-python.readthedocs.io/en/latest/how-to/index.html>`_: Practical step-by-step recipes for specific tasks.
 * `Reference <https://nwp500-python.readthedocs.io/en/latest/reference/index.html>`_: Technical descriptions of the API, models, and protocol.

{nwp500_python-8.1.2 → nwp500_python-8.1.3}/README.rst

@@ -47,8 +47,6 @@ Quick Example
 Documentation
 =============
 
-The documentation follows the `Diátaxis <https://diataxis.fr/>`_ framework:
-
 * `Tutorials <https://nwp500-python.readthedocs.io/en/latest/tutorials/getting-started.html>`_: Start here if you're new to the library.
 * `How-to Guides <https://nwp500-python.readthedocs.io/en/latest/how-to/index.html>`_: Practical step-by-step recipes for specific tasks.
 * `Reference <https://nwp500-python.readthedocs.io/en/latest/reference/index.html>`_: Technical descriptions of the API, models, and protocol.

{nwp500_python-8.1.2 → nwp500_python-8.1.3}/src/nwp500/mqtt/reconnection.py

@@ -128,9 +128,42 @@ class MqttReconnectionHandler:
         # Reset reconnection attempts on successful connection
         self._reconnect_attempts = 0
 
-        # Cancel any pending reconnection task
-        if self._reconnect_task and not self._reconnect_task.done():
-            self._reconnect_task.cancel()
+        # Schedule cancellation of any pending reconnect task on the event loop.
+        # This method is called from an AWS SDK background thread; asyncio's
+        # Task.cancel() is NOT thread-safe when invoked directly from a
+        # non-event-loop thread.  If the event loop is busy (e.g. the sleeping
+        # task's timer callback was already queued) the cancellation can be
+        # silently dropped, leaving the stale _reconnect_with_backoff loop
+        # alive.  That loop then completes its sleep and calls _reconnect_func,
+        # tearing down a perfectly healthy connection and restarting the
+        # disconnect/reconnect cycle.
+        self._schedule_coroutine(self._cancel_pending_reconnect())
+
+    async def _cancel_pending_reconnect(self) -> None:
+        """Cancel any pending reconnect task.
+
+        Must be called on the event loop (via _schedule_coroutine) so that
+        asyncio Task operations are thread-safe.
+
+        Uses an identity check before clearing _reconnect_task to avoid
+        accidentally wiping a new task that was created while the cancelled
+        task was being awaited.  Also clears stale references to already-done
+        tasks so the handler never holds on to finished task objects.
+        """
+        task = self._reconnect_task
+        if task is None:
+            return
+        if task.done():
+            # Clear stale reference to an already-finished task.
+            if self._reconnect_task is task:
+                self._reconnect_task = None
+            return
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+        # Only clear the reference if it still points to the same task.
+        # A new reconnect task may have been created while we were awaiting.
+        if self._reconnect_task is task:
             self._reconnect_task = None
 
     async def _start_reconnect_task(self) -> None:
@@ -142,7 +175,7 @@ class MqttReconnectionHandler:
 
         The is_connected guard is re-checked here because this coroutine may
         be queued via run_coroutine_threadsafe and run after the connection
-        has already been restored (e.g. by on_connection_resumed cancelling
+        has already been restored (e.g. by _cancel_pending_reconnect clearing
         _reconnect_task), in which case starting a new backoff loop would
         incorrectly tear down a healthy connection.
         """

{nwp500_python-8.1.2 → nwp500_python-8.1.3/src/nwp500_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nwp500-python
-Version: 8.1.2
+Version: 8.1.3
 Summary: A library for controlling Navien NWP500 Water Heaters via NaviLink
 Home-page: https://github.com/eman/nwp500-python
 Author: Emmanuel Levijarvi
@@ -89,8 +89,6 @@ Quick Example
 Documentation
 =============
 
-The documentation follows the `Diátaxis <https://diataxis.fr/>`_ framework:
-
 * `Tutorials <https://nwp500-python.readthedocs.io/en/latest/tutorials/getting-started.html>`_: Start here if you're new to the library.
 * `How-to Guides <https://nwp500-python.readthedocs.io/en/latest/how-to/index.html>`_: Practical step-by-step recipes for specific tasks.
 * `Reference <https://nwp500-python.readthedocs.io/en/latest/reference/index.html>`_: Technical descriptions of the API, models, and protocol.

{nwp500_python-8.1.2 → nwp500_python-8.1.3}/tests/test_mqtt_reconnection_storm.py

@@ -1,7 +1,7 @@
 """
 Tests for the MQTT reconnection storm fix.
 
-Two bugs were fixed:
+Three bugs were fixed:
 
 Bug 1 — Stale interruption events fire after a resume clears _reconnect_task.
   The AWS SDK fires on_connection_interrupted callbacks from background threads
@@ -21,6 +21,18 @@ Bug 2 — Closing the old connection inside _active_reconnect / _deep_reconnect
   Fix: _actively_reconnecting flag suppresses the reconnection-handler
   delegation in _on_connection_interrupted_internal while the intentional
   teardown is in progress.
+
+Bug 3 — on_connection_resumed calls Task.cancel() directly from an AWS SDK
+  background thread.  asyncio.Task.cancel() is NOT thread-safe; when the event
+  loop is busy (e.g. the sleeping task's timer callback was already enqueued)
+  the cancellation can be silently dropped.  The stale _reconnect_with_backoff
+  task then completes its sleep, calls _reconnect_func, and tears down an
+  otherwise healthy connection, restarting the entire
+  disconnect/reconnect cycle.
+
+  Fix: on_connection_resumed schedules _cancel_pending_reconnect() via
+  _schedule_coroutine so the cancellation runs on the event loop where asyncio
+  Task operations are safe.
 """
 
 from __future__ import annotations
@@ -177,10 +189,11 @@ class TestReconnectionHandlerIsConnectedGuard:
 
         1. Connection drops  → on_connection_interrupted schedules
            _start_reconnect_task (coroutine A queued but not yet run).
-        2. Connection resumes → on_connection_resumed cancels task, sets
-           _reconnect_task = None.
-        3. Coroutine A finally runs – without the fix it would see
-           _reconnect_task=None and create a new backoff loop.
+        2. Connection resumes → on_connection_resumed schedules
+           _cancel_pending_reconnect (no task to cancel yet since A hasn't run).
+        3. Both queued coroutines finally run – without the is_connected guard
+           coroutine A would see _reconnect_task=None and create a new backoff
+           loop even though the client is now healthy.
         """
         state = {"connected": False}
         config = MqttConnectionConfig(auto_reconnect=True)
@@ -200,13 +213,16 @@ class TestReconnectionHandlerIsConnectedGuard:
         handler.on_connection_interrupted(Exception("dropped"))
         assert len(scheduled) == 1
 
-        # Step 2: connection resumes before the scheduled coroutine runs
+        # Step 2: connection resumes before the scheduled coroutine runs.
+        # on_connection_resumed now schedules _cancel_pending_reconnect rather
+        # than calling task.cancel() directly (Bug 3 fix).
         state["connected"] = True
         handler.on_connection_resumed(return_code=0, session_present=False)
-        assert handler._reconnect_task is None
+        assert handler._reconnect_task is None  # no task was ever created
 
-        # Step 3: the stale coroutine from step 1 runs now
-        await scheduled[0]
+        # Step 3: run all scheduled coroutines
+        # (_start_reconnect_task + _cancel_pending_reconnect)
+        await asyncio.gather(*scheduled, return_exceptions=True)
 
         # With the fix, no new task must have been created
         assert handler._reconnect_task is None
@@ -232,6 +248,136 @@ class TestReconnectionHandlerIsConnectedGuard:
         await asyncio.gather(handler._reconnect_task, return_exceptions=True)
 
 
+# ---------------------------------------------------------------------------
+# Bug 3: on_connection_resumed must cancel via the event loop, not directly
+# ---------------------------------------------------------------------------
+
+
+class TestThreadSafeTaskCancellation:
+    """Bug 3 – on_connection_resumed must not call Task.cancel() from a thread.
+
+    asyncio.Task.cancel() is NOT thread-safe.  When called from an AWS SDK
+    background thread, the cancellation can be silently dropped if the event
+    loop is busy (e.g. the sleep timer fires at the same moment).  The stale
+    task then triggers a spurious reconnection.
+
+    Fix: on_connection_resumed schedules _cancel_pending_reconnect() via
+    _schedule_coroutine so the cancellation happens on the event loop.
+    """
+
+    @pytest.mark.asyncio(loop_scope="function")
+    async def test_on_connection_resumed_schedules_cancel_not_direct_call(
+        self,
+    ):
+        """
+        on_connection_resumed must schedule _cancel_pending_reconnect via
+        _schedule_coroutine rather than calling task.cancel() directly.
+        """
+        handler, scheduled = _make_handler(connected=False)
+
+        # Let a reconnect task start (sleeping in backoff)
+        handler.on_connection_interrupted(Exception("dropped"))
+        await asyncio.gather(*scheduled, return_exceptions=True)
+        scheduled.clear()
+
+        assert handler._reconnect_task is not None
+        assert not handler._reconnect_task.done()
+
+        # Simulate connection resuming from a background thread
+        handler.on_connection_resumed(return_code=0, session_present=True)
+
+        # _cancel_pending_reconnect must have been *scheduled*, not run yet
+        assert len(scheduled) == 1
+        # No direct Task.cancel() was called from the background thread:
+        # task.cancelling() == 0 proves no cancellation request is pending
+        # before the event loop runs _cancel_pending_reconnect.
+        assert handler._reconnect_task is not None
+        assert handler._reconnect_task.cancelling() == 0
+
+        # Now let the event loop process the cancellation
+        await asyncio.gather(*scheduled, return_exceptions=True)
+
+        # After the event-loop cancellation, the task must be cleared
+        assert handler._reconnect_task is None
+
+    @pytest.mark.asyncio(loop_scope="function")
+    async def test_cancel_pending_reconnect_is_idempotent_with_no_task(self):
+        """_cancel_pending_reconnect is a no-op when _reconnect_task is None."""
+        handler, _ = _make_handler(connected=True)
+        assert handler._reconnect_task is None
+
+        # Should not raise
+        await handler._cancel_pending_reconnect()
+
+        assert handler._reconnect_task is None
+
+    @pytest.mark.asyncio(loop_scope="function")
+    async def test_cancel_pending_reconnect_clears_completed_task(self):
+        """_cancel_pending_reconnect clears a stale reference to a done task."""
+        handler, scheduled = _make_handler(connected=False)
+
+        handler.on_connection_interrupted(Exception("dropped"))
+        await asyncio.gather(*scheduled, return_exceptions=True)
+        scheduled.clear()
+
+        task = handler._reconnect_task
+        assert task is not None
+
+        # Cancel and drain the task manually, leaving a stale reference
+        task.cancel()
+        await asyncio.gather(task, return_exceptions=True)
+        assert task.done()
+        assert handler._reconnect_task is task  # stale reference still held
+
+        # _cancel_pending_reconnect must clear the stale reference
+        await handler._cancel_pending_reconnect()
+        assert handler._reconnect_task is None
+
+    @pytest.mark.asyncio(loop_scope="function")
+    async def test_resumed_then_interrupted_creates_new_task(self):
+        """
+        After resume cancels an existing task, a subsequent genuine drop must
+        still be able to start a fresh reconnect task.
+        """
+        state = {"connected": False}
+        config = MqttConnectionConfig(auto_reconnect=True)
+        scheduled = []
+
+        handler = MqttReconnectionHandler(
+            config=config,
+            is_connected_func=lambda: state["connected"],
+            schedule_coroutine_func=lambda coro: scheduled.append(
+                asyncio.ensure_future(coro)
+            ),
+            reconnect_func=AsyncMock(),
+        )
+        handler.enable()
+
+        # Connection drops → backoff task starts
+        handler.on_connection_interrupted(Exception("first drop"))
+        await asyncio.gather(*scheduled, return_exceptions=True)
+        scheduled.clear()
+        assert handler._reconnect_task is not None
+
+        # Connection resumes → cancel scheduled on loop
+        state["connected"] = True
+        handler.on_connection_resumed(return_code=0, session_present=True)
+        await asyncio.gather(*scheduled, return_exceptions=True)
+        scheduled.clear()
+        assert handler._reconnect_task is None
+
+        # Connection drops again (genuine)
+        state["connected"] = False
+        handler.on_connection_interrupted(Exception("second drop"))
+        await asyncio.gather(*scheduled, return_exceptions=True)
+        scheduled.clear()
+
+        # A new reconnect task must have been created
+        assert handler._reconnect_task is not None
+        handler._reconnect_task.cancel()
+        await asyncio.gather(handler._reconnect_task, return_exceptions=True)
+
+
 # ---------------------------------------------------------------------------
 # Bug 2: _actively_reconnecting suppresses spurious interrupt callbacks
 # ---------------------------------------------------------------------------