span-panel-api 2.6.2__tar.gz → 2.6.4__tar.gz

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/.github/workflows/dependabot-auto-approve.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v3.0.0
+        uses: dependabot/fetch-metadata@v3.1.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/.github/workflows/dependabot-auto-merge.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v3.0.0
+        uses: dependabot/fetch-metadata@v3.1.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.6.4] - 05/2026
+
+### Fixed
+
+- **MQTT reconnect now self-heals after persistent failure** — `AsyncMqttBridge._reconnect_loop` rebuilds the paho client from scratch (re-fetching the panel CA, constructing a fresh client, resetting the Homie accumulator) after
+  `MQTT_FULL_REBUILD_AFTER_FAILURES` (3) consecutive failures, or immediately on any `ssl.SSLError`. The previous behavior pinned the panel's CA certificate into the paho client once at `connect()` time and re-used it across all reconnect attempts; if the
+  panel rotated its private CA — most plausibly during a firmware upgrade — every subsequent reconnect raised `ssl.SSLCertVerificationError` (caught by the broad `OSError` clause and silently retried) and the bridge could not recover without a config-entry
+  reload. The rebuild mirrors what a manual reload does without going through HA's `config_entry` teardown, so entities stay registered and the integration's grace-period logic continues to apply unchanged. The threshold-cadence design (counter reset on
+  every rebuild attempt, success or fail) keeps the recovery path active throughout extended outages — multi-day disconnections recover whenever the panel becomes usable again, including if the CA rotates a second time mid-outage. See
+  `SpanPanel_Docs/span-panel-api/2026-05-17-mqtt-ca-refresh-on-reconnect-design.md` for the full design.
+
+### Added
+
+- **`AsyncMqttBridge._rebuild_client()`** — internal recovery method invoked by the reconnect loop on persistent failure. Re-fetches the panel CA via `download_ca_cert()`, builds a fresh paho client via the new `_make_paho_client()` factory, fires the
+  optional pre-rebuild callback so consumers can reset their own state, tears down the old client, and submits the initial connect via the executor. Restores the previous client on any failure.
+- **`AsyncMqttBridge.set_pre_rebuild_callback()`** — internal API for `SpanMqttClient` to register a hook that fires before each rebuild. Used to reset the Homie accumulator so retained messages on the new subscription start from a clean slate.
+- **`MQTT_FULL_REBUILD_AFTER_FAILURES`** constant in `mqtt/const.py`.
+
+### Changed
+
+- **`SpanPanelAPIError` now in the bridge's CA-fetch exception list** — a `download_ca_cert()` failure during rebuild (e.g. panel returns HTTP 502 mid-outage) is caught, logged at WARNING, and the loop continues retrying with the previous client instead of
+  letting the reconnect task die.
+
 ## [2.6.2] - 04/2026
 
 ### Changed

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: span-panel-api
-Version: 2.6.2
+Version: 2.6.4
 Summary: A client library for SPAN Panel API
 Project-URL: Homepage, https://github.com/SpanPanel/span-panel-api
 Project-URL: Issues, https://github.com/SpanPanel/span-panel-api/issues

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "span-panel-api"
-version = "2.6.2"
+version = "2.6.4"
 description = "A client library for SPAN Panel API"
 authors = [
     {name = "SpanPanel"}
@@ -129,7 +129,6 @@ omit = [
     "*/tests/*",
     "*/.venv/*",
     "*/venv/*",
-    "src/span_panel_api/mqtt/connection.py",
 ]
 
 [tool.coverage.report]

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/src/span_panel_api/factory.py

@@ -24,6 +24,7 @@ async def create_span_client(
     passphrase: str | None = None,
     mqtt_config: MqttClientConfig | None = None,
     serial_number: str | None = None,
+    port: int = 80,
 ) -> SpanMqttClient:
     """Create a SPAN Panel MQTT client.
 
@@ -32,6 +33,7 @@ async def create_span_client(
         passphrase: Panel passphrase for v2 registration.
         mqtt_config: Pre-built MQTT broker configuration.
         serial_number: Panel serial number (extracted from detection/registration if omitted).
+        port: HTTP port of the panel bootstrap API used for registration and detection.
 
     Returns:
         A connected-ready SpanMqttClient instance.
@@ -45,7 +47,7 @@ async def create_span_client(
     if mqtt_config is None:
         if passphrase is None:
             raise SpanPanelAuthError("Neither mqtt_config nor passphrase provided")
-        auth_response = await register_v2(host, _V2_CLIENT_NAME, passphrase)
+        auth_response = await register_v2(host, _V2_CLIENT_NAME, passphrase, port=port)
         mqtt_config = MqttClientConfig(
             broker_host=auth_response.ebus_broker_host,
             username=auth_response.ebus_broker_username,
@@ -59,11 +61,13 @@ async def create_span_client(
 
     if serial_number is None:
         # Try to detect from panel status
-        result = await detect_api_version(host)
+        result = await detect_api_version(host, port=port)
         if result.status_info is not None:
             serial_number = result.status_info.serial_number
 
     if serial_number is None:
         raise SpanPanelAuthError("serial_number is required for MQTT transport but could not be determined")
 
-    return SpanMqttClient(host, serial_number, mqtt_config)
+    client = SpanMqttClient(host, serial_number, mqtt_config, panel_http_port=port)
+    await client.connect()
+    return client

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/src/span_panel_api/mqtt/client.py

@@ -63,6 +63,10 @@ class SpanMqttClient:
         self._field_metadata: dict[str, FieldMetadata] | None = None
         self._schema_hash: str | None = None
         self._previous_schema_types: HomieSchemaTypes | None = None
+        # Cached at connect() so the pre-rebuild hook can reconstruct the
+        # Homie accumulator with the same panel size after a transport-level
+        # rebuild. Schema cannot change within a session, so caching is safe.
+        self._panel_size: int | None = None
 
     def _require_homie(self) -> HomieDeviceConsumer:
         """Return the HomieDeviceConsumer, raising if not yet connected."""
@@ -115,6 +119,7 @@ class SpanMqttClient:
 
         # Fetch schema to determine panel size and build field metadata
         schema = await get_homie_schema(self._host, port=self._panel_http_port)
+        self._panel_size = schema.panel_size
         self._accumulator = HomiePropertyAccumulator(self._serial_number)
         self._homie = HomieDeviceConsumer(self._accumulator, schema.panel_size)
 
@@ -157,6 +162,11 @@ class SpanMqttClient:
         # Wire message handler
         self._bridge.set_message_callback(self._on_message)
         self._bridge.set_connection_callback(self._on_connection_change)
+        # Pre-rebuild hook: reset Homie accumulator before the bridge swaps
+        # paho clients, so retained messages on the new subscription start
+        # from a clean slate (no stale `$state=disconnected` cached from
+        # the original outage).
+        self._bridge.set_pre_rebuild_callback(self._on_pre_rebuild)
 
         # Connect to broker
         _LOGGER.debug("MQTT: Connecting to broker...")
@@ -369,6 +379,30 @@ class SpanMqttClient:
             except Exception:  # pylint: disable=broad-exception-caught
                 _LOGGER.warning("Connection callback raised", exc_info=True)
 
+    def _on_pre_rebuild(self) -> None:
+        """Reset Homie accumulator state before the bridge rebuilds its paho client.
+
+        Called synchronously from the bridge's `_rebuild_client` before the
+        old paho client is torn down and the new one is wired up. Discards
+        any stale `$state=disconnected` cached during the outage so the
+        new subscription's retained messages repopulate from a clean slate.
+
+        Schema-derived state (`_field_metadata`, `_schema_hash`,
+        `_previous_schema_types`) is intentionally preserved — the Homie
+        schema cannot change within a session, so the cache remains valid
+        and a refetch would just add cost. If the panel reboots and the
+        schema actually changed, the existing drift-detection log fires on
+        the next session's `connect()`.
+        """
+        if self._panel_size is None:
+            # Pre-rebuild fired before connect() cached the panel size.
+            # Treat as a no-op — there is no accumulator state to reset
+            # because connect() never completed.
+            return
+        _LOGGER.debug("Pre-rebuild — resetting Homie accumulator")
+        self._accumulator = HomiePropertyAccumulator(self._serial_number)
+        self._homie = HomieDeviceConsumer(self._accumulator, self._panel_size)
+
     async def _wait_for_circuit_names(self, timeout: float) -> None:
         """Wait for all circuit-like nodes to have a ``name`` property.
 

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/src/span_panel_api/mqtt/connection.py

@@ -23,10 +23,11 @@ from paho.mqtt.properties import Properties
 from paho.mqtt.reasoncodes import ReasonCode
 
 from ..auth import download_ca_cert
-from ..exceptions import SpanPanelConnectionError, SpanPanelTimeoutError
+from ..exceptions import SpanPanelAPIError, SpanPanelConnectionError, SpanPanelTimeoutError
 from .async_client import AsyncMQTTClient
 from .const import (
     MQTT_CONNECT_TIMEOUT_S,
+    MQTT_FULL_REBUILD_AFTER_FAILURES,
     MQTT_KEEPALIVE_S,
     MQTT_RECONNECT_BACKOFF_MULTIPLIER,
     MQTT_RECONNECT_MAX_DELAY_S,
@@ -97,6 +98,7 @@ class AsyncMqttBridge:
 
         self._message_callback: Callable[[str, str], None] | None = None
         self._connection_callback: Callable[[bool], None] | None = None
+        self._pre_rebuild_callback: Callable[[], None] | None = None
 
     def is_connected(self) -> bool:
         """Return whether the MQTT client is currently connected."""
@@ -110,6 +112,44 @@ class AsyncMqttBridge:
         """Set callback for connection state changes: callback(is_connected)."""
         self._connection_callback = callback
 
+    def set_pre_rebuild_callback(self, callback: Callable[[], None]) -> None:
+        """Set callback invoked just before the bridge rebuilds its paho client.
+
+        Used by SpanMqttClient to reset its Homie accumulator so any stale
+        in-memory state (e.g. cached `$state=disconnected`) is discarded
+        before the new client subscribes and retained messages flow in.
+
+        Callback runs synchronously inside `_rebuild_client` before the old
+        paho client is torn down. Exceptions are caught and logged so a
+        misbehaving subscriber cannot prevent the rebuild.
+        """
+        self._pre_rebuild_callback = callback
+
+    def _make_paho_client(self, ssl_context: ssl.SSLContext | None) -> AsyncMQTTClient:
+        """Build and wire a fresh paho client.
+
+        Shared by connect() (initial connect) and _rebuild_client() (in-loop
+        rebuild). Keeps the callback wiring in one place so a rebuild is
+        provably symmetric with initial connect.
+        """
+        client = AsyncMQTTClient(
+            callback_api_version=CallbackAPIVersion.VERSION2,
+            transport=self._transport,
+            reconnect_on_failure=False,
+        )
+        client.setup()
+        client.username_pw_set(self._username, self._password)
+        # Wire socket callbacks (async versions by default)
+        client.on_socket_close = self._async_on_socket_close
+        client.on_socket_unregister_write = self._async_on_socket_unregister_write
+        # Wire MQTT callbacks (run directly on event loop — no thread dispatch)
+        client.on_connect = self._on_connect
+        client.on_disconnect = self._on_disconnect
+        client.on_message = self._on_message
+        if ssl_context is not None:
+            client.tls_set_context(ssl_context)
+        return client
+
     async def connect(self) -> None:
         """Connect to the MQTT broker.
 
@@ -142,26 +182,7 @@ class AsyncMqttBridge:
             except (ssl.SSLError, ValueError) as exc:
                 raise SpanPanelConnectionError(f"Failed to build SSL context for {self._panel_host}") from exc
 
-        self._client = AsyncMQTTClient(
-            callback_api_version=CallbackAPIVersion.VERSION2,
-            transport=self._transport,
-            reconnect_on_failure=False,
-        )
-        self._client.setup()
-
-        self._client.username_pw_set(self._username, self._password)
-
-        # Wire socket callbacks (async versions by default)
-        self._client.on_socket_close = self._async_on_socket_close
-        self._client.on_socket_unregister_write = self._async_on_socket_unregister_write
-
-        # Wire MQTT callbacks (run directly on event loop — no thread dispatch)
-        self._client.on_connect = self._on_connect
-        self._client.on_disconnect = self._on_disconnect
-        self._client.on_message = self._on_message
-
-        if ssl_context is not None:
-            self._client.tls_set_context(ssl_context)
+        self._client = self._make_paho_client(ssl_context)
 
         # Connect in executor (blocking: DNS, TCP, TLS handshake).
         # During executor connect, socket callbacks bridge to the event
@@ -395,9 +416,132 @@ class AsyncMqttBridge:
 
     # -- Reconnection -------------------------------------------------------
 
+    async def _rebuild_client(self) -> bool:
+        """Tear down the paho client and rebuild it from scratch.
+
+        Replicates what a manual integration reload does without going
+        through HA's config_entry teardown. Re-fetches the panel CA,
+        builds a fresh paho client with the same callbacks, fires the
+        pre-rebuild callback so SpanMqttClient can reset its accumulator,
+        and submits an initial connect via the executor.
+
+        Returns True when the new client was built and the initial connect
+        was successfully submitted. Returns False on any failure (panel
+        unreachable, CA endpoint down, executor connect raised) — the
+        previous client is left in place and the reconnect loop continues
+        retrying with it.
+
+        Recovery target: CA rotation (firmware upgrade), stale paho client
+        internal state, stuck Homie accumulator. See the design doc at
+        SpanPanel_Docs/span-panel-api/2026-05-17-mqtt-ca-refresh-on-reconnect-design.md.
+        """
+        if self._loop is None:
+            return False
+
+        old_client = self._client
+
+        # Fetch fresh CA (TLS bridges only). Failure is non-fatal — old
+        # client stays in place and the loop retries on the next tick.
+        ssl_context: ssl.SSLContext | None = None
+        if self._use_tls:
+            try:
+                ca_pem = await download_ca_cert(self._panel_host, port=self._panel_http_port)
+                ssl_context = _build_ssl_context(ca_pem)
+            except (
+                OSError,
+                SpanPanelConnectionError,
+                SpanPanelTimeoutError,
+                SpanPanelAPIError,
+                ssl.SSLError,
+                ValueError,
+            ) as exc:
+                _LOGGER.warning("Client rebuild — CA fetch failed: %s", exc)
+                return False
+
+        # Fire pre-rebuild hook before we touch any state. SpanMqttClient
+        # uses this to discard its stale Homie accumulator so retained
+        # messages on the new subscription start from a clean slate.
+        if self._pre_rebuild_callback is not None:
+            try:
+                self._pre_rebuild_callback()
+            except Exception:  # pylint: disable=broad-exception-caught
+                _LOGGER.warning("Pre-rebuild callback raised", exc_info=True)
+
+        # Everything past this point is wrapped in a broad catch so that
+        # unexpected failures (paho construction errors, etc.) cannot kill
+        # the reconnect task. The whole point of self-heal is that the
+        # loop survives — we never want the recovery path itself to be a
+        # source of unrecoverable failure.
+        try:
+            # Best-effort teardown of the old paho client. paho's disconnect()
+            # is synchronous and only severs the socket; the object itself is
+            # no longer used.
+            if old_client is not None:
+                try:
+                    old_client.disconnect()
+                except Exception:  # pylint: disable=broad-exception-caught
+                    _LOGGER.debug("Old paho client disconnect raised", exc_info=True)
+
+            # Build fresh client and assign it BEFORE the executor await so
+            # that a CONNACK arriving during the await sees the right client.
+            # Without this, the _on_connect → re-subscribe path would route
+            # through self._client which would still be the (disconnected)
+            # old_client, and the new client's subscription would never run.
+            new_client = self._make_paho_client(ssl_context)
+            new_client.on_socket_open = self._on_socket_open_sync
+            new_client.on_socket_register_write = self._on_socket_register_write_sync
+            self._client = new_client
+
+            def _blocking_connect() -> None:
+                new_client.connect(
+                    host=self._host,
+                    port=self._port,
+                    keepalive=MQTT_KEEPALIVE_S,
+                )
+
+            try:
+                await self._loop.run_in_executor(None, _blocking_connect)
+            except asyncio.CancelledError:
+                # Bridge teardown or _on_connect cancelled us mid-rebuild.
+                # Restore the previous client reference so post-teardown
+                # state stays consistent, then re-raise — CancelledError
+                # must propagate to keep the loop's cancel semantics intact.
+                self._client = old_client
+                raise
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                _LOGGER.warning("Client rebuild — initial connect failed: %s", exc)
+                # Restore the previous client so the loop keeps retrying
+                # with what it had. The new client's socket was never opened.
+                self._client = old_client
+                return False
+            finally:
+                new_client.on_socket_open = self._async_on_socket_open
+                new_client.on_socket_register_write = self._async_on_socket_register_write
+
+            _LOGGER.info("MQTT client rebuilt for reconnect (TLS=%s)", self._use_tls)
+            return True
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            # _make_paho_client raised, or some other unforeseen failure
+            # after the CA was fetched. Reconnect loop MUST survive — log
+            # with traceback for triage and leave whatever client reference
+            # is current in place. CancelledError is BaseException in 3.8+
+            # so it bypasses this clause and propagates naturally.
+            _LOGGER.warning("Client rebuild — unexpected error: %s", exc, exc_info=True)
+            return False
+
     async def _reconnect_loop(self) -> None:
-        """Reconnect with exponential backoff."""
+        """Reconnect with exponential backoff.
+
+        Every MQTT_FULL_REBUILD_AFTER_FAILURES consecutive non-SSL failures
+        (or on any ssl.SSLError), rebuild the paho client from scratch —
+        re-fetching the panel CA and resetting any stale in-memory state.
+        Mirrors what a manual integration reload does without going through
+        HA's config_entry teardown. The counter resets after every rebuild
+        attempt (success or fail) and on `_connected == True`, so the
+        cadence holds throughout extended outages.
+        """
         delay = MQTT_RECONNECT_MIN_DELAY_S
+        failures_since_rebuild_attempt = 0
         while self._should_reconnect:
             if not self._connected and self._client is not None:
                 try:
@@ -407,22 +551,37 @@ class AsyncMqttBridge:
                     self._client.on_socket_open = self._on_socket_open_sync
                     self._client.on_socket_register_write = self._on_socket_register_write_sync
                     await self._loop.run_in_executor(None, self._client.reconnect)
+                except ssl.SSLError as exc:
+                    # TLS verification failure — most likely a CA rotation
+                    # (firmware upgrade). ssl.SSLError must be caught before
+                    # OSError because it is an OSError subclass.
+                    _LOGGER.warning("Reconnect TLS failure (%s), rebuilding client", exc)
+                    await self._rebuild_client()
+                    failures_since_rebuild_attempt = 0
                 except (OSError, TimeoutError) as exc:
-                    # Expected transient failures — panel offline, DNS miss, socket
-                    # timeout, refused connection. The exception type and errno are
-                    # the full diagnostic; paho/stdlib stack frames add no signal.
-                    # ssl.SSLError is an OSError subclass and falls in here too;
-                    # SSL misconfiguration would have failed at setup, so a
-                    # reconnect-time SSL error is handled as a transient failure.
+                    # Expected transient failures — panel offline, DNS miss,
+                    # socket timeout, refused connection. paho also wraps
+                    # some TLS handshake errors as generic OSError on the
+                    # executor connect path; the rebuild after threshold
+                    # catches those.
+                    failures_since_rebuild_attempt += 1
                     _LOGGER.warning("Reconnect failed (%s), retrying in %ss", exc, delay)
+                    if failures_since_rebuild_attempt >= MQTT_FULL_REBUILD_AFTER_FAILURES:
+                        await self._rebuild_client()
+                        failures_since_rebuild_attempt = 0
                 except Exception:  # pylint: disable=broad-exception-caught
                     # Unknown territory — keep the traceback so support tickets
-                    # are actionable. Never let the reconnect loop die.
+                    # are actionable. Never let the reconnect loop die. No
+                    # rebuild here — unknown errors should not be masked
+                    # behind a recovery action whose effect we cannot predict.
+                    failures_since_rebuild_attempt += 1
                     _LOGGER.warning("Reconnect failed, retrying in %ss", delay, exc_info=True)
                 finally:
                     if self._client is not None:
                         self._client.on_socket_open = self._async_on_socket_open
                         self._client.on_socket_register_write = self._async_on_socket_register_write
+            else:
+                failures_since_rebuild_attempt = 0
             await asyncio.sleep(delay)
             delay = min(
                 delay * MQTT_RECONNECT_BACKOFF_MULTIPLIER,

{span_panel_api-2.6.2 → span_panel_api-2.6.4}/src/span_panel_api/mqtt/const.py

@@ -48,6 +48,12 @@ MQTT_RECONNECT_MIN_DELAY_S = 1.0
 MQTT_RECONNECT_MAX_DELAY_S = 60.0
 MQTT_RECONNECT_BACKOFF_MULTIPLIER = 2
 
+# Every this many consecutive reconnect failures (any reason), rebuild the paho client from scratch
+# and re-fetch the panel CA. Mirrors the recovery effect of a manual integration reload without
+# going through HA's config_entry teardown. Resets after every rebuild attempt so the cadence holds
+# throughout extended outages.
+MQTT_FULL_REBUILD_AFTER_FAILURES = 3
+
 # Lugs direction values
 LUGS_UPSTREAM = "UPSTREAM"
 LUGS_DOWNSTREAM = "DOWNSTREAM"