span-panel-api 3.0.0b7__tar.gz → 3.0.0b9__tar.gz

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.0.0b9]
+
+### Fixed
+
+- **The retry window for a rebooting panel is actually widened this time.** b8 taught the schema fetch to treat a `502` as "not ready yet" but shipped with the old five attempts capped at eight seconds — about twenty-three seconds against a panel observed
+  taking four minutes to come back, still answering 502 when the broker returned. The widening was written, lost to a failed edit in the same change, and shipped without it; nothing failed, because catching the 502 and giving up early looks exactly like
+  working. Now twelve attempts backing off to thirty seconds, a little over four minutes, and pinned by a test that asserts the total window outlasts the observed reboot rather than checking the constants individually.
+
+## [3.0.0b8]
+
+### Fixed
+
+- **A panel answering `502` while it reboots no longer costs the automatic reload.** When a panel upgrades its firmware it drops MQTT, comes back, and serves HTTP a little later — and a booting device brings its network stack and reverse proxy up before
+  the application behind them, so the schema fetch is answered with `502` rather than refused. The retry that exists for exactly this handled "cannot reach" and "timed out" but not "answered, with 502", so the first attempt raised straight out of the loop,
+  out of the fire-and-forget task that called it, and the parser was never swapped. Caught on two Home Assistant instances watching one panel through the same live upgrade: both logged `Task exception was never retrieved`, both stayed on the old parser,
+  and neither recovered without a manual reload. `get_homie_schema` now raises `SpanPanelServerError` for any 5xx — "not ready yet", distinct from a 4xx that will not fix itself — and the retry treats it as retryable.
+- **The wait is now the length of a real reboot.** Five attempts backing off to 8s gave up after about 23 seconds. The observed upgrade took four minutes from MQTT dropping to the broker returning, with HTTP still answering 502 at that point. Twelve
+  attempts backing off to 30s covers it.
+- **Nothing escapes the redispatch task any more.** An unexpected failure there used to surface as a bare `Task exception was never retrieved` while the parser silently stayed on the old generation — the failure the redispatch exists to prevent, reached by
+  another route. It is now logged at ERROR naming the consequence and the remedy, because a reload is the user's only move and nothing else was going to tell them.
+
 ## [3.0.0b7]
 
 ### Changed

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: span-panel-api
-Version: 3.0.0b7
+Version: 3.0.0b9
 Summary: A client library for SPAN Panel API
 Project-URL: Homepage, https://github.com/SpanPanel/span-panel-api
 Project-URL: Issues, https://github.com/SpanPanel/span-panel-api/issues

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "span-panel-api"
-version = "3.0.0b7"
+version = "3.0.0b9"
 description = "A client library for SPAN Panel API"
 authors = [
     {name = "SpanPanel"}

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/src/span_panel_api/auth.py

@@ -15,7 +15,13 @@ import uuid
 import httpx
 
 from ._http import _build_url, _get_client
-from .exceptions import SpanPanelAPIError, SpanPanelAuthError, SpanPanelConnectionError, SpanPanelTimeoutError
+from .exceptions import (
+    SpanPanelAPIError,
+    SpanPanelAuthError,
+    SpanPanelConnectionError,
+    SpanPanelServerError,
+    SpanPanelTimeoutError,
+)
 from .models import HomieSchemaTypes, V2AuthResponse, V2HomieSchema, V2StatusInfo
 
 
@@ -187,8 +193,22 @@ async def get_homie_schema(
     except httpx.TimeoutException as exc:
         raise SpanPanelTimeoutError(f"Timed out connecting to {host}") from exc
 
+    if response.status_code >= 500:
+        # A rebooting panel answers 502 from its front end while the application
+        # behind it is still starting. That is "not ready yet", not "wrong" --
+        # and it is the ordinary shape of a firmware upgrade, because a device
+        # brings its network stack and proxy up before its application. Raised as
+        # a distinct class so a caller can retry it and fail fast on a 4xx, which
+        # will not fix itself.
+        raise SpanPanelServerError(
+            f"Panel not ready: HTTP {response.status_code} fetching the Homie schema",
+            status_code=response.status_code,
+        )
     if response.status_code != 200:
-        raise SpanPanelAPIError(f"Failed to fetch Homie schema: HTTP {response.status_code}")
+        raise SpanPanelAPIError(
+            f"Failed to fetch Homie schema: HTTP {response.status_code}",
+            status_code=response.status_code,
+        )
 
     data: dict[str, object] = response.json()
 

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/src/span_panel_api/mqtt/client.py

@@ -48,9 +48,21 @@ _CIRCUIT_NAMES_POLL_INTERVAL_S = 0.25
 # Re-reading the schema after a suspected generation change. Bounded because the
 # caller is a fire-and-forget task on a live connection, and generous enough to
 # outlast a panel that is still binding its HTTP port after a restart.
-_REDISPATCH_RETRY_ATTEMPTS = 5
+_REDISPATCH_RETRY_ATTEMPTS = 12
 _REDISPATCH_RETRY_INITIAL_S = 1.0
-_REDISPATCH_RETRY_MAX_S = 8.0
+_REDISPATCH_RETRY_MAX_S = 30.0
+"""How long to wait for the panel's HTTP endpoint after it returns on MQTT.
+
+Sized from a live firmware upgrade rather than guessed. The panel dropped MQTT at
+11:22:07 and the broker was back at 11:26:15 -- four minutes -- and its HTTP
+front end was still answering 502 at that moment. Five attempts capped at 8s
+gives up after about 23 seconds, which is not the same order of magnitude as a
+device that is still booting: catching the 502 buys nothing if the loop stops
+before the panel is ready.
+
+Twelve attempts backing off to 30s is a little over four minutes. Each one is a
+single GET, and the panel is the only thing that can end the wait.
+"""
 
 
 def _metadata_for_the_log() -> tuple[list[str], str]:
@@ -823,7 +835,20 @@ class SpanMqttClient:
         for _ in range(_REDISPATCH_RETRY_ATTEMPTS):
             try:
                 return await get_homie_schema(self._host, port=self._panel_http_port, httpx_client=self._httpx_client)
-            except (SpanPanelConnectionError, SpanPanelTimeoutError) as exc:
+            except (
+                SpanPanelConnectionError,
+                SpanPanelTimeoutError,
+                # The third way HTTP lags the broker, and the one a real upgrade
+                # actually produced: the panel answers, with 502. Its front end is
+                # up while the application behind it is still starting, which is
+                # the ordinary order for a booting device. Omitting this meant the
+                # first attempt raised straight out of this loop, out of the
+                # fire-and-forget task that called it, and the parser was never
+                # swapped -- observed on two Home Assistant instances watching one
+                # panel through the same upgrade, neither of which recovered
+                # without a manual reload.
+                SpanPanelServerError,
+            ) as exc:
                 last = exc
                 await asyncio.sleep(delay)
                 delay = min(delay * 2, _REDISPATCH_RETRY_MAX_S)
@@ -862,6 +887,23 @@ class SpanMqttClient:
         """
         try:
             await self._redispatch_once()
+        except Exception:  # pylint: disable=broad-exception-caught
+            # Nothing may escape here. This runs as a fire-and-forget task, so an
+            # escaping exception becomes "Task exception was never retrieved" in
+            # the log and the parser silently stays on the old generation --
+            # which is the failure this whole method exists to prevent, arrived at
+            # by a different route. That is not hypothetical: a 502 from a
+            # rebooting panel did exactly this on two live installs.
+            #
+            # Logged at ERROR with the consequence spelled out, because the user's
+            # remedy is a reload and nothing else will tell them so.
+            _LOGGER.error(
+                "Could not follow the panel's schema-generation change; the %r parser is "
+                "unchanged and its data will read as missing. Reload the integration once "
+                "the panel is fully back up.",
+                self._data_model_version,
+                exc_info=True,
+            )
         finally:
             # Released only when the swap is finished, not when the fetch is.
             # Clearing it after the fetch left a window that the slowest step in

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/tests/test_auth_and_homie_helpers.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import json
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
 import pytest
@@ -11,7 +11,12 @@ import pytest
 from span_panel_api_schema_0.accumulator import HomiePropertyAccumulator
 from span_panel_api_schema_0.consumer import HomieDeviceConsumer, _parse_int
 from span_panel_api.auth import _int, download_ca_cert, get_homie_schema
-from span_panel_api.exceptions import SpanPanelConnectionError, SpanPanelTimeoutError
+from span_panel_api.exceptions import (
+    SpanPanelAPIError,
+    SpanPanelConnectionError,
+    SpanPanelServerError,
+    SpanPanelTimeoutError,
+)
 
 # ---------------------------------------------------------------------------
 # auth._int edge cases (lines 29-31)
@@ -34,6 +39,17 @@ class TestIntHelper:
 # ---------------------------------------------------------------------------
 
 
+def _mock_response(method: str, status_code: int) -> AsyncMock:
+    """A client whose request completes and returns `status_code`."""
+    response = MagicMock()
+    response.status_code = status_code
+    mock = AsyncMock()
+    setattr(mock, method, AsyncMock(return_value=response))
+    mock.__aenter__ = AsyncMock(return_value=mock)
+    mock.__aexit__ = AsyncMock(return_value=False)
+    return mock
+
+
 def _mock_client(method: str, side_effect: Exception) -> AsyncMock:
     mock = AsyncMock()
     setattr(mock, method, AsyncMock(side_effect=side_effect))
@@ -78,6 +94,32 @@ class TestGetHomieSchemaErrors:
             with pytest.raises(SpanPanelTimeoutError):
                 await get_homie_schema("192.168.1.1")
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("status", [500, 502, 503, 504])
+    async def test_a_server_status_is_not_ready_rather_than_wrong(self, status: int) -> None:
+        """A rebooting panel answers from its front end while the app behind it starts.
+
+        Raised as `SpanPanelServerError` so a caller can tell "not yet" from
+        "no". The redispatch retry depends on this distinction: a live firmware
+        upgrade produced 502 here, the retry loop did not catch the general
+        `SpanPanelAPIError` it used to be, and the parser was never swapped.
+        """
+        with patch("span_panel_api._http.httpx.AsyncClient") as cls:
+            cls.return_value = _mock_response("get", status)
+            with pytest.raises(SpanPanelServerError) as caught:
+                await get_homie_schema("192.168.1.1")
+        assert caught.value.status_code == status
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("status", [401, 404])
+    async def test_a_client_status_is_not_retryable(self, status: int) -> None:
+        """These do not fix themselves, so they must not look like "not ready yet"."""
+        with patch("span_panel_api._http.httpx.AsyncClient") as cls:
+            cls.return_value = _mock_response("get", status)
+            with pytest.raises(SpanPanelAPIError) as caught:
+                await get_homie_schema("192.168.1.1")
+        assert not isinstance(caught.value, SpanPanelServerError)
+
 
 # ---------------------------------------------------------------------------
 # homie._parse_int failure path (lines 51-52)

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/tests/test_redispatch_on_reconnect.py

@@ -30,8 +30,13 @@ from unittest.mock import patch
 
 import pytest
 
-from span_panel_api.exceptions import SpanPanelConnectionError
-from span_panel_api.mqtt.client import _REDISPATCH_RETRY_ATTEMPTS, SpanMqttClient
+from span_panel_api.exceptions import SpanPanelConnectionError, SpanPanelServerError
+from span_panel_api.mqtt.client import (
+    _REDISPATCH_RETRY_ATTEMPTS,
+    _REDISPATCH_RETRY_INITIAL_S,
+    _REDISPATCH_RETRY_MAX_S,
+    SpanMqttClient,
+)
 from span_panel_api.mqtt.models import MqttClientConfig
 
 from conftest import SERIAL
@@ -299,3 +304,94 @@ async def test_a_raising_consumer_does_not_break_the_swap() -> None:
 
     assert client.data_model_version == "1.0", "the swap must stand"
     assert reached == ["second"], "one raising subscriber must not starve the others"
+
+
+@pytest.mark.asyncio
+async def test_a_rebooting_panel_answering_502_is_waited_for_not_abandoned() -> None:
+    """The failure that cost a live firmware upgrade its automatic reload.
+
+    A panel accepts MQTT before it serves HTTP, and the retry loop above exists
+    for that. But there are three ways HTTP lags the broker, and this loop
+    originally handled two: it caught "cannot reach" and "timed out" and not
+    "answered, with 502". A booting device brings its network stack and reverse
+    proxy up before the application behind them, so 502 is the *ordinary* shape,
+    not an exotic one.
+
+    Because `SpanPanelServerError` was not caught, the very first attempt raised
+    straight out of the loop, out of the fire-and-forget task that called it, and
+    the parser was never swapped. Observed on two Home Assistant instances
+    watching one panel through the same upgrade: both logged `Task exception was
+    never retrieved`, both stayed on the flat parser, and neither recovered
+    without a manual reload.
+    """
+    client, _ = _client(None)
+    before = client.adapter
+    attempts = 0
+
+    def _five_oh_two_then_ready(*_a: object, **_k: object) -> _Schema:
+        nonlocal attempts
+        attempts += 1
+        if attempts < 3:
+            raise SpanPanelServerError("Panel not ready: HTTP 502 fetching the Homie schema", 502)
+        return _Schema("1.0")
+
+    with (
+        patch("span_panel_api.mqtt.client.get_homie_schema", side_effect=_five_oh_two_then_ready),
+        patch("span_panel_api.mqtt.client._REDISPATCH_RETRY_INITIAL_S", 0),
+        patch("span_panel_api.mqtt.client._REDISPATCH_RETRY_MAX_S", 0),
+    ):
+        await _panel_publishes_version(client, "1.0")
+
+    assert attempts >= 3, "a 502 must be retried rather than ending the attempt"
+    assert client.adapter is not before, "the parser must swap once the panel answers"
+
+
+@pytest.mark.asyncio
+async def test_an_unexpected_failure_leaves_a_usable_message_rather_than_a_bare_traceback(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Nothing may escape the fire-and-forget task.
+
+    An escaping exception surfaces as "Task exception was never retrieved" and
+    the parser silently stays on the old generation -- the exact failure this
+    method exists to prevent, reached by a different route. The user's remedy is
+    a reload, and nothing else is going to tell them so.
+    """
+    client, _ = _client(None)
+    before = client.adapter
+
+    with patch(
+        "span_panel_api.mqtt.client.get_homie_schema",
+        side_effect=RuntimeError("something nobody predicted"),
+    ):
+        await _panel_publishes_version(client, "1.0")
+
+    assert client.adapter is before
+    assert "Reload the integration" in caplog.text
+    assert "something nobody predicted" in caplog.text
+
+
+def test_the_retry_window_outlasts_a_real_panel_reboot() -> None:
+    """Catching the 502 buys nothing if the loop gives up before the panel is ready.
+
+    Measured rather than assumed. On a live firmware upgrade the panel dropped
+    MQTT at 11:22:07 and the broker was back at 11:26:15 — four minutes — and its
+    HTTP front end was still answering 502 at that moment, which is when this
+    loop starts.
+
+    Pinned as a total because the three constants only mean something together,
+    and because the widening was written once, lost to a failed edit, and shipped
+    without it. Nothing failed: the 502 was caught and the loop still gave up
+    after twenty-three seconds. A test on the constants is the only thing that
+    would have noticed.
+    """
+    delay = _REDISPATCH_RETRY_INITIAL_S
+    total = 0.0
+    for _ in range(_REDISPATCH_RETRY_ATTEMPTS):
+        total += delay
+        delay = min(delay * 2, _REDISPATCH_RETRY_MAX_S)
+
+    observed_reboot_s = 4 * 60
+    assert total >= observed_reboot_s, (
+        f"the retry window is {total:.0f}s, shorter than the {observed_reboot_s}s reboot " "this loop exists to wait out"
+    )

{span_panel_api-3.0.0b7 → span_panel_api-3.0.0b9}/tests/test_schema_one_conformance.py

@@ -37,6 +37,7 @@ import ast
 import importlib
 import json
 import os
+import subprocess
 from pathlib import Path
 import re
 from typing import NoReturn
@@ -572,26 +573,64 @@ def test_nothing_is_recorded_as_unexercised_once_the_simulator_publishes_it() ->
 
 
 def test_vendored_catalogs_are_byte_identical_to_the_specification() -> None:
-    """Byte comparison against the specification at `synced_commit`.
-
-    Skipped rather than failed without a checkout: the checks above are the ones
-    that must run everywhere, and making them depend on a second repository would
-    mean they stop running.
+    """Are the bytes we vendored the bytes we claim they are?
+
+    Read out of git **at `synced_commit`** rather than from the checkout's working
+    tree, so the answer does not depend on what that clone happens to be sitting
+    on. This used to read the working tree while its own docstring claimed
+    otherwise, and `synced_commit` appeared only in the failure message. That is
+    wrong three ways, and one of them is the dangerous one:
+
+    * it **fails** when the clone has moved *ahead* of the pin, which is ordinary
+      currency drift and not a defect here -- observed the day the specification
+      went to `power-flows` 0.3;
+    * it **fails spuriously** with the clone on an unrelated branch;
+    * it **passes falsely** with a clone itself stale at the pinned commit while
+      the specification has moved on.
+
+    **Integrity, deliberately not currency.** Whether upstream has moved past our
+    pin is a separate question whose answer is normally "yes, a little", and it
+    must not fail a build. Conflating the two is what made this unreliable.
+    Currency is not checked by anything automatic here, and wants a scheduled job
+    rather than a gate.
+
+    The upstream reference producer fixed the same defect in its own copy of this
+    check (`distribution-enclosure-simulator` #47), which is where the framing
+    comes from.
     """
     spec = _checkout(
         "EBUS_SPEC_DIR",
         "a specification checkout to verify vendored bytes",
         expect="capabilities",
     )
-    differing = [
-        path.name
-        for path in sorted(_CATALOGS.glob("*.json"))
-        if (spec / "capabilities" / path.name).read_bytes() != path.read_bytes()
-    ]
+    commit = _lock()["synced_commit"]
+    differing: list[str] = []
+    for path in sorted(_CATALOGS.glob("*.json")):
+        blob = subprocess.run(
+            ["git", "-C", str(spec), "show", f"{commit}:capabilities/{path.name}"],
+            capture_output=True,
+            # Stripped, because `-C` does not beat them. Git hooks export `GIT_DIR`
+            # and `GIT_INDEX_FILE` pointing at the repository being committed to,
+            # and an exported `GIT_DIR` wins over directory discovery -- so under
+            # pre-commit this read the *consumer's* object store, could not find a
+            # specification commit there, and failed with a fetch instruction for a
+            # commit the clone already had. Caught by the hook that causes it.
+            env={k: v for k, v in os.environ.items() if not k.startswith("GIT_")},
+        )
+        if blob.returncode != 0:
+            # A clone that cannot resolve the pin fails rather than skipping: a
+            # silent skip reads exactly like a pass on the one check that proves
+            # the vendored bytes are what the lockfile says.
+            pytest.fail(
+                f"{spec} cannot resolve {commit} (needed to read capabilities/{path.name}). "
+                f"Fetch it: git -C {spec} fetch origin {commit}"
+            )
+        if blob.stdout != path.read_bytes():
+            differing.append(path.name)
 
     assert not differing, (
-        f"vendored catalogs differ from {spec} (lockfile pins {_lock()['synced_commit']}): {differing}. "
-        "Check the checkout is at synced_commit before assuming the copies are wrong."
+        f"vendored catalogs differ from the specification at {commit}: {differing}. "
+        "These are byte copies, so this is a vendoring defect rather than upstream having moved."
     )