pyobs-core 2.0.0.dev31__tar.gz → 2.0.0.dev33__tar.gz

{pyobs_core-2.0.0.dev31 → pyobs_core-2.0.0.dev33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyobs-core
-Version: 2.0.0.dev31
+Version: 2.0.0.dev33
 Summary: robotic telescope software
 Author-email: Tim-Oliver Husser <thusser@uni-goettingen.de>
 License-Expression: MIT

{pyobs_core-2.0.0.dev31 → pyobs_core-2.0.0.dev33}/pyobs/comm/xmpp/xmppclient.py

@@ -15,7 +15,14 @@ log = logging.getLogger(__name__)
 class XmppClient(slixmpp.ClientXMPP):
     """XMPP client for pyobs."""
 
-    def __init__(self, jid: str, password: str, **kwargs: Any):
+    def __init__(
+        self,
+        jid: str,
+        password: str,
+        ping_interval: float = 300.0,
+        ping_timeout: float = 30.0,
+        **kwargs: Any,
+    ):
         """Create a new XMPP client.
 
         An XmppClient handles the actual XMPP communication for the XmppComm module.
@@ -23,6 +30,12 @@ class XmppClient(slixmpp.ClientXMPP):
         Args:
             jid: Connect to the XMPP server with the given JID.
             password: Password to use for connection.
+            ping_interval: Seconds between XEP-0199 keepalive pings.
+            ping_timeout: Seconds to wait for a ping reply before treating the connection as
+                dead and reconnecting. If the server-side shaper can delay an IQ reply by more
+                than this, keepalive pings misfire as false disconnects -- raise this to be more
+                tolerant of a throttled/slow server instead of only tolerating a genuinely dead
+                connection.
         """
 
         slixmpp.ClientXMPP.__init__(self, jid, password, **kwargs)
@@ -51,7 +64,7 @@ class XmppClient(slixmpp.ClientXMPP):
         self.register_plugin("xep_0199")  # XMPP Ping
 
         # enable keep alive pings
-        self.plugin["xep_0199"].enable_keepalive()
+        self.plugin["xep_0199"].enable_keepalive(interval=ping_interval, timeout=ping_timeout)
 
         # handle session_start and message events
         self.add_event_handler("session_start", self.session_start)

{pyobs_core-2.0.0.dev31 → pyobs_core-2.0.0.dev33}/pyobs/comm/xmpp/xmppcomm.py

@@ -4,6 +4,7 @@ import asyncio
 import functools
 import json
 import logging
+import random
 import re
 import ssl
 import time
@@ -36,6 +37,17 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 
+def _retry_delay(attempt: int, cap: float = 30.0, base: float = 1.0) -> float:
+    """Capped exponential backoff with full jitter.
+
+    Used for retry loops that can end up running on every module in the fleet at once (e.g.
+    every module reconnecting to ejabberd simultaneously after a mass restart). Fixed-interval
+    retries stay in lockstep across all of them, repeatedly hammering the server at the same
+    instants; jitter decorrelates that so retries spread out over time instead.
+    """
+    return random.uniform(0, min(cap, base * (2**attempt)))
+
+
 class EventStanza(ElementBase):
     name = "event"
     namespace = "pyobs:event"
@@ -95,6 +107,16 @@ class XmppComm(Comm):
     Finally, always make sure that ``use_tls`` is set according to the server's settings, i.e. if it uses TLS, this
     parameter must be True, and False otherwise. Cryptic error messages will follow, if one does not set this properly.
 
+    ``ping_interval``/``ping_timeout`` control the XEP-0199 keepalive ping that detects a dead connection.
+    ``ping_timeout`` in particular may need raising above its 30s default on a server whose shaper can delay an IQ
+    reply that long under load -- otherwise a merely-slow reply is indistinguishable from a dead connection and
+    triggers a reconnect that didn't need to happen::
+
+        comm:
+            class: pyobs.comm.xmpp.XmppComm
+            jid: someuser@example.com/pyobs
+            ping_timeout: 60
+
     """
 
     __module__ = "pyobs.comm.xmpp"
@@ -109,6 +131,8 @@ class XmppComm(Comm):
         server: str | None = None,
         use_tls: bool = False,
         ignore_cert_errors: bool = False,
+        ping_interval: float = 300.0,
+        ping_timeout: float = 30.0,
         *args: Any,
         **kwargs: Any,
     ):
@@ -124,6 +148,8 @@ class XmppComm(Comm):
             password: Password for given JID.
             server: Server to connect to. If not given, domain from JID is used.
             use_tls: Whether to use TLS.
+            ping_interval: Seconds between XEP-0199 keepalive pings.
+            ping_timeout: Seconds to wait for a ping reply before reconnecting.
         """
         Comm.__init__(self, *args, **kwargs)
 
@@ -140,6 +166,8 @@ class XmppComm(Comm):
         self._server = server
         self._use_tls = use_tls
         self._ignore_cert_errors = ignore_cert_errors
+        self._ping_interval = ping_interval
+        self._ping_timeout = ping_timeout
         self._loop = asyncio.get_event_loop()
         self._safe_send_attempts = 5
         self._safe_send_wait = 1
@@ -217,7 +245,9 @@ class XmppComm(Comm):
             self._xmpp.abort()
 
         # create client
-        self._xmpp = XmppClient(self._jid, self._password)
+        self._xmpp = XmppClient(
+            self._jid, self._password, ping_interval=self._ping_interval, ping_timeout=self._ping_timeout
+        )
 
         # presence gating (see mark_ready()) only applies to a comm with an actual Module
         # attached that hasn't finished starting yet -- a module-less XmppComm (a GUI, an
@@ -929,8 +959,12 @@ class XmppComm(Comm):
     async def _get_capabilities(self, module: str, interface: type[Interface]) -> Any | None:
         """Fetch and deserialize capabilities for a remote module's interface.
 
-        Retries a few times, since this may run right as the peer is still starting up and hasn't
-        published its capabilities yet.
+        Retries indefinitely (capped exponential backoff with jitter) instead of giving up after
+        a fixed budget -- a peer that's merely slow to respond (e.g. every module in the fleet
+        reconnecting to ejabberd at once) must still eventually get its capabilities fetched
+        without requiring a full disconnect/reconnect of that peer to retrigger discovery. Only
+        stops if the peer itself goes offline in the meantime, since a fresh fetch is triggered
+        from scratch the next time it comes back online (see _got_online).
         """
         if interface.capabilities is None:
             return None
@@ -938,24 +972,36 @@ class XmppComm(Comm):
 
         result = None
         last_error: BaseException | None = None
-        for attempt in range(3):
+        attempt = 0
+        while True:
             # Use full JID (with resource) if we know it — bare JID may not route correctly
-            full_jid = next(
-                (jid for jid in self._online_clients if jid.startswith(f"{module}@")),
-                f"{module}@{self._domain}",
-            )
+            full_jid = next((jid for jid in self._online_clients if jid.startswith(f"{module}@")), None)
+            if full_jid is None and attempt > 0:
+                log.debug(
+                    "Giving up fetching capabilities for %s from %s: peer went offline",
+                    interface.__name__,
+                    module,
+                )
+                return None
             try:
                 result = await asyncio.wait_for(
-                    self.client.plugin["xep_0030"].get_info(jid=JID(full_jid)), timeout=10.0
+                    self.client.plugin["xep_0030"].get_info(jid=JID(full_jid or f"{module}@{self._domain}")),
+                    timeout=10.0,
                 )
                 break
-            except (TimeoutError, Exception) as e:
+            except Exception as e:
                 last_error = e
-                if attempt < 2:
-                    await asyncio.sleep(2)
-        else:
-            log.warning("Failed to get capabilities for %s from %s: %s", interface.__name__, module, last_error)
-            return None
+                attempt += 1
+                if attempt == 3:
+                    log.warning(
+                        "Still failing to get capabilities for %s from %s after %d attempts (%s), "
+                        "will keep retrying",
+                        interface.__name__,
+                        module,
+                        attempt,
+                        last_error,
+                    )
+                await asyncio.sleep(_retry_delay(attempt))
         log.debug("get_capabilities disco result XML: %s", ET.tostring(result.xml).decode()[:500])
         # result.xml is the <iq> — the <query> is its child, capabilities are grandchildren
         for child in result.xml:
@@ -1012,17 +1058,30 @@ class XmppComm(Comm):
     async def _subscribe_with_retry(self, node: str, interface: type[Interface]) -> None:
         """Subscribe to a pubsub node, retrying until the node exists.
 
-        Runs as a background task so _subscribe_state returns immediately.
+        Runs as a background task so _subscribe_state returns immediately. Retries indefinitely
+        (capped exponential backoff with jitter) instead of giving up after a fixed budget -- a
+        node that's merely slow to appear (e.g. every module in the fleet reconnecting to
+        ejabberd at once, or the publisher hasn't started up yet) must still eventually get
+        subscribed without manual intervention. Stops only if _unsubscribe_state removes the
+        last callback for this node in the meantime -- the while condition then goes False,
+        which exits the loop into the else clause (not the break path) and returns.
         Once subscribed, fetches the current value and dispatches it.
         """
-        for _attempt in range(30):
+        attempt = 0
+        while node in self._state_node_handlers:
             try:
                 await self._safe_send(self.client.plugin["xep_0060"].subscribe, self._pubsub_service, node)
                 break
             except (slixmpp.exceptions.IqError, slixmpp.exceptions.IqTimeout):
-                await asyncio.sleep(1)
+                attempt += 1
+                if attempt == 30:
+                    log.warning(
+                        "Still failing to subscribe to state node %s after %d attempts, will keep retrying",
+                        node,
+                        attempt,
+                    )
+                await asyncio.sleep(_retry_delay(attempt))
         else:
-            log.warning("Could not subscribe to state node %s after 30 attempts", node)
             return
 
         # Fetch current value immediately after subscribing

{pyobs_core-2.0.0.dev31 → pyobs_core-2.0.0.dev33}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyobs-core"
-version = "2.0.0.dev31"
+version = "2.0.0.dev33"
 description = "robotic telescope software"
 authors = [{ name = "Tim-Oliver Husser", email = "thusser@uni-goettingen.de" }]
 requires-python = ">=3.11"