pyobs-core 2.0.0.dev45__py3-none-any.whl → 2.0.0.dev47__py3-none-any.whl

pyobs/comm/xmpp/xmppcomm.py

@@ -1018,7 +1018,7 @@ class XmppComm(Comm):
                 attempt += 1
                 if attempt == 3:
                     log.warning(
-                        "Still failing to get capabilities for %s from %s after %d attempts (%s), "
+                        "Still failing to get capabilities for %s from %s after %d attempts (%r), "
                         "will keep retrying",
                         interface.__name__,
                         module,

pyobs/modules/module.py

@@ -126,6 +126,12 @@ class Module(Object, IModule, IConfig):
         """
         Object.__init__(self, **kwargs)
 
+        # detect the event loop itself stalling (e.g. a blocking call somewhere in this
+        # module or one of its background tasks) directly in this module's own log, instead
+        # of only showing up indirectly as this module failing to answer other modules'
+        # requests in time
+        self.add_background_task(self._watch_event_loop_lag)
+
         # get list of client interfaces
         self._interfaces: list[type[Interface]] = []
         self._methods: dict[str, tuple[Callable[..., Any], inspect.Signature, dict[Any, Any]]] = {}
@@ -170,6 +176,38 @@ class Module(Object, IModule, IConfig):
         self._remote_exception_log: dict[tuple[type[exc.PyobsError], str], list[exc.LoggedException]] = {}
         self._exception_handlers: list[exc.ExceptionHandler] = []
 
+    # how often to check for event loop lag, and how much lag counts as worth a warning
+    _LOOP_LAG_CHECK_INTERVAL = 1.0
+    _LOOP_LAG_WARN_THRESHOLD = 0.5
+
+    async def _watch_event_loop_lag(self) -> None:
+        """Detects the event loop stalling by timing its own wakeups against how long it
+        actually asked to sleep for. Works even though it runs on the loop it's measuring:
+        if something else blocks the loop synchronously, this task's own wakeup is delayed
+        right along with everything else, and the resulting gap shows up here retroactively.
+
+        Logs once when a stall starts and once when it clears, rather than on every check
+        while it persists -- a single long block only ever produces one check anyway (this
+        task can't run until the block clears either), but a sustained-yet-fluctuating
+        overload (still yielding briefly, but staying over threshold for many checks in a
+        row) would otherwise log once per check for as long as it lasts.
+        """
+        loop = asyncio.get_running_loop()
+        last = loop.time()
+        stalled_since: float | None = None
+        while True:
+            await asyncio.sleep(self._LOOP_LAG_CHECK_INTERVAL)
+            now = loop.time()
+            lag = now - last - self._LOOP_LAG_CHECK_INTERVAL
+            if lag > self._LOOP_LAG_WARN_THRESHOLD:
+                if stalled_since is None:
+                    stalled_since = last
+                    log.warning("Event loop stalled for %.2fs -- some call is blocking it.", lag)
+            elif stalled_since is not None:
+                log.warning("Event loop recovered after being stalled for %.2fs total.", now - stalled_since)
+                stalled_since = None
+            last = now
+
     # exception types that always need local attention, regardless of _disable_exception_logging:
     # ModuleError means the module itself is broken; UnclassifiedError means something escaped
     # un-typed, either locally or across an RPC boundary. Neither is part of the deliberate

{pyobs_core-2.0.0.dev45.dist-info → pyobs_core-2.0.0.dev47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyobs-core
-Version: 2.0.0.dev45
+Version: 2.0.0.dev47
 Summary: robotic telescope software
 Author-email: Tim-Oliver Husser <thusser@uni-goettingen.de>
 License-Expression: MIT

{pyobs_core-2.0.0.dev45.dist-info → pyobs_core-2.0.0.dev47.dist-info}/RECORD

@@ -22,7 +22,7 @@ pyobs/comm/xmpp/__init__.py,sha256=U8A_ZAdKS3qH-SjdzZ28tEBzCrnnpAL3qrfoEl92Q_E,5
 pyobs/comm/xmpp/rpc.py,sha256=gUenqPK3FedaJCFOxc-uJ_bqokmkyXkoQ2wk2mkZrk8,14917
 pyobs/comm/xmpp/serializer.py,sha256=HFCPuzWIp3lnnJH_8WHiagzVUQaXyJEhKw6wZ1uPnw0,18999
 pyobs/comm/xmpp/xmppclient.py,sha256=d-5NqQ2ZXBFiyLbqdaJHrXVNrS4Z_19Zrnnjtsd_Fcs,7940
-pyobs/comm/xmpp/xmppcomm.py,sha256=qUNw370UvIBLd06PV0zxATIbkyl1IitO69FaQt5V7_w,48707
+pyobs/comm/xmpp/xmppcomm.py,sha256=41aB4VHlAs_HwLFsY2Dx9sXQzHKrtKqOUXOusONNMQE,48707
 pyobs/comm/xmpp/xep_0009/__init__.py,sha256=YbgcfeEcRH-M3HQ0IyHEPhPLHIj8xcWYKHSRYIIuX8k,103
 pyobs/comm/xmpp/xep_0009/binding.py,sha256=bbBwpc68e9hkt9hwU8AE_OHpt-Rpakl6JvNAg2xFHZM,5995
 pyobs/comm/xmpp/xep_0009/rpc.py,sha256=_Xzzghl4yDsdVwjL3QjmgzaiqAYhQN0y9R4vlYnTinw,2067
@@ -199,7 +199,7 @@ pyobs/mixins/pipeline.py,sha256=2QIe7ihEX2iD07U-6TJ9NoFhEKJflqM_OA5uVyQPPTY,1437
 pyobs/mixins/waitformotion.py,sha256=PFm51CqPjn23_SZ3Mgi5RPfvVBsbPaP5YF0J8G8PWSo,3002
 pyobs/mixins/weatheraware.py,sha256=CEdDCCGwFx5OSbjlS5yS6Yvloujeg8gporpJyr7o8xk,5745
 pyobs/modules/__init__.py,sha256=3jIJUmO1Vc7PD3rMCzhtYNanOME7f-2z-mot5f8ugwY,2447
-pyobs/modules/module.py,sha256=UVPfLAroom_mcnJ_ESWXA7emRLQf2ZIWdvn0bWJmSyc,41492
+pyobs/modules/module.py,sha256=sH1f31z29qtU2yvN9GcxgmyOja9ImTp7EGFkHItDcnA,43601
 pyobs/modules/camera/__init__.py,sha256=ZlkLRWssKCPi6jIFEw275jXcpv9ORjVCjsT8Ag3fK2E,421
 pyobs/modules/camera/adaptive.py,sha256=o06lkJSNnLPT19Abql0npi0RtIARKgn_CROwr7556qU,14013
 pyobs/modules/camera/basecamera.py,sha256=NaE-KP5wkr8_YmsVQ5T8mxbPS84sG0jb7kjxIr3wkWs,21267
@@ -439,8 +439,8 @@ pyobs/vfs/vfs.py,sha256=buHTmO6VUjSypkEMVxY8rDKrpMz65AY2OE2hGEsQsLI,10780
 pyobs/vfs/filelists/__init__.py,sha256=NaTGFjLGh043j_F7kWEx_cXaII7t3CbYguRk2g5hEqw,111
 pyobs/vfs/filelists/filelist.py,sha256=kCco_5G2gLH-nxWik-wcs6lGuc_-3LeeyR9L4_srf30,201
 pyobs/vfs/filelists/testing.py,sha256=M8f2BULdjRCc7JFOKhAH7Kwre_uVQC0xLcv9xGTfNFw,425
-pyobs_core-2.0.0.dev45.dist-info/METADATA,sha256=1XyRb2dUXGaVR74fnemz1Syc0eRl4tJoIuyxaAKnK_A,1990
-pyobs_core-2.0.0.dev45.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-pyobs_core-2.0.0.dev45.dist-info/entry_points.txt,sha256=0ceIn3dTDwWdGdJ5VdTvYim7vKkdTe6aqqpoaG065UY,109
-pyobs_core-2.0.0.dev45.dist-info/licenses/LICENSE,sha256=gC8djBliCENDmuRvzCoejpWbsn8eqKx61Hc8bmDZ2LI,1099
-pyobs_core-2.0.0.dev45.dist-info/RECORD,,
+pyobs_core-2.0.0.dev47.dist-info/METADATA,sha256=NXVszl4w8Vf_u-jFr0ahuhk-3iEUG-hVZqF3HQUQT2s,1990
+pyobs_core-2.0.0.dev47.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+pyobs_core-2.0.0.dev47.dist-info/entry_points.txt,sha256=0ceIn3dTDwWdGdJ5VdTvYim7vKkdTe6aqqpoaG065UY,109
+pyobs_core-2.0.0.dev47.dist-info/licenses/LICENSE,sha256=gC8djBliCENDmuRvzCoejpWbsn8eqKx61Hc8bmDZ2LI,1099
+pyobs_core-2.0.0.dev47.dist-info/RECORD,,