cgse-core 0.17.3__py3-none-any.whl → 0.18.1__py3-none-any.whl

cgse_core/_stop.py

@@ -1,9 +1,8 @@
 import logging
 import subprocess
-import sys
-from pathlib import Path
 
 import rich
+import sys
 
 from egse.log import logger
 from egse.process import is_process_running

{cgse_core-0.17.3.dist-info → cgse_core-0.18.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse-core
-Version: 0.17.3
+Version: 0.18.1
 Summary: Core services for the CGSE framework
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse_core-0.17.3.dist-info → cgse_core-0.18.1.dist-info}/RECORD

@@ -1,15 +1,15 @@
 cgse_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cgse_core/_start.py,sha256=m0JDrtEVRRmRA3nL8kQhEKq7eWiF5lf5kQ3fq9-JrkM,2126
 cgse_core/_status.py,sha256=Quz7hNtqSZPWn8xZ9ZmmF2i5hGY_bMUoybN2UNNBsHc,3259
-cgse_core/_stop.py,sha256=ZXTzvWjJ3bVNmNcmsP8NP5G6AHvegamyfu_cNn8qa6A,3588
+cgse_core/_stop.py,sha256=MTXtnYjrqJdwPOkQ7h1Zx-C57XeXxqrL6KijYhLL6AE,3563
 cgse_core/cgse_explore.py,sha256=8jxAxYDsjPUZftUccPCneHaqijRHxBPZuaOo0ESmBUs,460
 cgse_core/services.py,sha256=_a1d1Dcf9L0F1IF3w6kHrACE5EDnqtNUd8KzrTZDu5U,8595
 cgse_core/settings.yaml,sha256=6RcZ3aI7IUSB6GaZ8e9b_JEiql0l3l7h-VowDycv5AE,3625
 egse/_setup_core.py,sha256=ei4a7tGYfDxc16kfhrF11Tm3cOSnl48MsSGfrWu0vVg,5983
 egse/command.py,sha256=529_T619qG3Xif9UkxZS8mj2ibr6eljfbVsMM-K_AII,21965
-egse/connect.py,sha256=BDMzTt4URx7TpNTPyzb1Ye3Ch09enxm5yIzcxMRga_A,1834
-egse/control.py,sha256=u4bWsKNW0tl4gNMaSMyUA02xXwvcxCVe2TsdfJpTkMs,28424
-egse/dummy.py,sha256=I63wXBdC30WqnP6gz1sJ4YBHjrWumeXhiHbF49oF_ZQ,21909
+egse/connect.py,sha256=QiG5G0qnCI3EeDhmfj-gBlhI9RmKvnd0dGXu0ZBIZ90,21556
+egse/control.py,sha256=pKhQjp3YGkw8J4KyDTO3BUwBePSBJAQnfwqBPsZtpSs,28468
+egse/dummy.py,sha256=N1v9bCOF3saY6J_ip2h1pbgregQNeT4YEpDIhhKUKps,22721
 egse/listener.py,sha256=kD62oD7w1yBVV0M2Eq5MNEQyVaJvlsl0RyhYCzVkgQs,6493
 egse/mixin.py,sha256=fklH61C7TtaTnXx7FQjjJCgUQ2PsQv5baw0SVBBdbH0,20265
 egse/monitoring.py,sha256=0Hbd5SHUU2YWFm8gbzlyPMUMjTezKap_q9yE4T6Vkk0,9716
@@ -28,30 +28,30 @@ egse/icons/simulator-mode.svg,sha256=iDU6i0nTqyeYJOY9NAIj6NvjBQWAo2XSJ__35funhXY
 egse/icons/start-process-button.svg,sha256=lvOmRzafhldlAbjKtBjuqrRDjBwyaVnd3yK2ees22JM,476
 egse/icons/stop-process-button.svg,sha256=ZysOJooxOXEbup9zd6077M36OS0S5jQQkny9jfQWBfI,456
 egse/icons/user-interface.svg,sha256=q_KWgHJ9ATdbw-HFghoaGQIBe52ZyxyAKitHpDaFWm8,2375
-egse/logger/__init__.py,sha256=5Nku3dpZVBdawiKZJgNyhuw5-ppm1K9lSb_p5uinp-o,10763
+egse/logger/__init__.py,sha256=IhqNw4Yvm26Q5w7m3NuTAL8ZkywtN9dT4OaTpc3xkuo,10782
 egse/logger/__main__.py,sha256=_etegNrUM9IWHNTOObDVY1DiLn9SJyf8eHktVBbN-OQ,1765
 egse/logger/log_cs.py,sha256=3k-HUnfrJY2072cjF0TXBv9TyNd4GzCSzazx03zfJss,14296
 egse/notifyhub/__init__.py,sha256=pbCy5PHSR7OKcD1heaMIDO6ztkyFlpsvc-i2pyOdCpU,2438
 egse/notifyhub/client.py,sha256=VOFlxcotEm3JAe5eGMDXpCRejXzDb3-7k4gIPgcgmpY,10536
 egse/notifyhub/event.py,sha256=Yo8uvnK27uzaCLjJa8IK734Hcg3eLuD_KCd7eZ80XBQ,792
-egse/notifyhub/server.py,sha256=_bK45WZhu6KKuJMLLzQdDLBVz6Z_d4n0UFf77ydiDns,14520
+egse/notifyhub/server.py,sha256=XW4lbmz1IUhhPUB3frbgvAWUyAXZy1Wr1ofC10vAoM0,14553
 egse/notifyhub/services.py,sha256=LpKxJfFszwoQN9Kb5f5lkjjKLbvwCfHSC2mt1WZt1wc,10494
 egse/procman/__init__.py,sha256=w2ilHHVuznkNCK-Syd9WVayktW8um8_EsKF5JQJDZK0,13946
 egse/procman/procman.yaml,sha256=miAq3GE4PSKm8QZDfjvmtBrYCt8GyzJDJm-JzzgdznQ,847
-egse/procman/procman_cs.py,sha256=lvuhmGdd1GmkdkXYNS9slNT9VJF09OtffZrZmCdy97I,5390
+egse/procman/procman_cs.py,sha256=2FxpDI4h-3312rt84J4rcX1QNBFFst0Q1cLy5Hex3ro,5381
 egse/procman/procman_protocol.py,sha256=kFUWQb66d5aDE7qHM9xYmsO07UB-6ro2hJ-E7o_OPlE,3778
-egse/procman/procman_ui.py,sha256=5pU4-Dtma5aY8k4PaHQpJUe-DSwjpP-V1sJpbuUMlLg,42070
+egse/procman/procman_ui.py,sha256=IzusnQymoBsiQ50lCH7gopbmNoevotC2vmRCeTTR1F8,42184
 egse/registry/__init__.py,sha256=U7ap_245Bid74nn7Ncd0GXu0aetR95dJHltGRwBCVb8,1349
 egse/registry/backend.py,sha256=-kRXncO949YETb1S2GFedhYTt1O0teiKYUUPxeQqX9E,22120
-egse/registry/client.py,sha256=-fivaDQSTb_SjbZWcir_MpH5hciMSJHVinuSS5RBmMU,42874
-egse/registry/server.py,sha256=1Zv-1VkGhpKRo_P3gRRnlN1UuXJbFnYWX3fOVtgKe-g,21869
-egse/registry/service.py,sha256=QnKVICWiuHmzESmy2H4VEtJ-tGX8hSV6n6qq6ejkWAY,15125
+egse/registry/client.py,sha256=InUttsnCUDUjWHyFQ5RFAUckmFo8deQOdFshXcUVubs,43395
+egse/registry/server.py,sha256=MTBS-f0kXtxgpxJWGtUNklZL405WEokS4rp_JpmIqQE,22296
+egse/registry/service.py,sha256=r6ThO_ur9WyFb_9JkpE__C5Y2P3VAeK8j_w9SFHiIBw,15375
 egse/storage/__init__.py,sha256=xtMdHdtPT9-oqTp2bpWw7Os3qUgN8TdqZNuaj3glxn0,43147
 egse/storage/__main__.py,sha256=LI9fxlsFWmEd5LcWUB0xA8i7Yt6UHgnblB4G0aTi3pI,28
 egse/storage/persistence.py,sha256=35fvuCPuGTSCc2MfmFLLNU03xYq3CEaJQspot4f-Pvw,18274
 egse/storage/storage.yaml,sha256=l3HtPx_bAbXoV4f3_PXWAa1tP-fY2S6roSBSBiOHodE,2712
 egse/storage/storage_cs.py,sha256=172llnKef1fdiDXcnzYllw_q12bVyuGJGh_3XpeDVCU,7377
-cgse_core-0.17.3.dist-info/METADATA,sha256=cG8dHirqx3LtMgrrOJkxOHtUJEyCrsU8zs4OfzP1hz0,582
-cgse_core-0.17.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cgse_core-0.17.3.dist-info/entry_points.txt,sha256=IwIG7aGgfUehol29ufcKd559S88t3TJdh1LMJ5YymCE,976
-cgse_core-0.17.3.dist-info/RECORD,,
+cgse_core-0.18.1.dist-info/METADATA,sha256=JhEOJtC_SXdr-CnBqg_wP9Q0bkBBO5w8hqZqUYadR1g,582
+cgse_core-0.18.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cgse_core-0.18.1.dist-info/entry_points.txt,sha256=IwIG7aGgfUehol29ufcKd559S88t3TJdh1LMJ5YymCE,976
+cgse_core-0.18.1.dist-info/RECORD,,

egse/connect.py

@@ -1,5 +1,12 @@
+import random
+import threading
+import time
+from enum import Enum
+from typing import Any
+
 from egse.env import bool_env
 from egse.log import logging
+from egse.system import type_name
 from egse.zmq_ser import connect_address
 
 logger = logging.getLogger("egse.connect")
@@ -10,7 +17,7 @@ VERBOSE_DEBUG = bool_env("VERBOSE_DEBUG")
 
 
 def get_endpoint(
-    service_type: str,
+    service_type: str | None = None,
     protocol: str = "tcp",
     hostname: str = "localhost",
     port: int = 0,
@@ -41,15 +48,481 @@ def get_endpoint(
             endpoint = reg.get_endpoint(service_type)
         if endpoint:
             if VERBOSE_DEBUG:
-                logger.debug(f"Endpoint for {service_type} found in registry: {endpoint}")
+                logger.debug(f"Endpoint for '{service_type}' found in registry: {endpoint}")
         else:
-            logger.warning(f"No endpoint for {service_type} found in registry.")
+            logger.warning(f"No endpoint for '{service_type}' found in registry.")
 
     if not endpoint:
         if port == 0:
-            raise RuntimeError(f"No service registered as {service_type} and no port provided.")
+            raise RuntimeError(f"No service registered as '{service_type}' and no port provided.")
         endpoint = connect_address(protocol, hostname, port)
         if VERBOSE_DEBUG:
             logger.debug(f"Endpoint constructed from protocol/hostname/port: {endpoint}")
 
     return endpoint
+
+
+class ConnectionState(Enum):
+    DISCONNECTED = "disconnected"
+    CONNECTING = "connecting"
+    CONNECTED = "connected"
+    CIRCUIT_OPEN = "circuit_open"  # Temporarily stopped trying
+
+
+class BackoffStrategy(Enum):
+    """
+    Specifies the strategy for increasing the delay between retry attempts
+    in backoff algorithms to reduce load and avoid overwhelming services.
+
+    Strategies:
+        EXPONENTIAL:
+            The delay doubles with each retry attempt (e.g., 1s, 2s, 4s, 8s).
+            This is the most widely used approach because it quickly reduces load on struggling systems.
+        LINEAR:
+            The delay increases by a fixed amount each time (e.g., 1s, 2s, 3s, 4s).
+            This provides a more gradual reduction in request rate.
+        FIXED:
+            Uses the same delay between all retry attempts.
+            Simple but less adaptive to system conditions.
+
+    References:
+        - AWS Architecture Blog: Exponential Backoff And Jitter
+    """
+
+    EXPONENTIAL = "exponential"
+    """The delay doubles with each retry attempt (e.g., 1s, 2s, 4s, 8s). 
+    This is the most widely used approach because it quickly reduces load on struggling systems."""
+    LINEAR = "linear"
+    """The delay increases by a fixed amount each time (e.g., 1s, 2s, 3s, 4s). 
+    This provides a more gradual reduction in request rate."""
+    FIXED = "fixed"
+    """Uses the same delay between all retry attempts. Simple but less adaptive to system conditions."""
+
+
+class JitterStrategy(Enum):
+    """
+    Specifies the strategy for applying jitter (randomization) to retry intervals
+    in backoff algorithms to avoid synchronized retries and reduce load spikes.
+
+    Strategies:
+        NONE:
+            No jitter is applied. The retry interval is deterministic.
+        FULL:
+            Applies full jitter by selecting a random value uniformly between 0 and the calculated interval.
+            This maximizes randomness but can result in very short delays.
+        EQUAL:
+            Applies "equal jitter" as described in the AWS Architecture Blog.
+            The interval is randomized within [interval/2, interval], ensuring a minimum delay of half the interval.
+            Note: This is not the same as "a jitter of 50% around interval" (which would be [0.5 * interval, 1.5 * interval]).
+        PERCENT_10:
+            Applies a jitter of ±10% around the base interval, resulting in a random interval within [0.9 * interval, 1.1 * interval].
+
+    References:
+        - AWS Architecture Blog: Exponential Backoff And Jitter
+    """
+
+    NONE = "none"
+    """No jitter is applied to the backoff."""
+    FULL = "full"
+    """Maximum distribution but can be too random with very short intervals."""
+    EQUAL = "equal"
+    """Best balance, maintains backoff properties while preventing synchronization."""
+    PERCENT_10 = "10%"
+    """Add a jitter of 10% around the base interval."""
+
+
+def calculate_retry_interval(
+    attempt_number,
+    base_interval,
+    max_interval,
+    backoff_strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL,
+    jitter_strategy: JitterStrategy = JitterStrategy.EQUAL,
+):
+    """
+    Calculates the next retry interval based on the given backoff and jitter strategies.
+
+    Args:
+        attempt_number (int): The current retry attempt (starting from 0).
+        base_interval (float): The initial interval in seconds.
+        max_interval (float): The maximum allowed interval in seconds.
+        backoff_strategy (BackoffStrategy): Strategy for increasing the delay (exponential, linear, or fixed).
+        jitter_strategy (JitterStrategy): Strategy for randomizing the delay to avoid synchronization.
+
+    Returns:
+        float: The computed retry interval in seconds.
+
+    Notes:
+        - See the docstrings for BackoffStrategy and JitterStrategy for details on each strategy.
+        - Based on best practices from the AWS Architecture Blog: Exponential Backoff And Jitter.
+    """
+
+    if backoff_strategy == BackoffStrategy.EXPONENTIAL:
+        interval = min(base_interval * (2**attempt_number), max_interval)
+    elif backoff_strategy == BackoffStrategy.LINEAR:
+        interval = min(base_interval + attempt_number, max_interval)
+    else:
+        interval = base_interval
+
+    if jitter_strategy == JitterStrategy.NONE:
+        return interval
+    elif jitter_strategy == JitterStrategy.FULL:
+        return random.uniform(0, interval)
+    elif jitter_strategy == JitterStrategy.EQUAL:
+        return interval / 2 + random.uniform(0, interval / 2)
+    elif jitter_strategy == JitterStrategy.PERCENT_10:
+        jitter_amount = interval * 0.1
+        return interval + random.uniform(-jitter_amount, jitter_amount)
+
+    return interval
+
+
+class AsyncServiceConnector:
+    """
+    Asynchronous base class for robust service connection management with retry, backoff, and circuit breaker logic.
+
+    This class is intended to be subclassed for managing persistent connections to external services
+    (such as devices, databases, or remote APIs) that may be unreliable or temporarily unavailable.
+
+    Features:
+        - Automatic retry with configurable backoff and jitter strategies.
+        - Circuit breaker to prevent repeated connection attempts after multiple failures.
+        - Connection state tracking (disconnected, connecting, connected, circuit open).
+
+    Usage:
+        1. Subclass `AsyncServiceConnector` and override the `connect_to_service()` coroutine with your
+           actual connection logic. Optionally, override `health_check()` for custom health verification.
+        2. Store the actual connection object (e.g., socket, transport) as an instance attribute in your subclass.
+        3. Use `attempt_connection()` to initiate connection attempts; it will handle retries and backoff automatically.
+        4. Use `is_connected()` to check connection status.
+
+    Example:
+        class MyConnector(AsyncServiceConnector):
+            async def connect_to_service(self):
+                self.connection = await create_socket()
+                return self.connection is not None
+
+            def get_connection(self):
+                return self.connection
+
+    Note:
+        The base class does not manage or expose the underlying connection object.
+        Your subclass should provide a method or property to access it as needed.
+    """
+
+    def __init__(
+        self,
+        service_name: str,
+        backoff_strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL,
+        jitter_strategy: JitterStrategy = JitterStrategy.EQUAL,
+    ):
+        self.state = ConnectionState.DISCONNECTED
+        self.last_attempt = 0
+        self.base_interval = 1
+        self.retry_interval = 1  # Start with 1 second
+        self.max_retry_interval = 300  # Max 5 minutes
+        self.failure_count = 0
+        self.max_failures_before_circuit_break = 5
+        self.circuit_break_duration = 60  # 1 minute
+        self.circuit_opened_at = None
+        self.backoff_strategy = backoff_strategy
+        self.jitter_strategy = jitter_strategy
+
+        self.service_name = service_name
+
+    async def connect_to_service(self) -> bool:
+        logger.warning(
+            f"The connect_to_service() method is not implemented for {self.service_name}, connection will always fail."
+        )
+        return False
+
+    async def disconnect_from_service(self) -> None:
+        """
+        Optional hook to cleanly disconnect / release resources for the service.
+        Default implementation is a no-op. Subclasses should override to:
+          - close async transports
+          - cancel background tasks
+          - set state to ConnectionState.DISCONNECTED
+          - call device.disconnect()
+        """
+        logger.debug(f"{self.service_name}: default async disconnect_from_service(): no-op")
+        self.state = ConnectionState.DISCONNECTED
+        return
+
+    async def health_check(self) -> bool:
+        logger.warning(
+            f"The health_check() method is not implemented for {self.service_name}, check will always return false."
+        )
+        return False
+
+    def should_attempt_connection(self) -> bool:
+        """Return True if we should attempt a new connection."""
+        now = time.monotonic()
+
+        # If circuit is open, check if we should close it
+        if self.state == ConnectionState.CIRCUIT_OPEN:
+            assert self.circuit_opened_at is not None
+            circuit_break_open_since = now - self.circuit_opened_at
+            logger.debug(f"{circuit_break_open_since=}")
+            if circuit_break_open_since > self.circuit_break_duration:
+                self.state = ConnectionState.DISCONNECTED
+                self.failure_count = 0
+                self.retry_interval = 1
+                return True
+            return False
+
+        # Regular backoff logic
+        return now - self.last_attempt >= self.retry_interval
+
+    async def attempt_connection(self):
+        """Try to connect to the service.
+
+        This will execute the `connect_to_service()` that was overridden by the subclass.
+        That function shall return True when the connection succeeded, False otherwise.
+        """
+        if self.state == ConnectionState.CONNECTED:
+            # ensure the CONNECTED state is validated before skipping reconnection attempts
+            # even is state is CONNECTED, the underlying connection could be stale or broken
+            # or closed externally and unless you check the health here, you will never attempt
+            # recovery.
+            try:
+                healthy = await self.health_check()
+            except Exception as exc:
+                logger.debug(f"health_check raised: {type_name(exc)} – {exc}")
+                healthy = False
+
+            if healthy:
+                if VERBOSE_DEBUG:
+                    logger.debug(f"{self.service_name} already connected and healthy")
+                return
+
+            logger.info(
+                f"{self.service_name} marked CONNECTED but health_check failed — disconnecting and reconnecting"
+            )
+            self.state = ConnectionState.DISCONNECTED
+            try:
+                # ensure the state is updated by disconnect hook (disconnect_from_service should set DISCONNECTED)
+                await self.disconnect_from_service()
+            except Exception as exc:
+                if VERBOSE_DEBUG:
+                    logger.debug(f"Couldn't disconnect from {self.service_name}")
+
+        if not self.should_attempt_connection():
+            logger.debug("Not time yet to attempt new connection")
+            return
+
+        self.state = ConnectionState.CONNECTING
+        self.last_attempt = time.monotonic()
+
+        try:
+            success = await self.connect_to_service()
+
+            if success:
+                self.state = ConnectionState.CONNECTED
+                self.failure_count = 0
+                self.retry_interval = 1  # Reset backoff
+                logger.info(f"Successfully connected to service {self.service_name}")
+            else:
+                # warning should have been logged by the connect_to_service() callable.
+                self.handle_connection_failure()
+
+        except Exception as exc:
+            logger.warning(f"Failed to connect to service {self.service_name}: {exc}")
+            self.handle_connection_failure()
+
+    def handle_connection_failure(self):
+        self.failure_count += 1
+
+        # Open circuit breaker if too many failures
+        if self.failure_count >= self.max_failures_before_circuit_break:
+            self.state = ConnectionState.CIRCUIT_OPEN
+            self.circuit_opened_at = time.monotonic()
+            logger.warning(
+                f"Circuit breaker opened for service {self.service_name} after {self.failure_count} failures"
+            )
+        else:
+            self.state = ConnectionState.DISCONNECTED
+            self.retry_interval = calculate_retry_interval(
+                self.failure_count,
+                self.base_interval,
+                self.max_retry_interval,
+                self.backoff_strategy,
+                self.jitter_strategy,
+            )
+            logger.debug(f"retry_interval={self.retry_interval}")
+
+    def is_connected(self) -> bool:
+        if VERBOSE_DEBUG:
+            logger.debug(f"Checking if {self.service_name} is connected: {self.state.name}")
+        return self.state == ConnectionState.CONNECTED
+
+    def get_connection(self) -> Any:
+        """
+        Optional method to return the underlying connection object.
+        Subclasses should override this method to return the actual connection
+        (e.g., socket, transport) if needed.
+        """
+        logger.warning(f"The get_connection() method is not implemented for {self.service_name}, returning None.")
+        return None
+
+
+class ServiceConnector:
+    """
+    Synchronous base class for robust service connection management with retry, backoff, and circuit breaker logic.
+
+    This class is intended to be subclassed for managing persistent connections to external services
+    (such as devices, databases, or remote APIs) that may be unreliable or temporarily unavailable.
+
+    Features:
+        - Automatic retry with configurable backoff and jitter strategies.
+        - Circuit breaker to prevent repeated connection attempts after multiple failures.
+        - Connection state tracking (disconnected, connecting, connected, circuit open).
+        - Thread-safe operation using a lock for all state changes.
+
+    Usage:
+        1. Subclass `ServiceConnector` and override the `connect_to_service()` method with your
+           actual connection logic. Optionally, override `health_check()` for custom health verification.
+        2. Store the actual connection object (e.g., socket, transport) as an instance attribute in your subclass.
+        3. Use `attempt_connection()` to initiate connection attempts; it will handle retries and backoff automatically.
+        4. Use `is_connected()` to check connection status.
+
+    Example:
+        class MyConnector(ServiceConnector):
+            def connect_to_service(self):
+                self.connection = create_socket()
+                return self.connection is not None
+
+            def get_connection(self):
+                return self.connection
+
+    Note:
+        The base class does not manage or expose the underlying connection object.
+        Your subclass should provide a method or property to access it as needed.
+    """
+
+    def __init__(
+        self,
+        service_name: str,
+        backoff_strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL,
+        jitter_strategy: JitterStrategy = JitterStrategy.EQUAL,
+    ):
+        self.state = ConnectionState.DISCONNECTED
+        self.last_attempt = 0
+        self.base_interval = 1
+        self.retry_interval = 1
+        self.max_retry_interval = 300
+        self.failure_count = 0
+        self.max_failures_before_circuit_break = 5
+        self.circuit_break_duration = 60
+        self.circuit_opened_at = None
+        self.service_name = service_name
+        self.backoff_strategy = backoff_strategy
+        self.jitter_strategy = jitter_strategy
+
+        self._lock = threading.RLock()
+
+    def connect_to_service(self) -> bool:
+        logger.warning(
+            f"The connect_to_service() method is not implemented for {self.service_name}, connection will always fail."
+        )
+        return False
+
+    def disconnect_from_service(self) -> None:
+        """
+        Optional hook to cleanly disconnect / release resources for the service. Default implementation is a no-op.
+        Subclasses should override and must be careful about thread-safety; the base class holds _lock which can be
+        used.
+        """
+        with self._lock:
+            logger.debug(f"{self.service_name}: default disconnect_from_service(): no-op")
+            self.state = ConnectionState.DISCONNECTED
+        return
+
+    def health_check(self) -> bool:
+        logger.warning(
+            f"The health_check() method is not implemented for {self.service_name}, check will always return false."
+        )
+        return False
+
+    def should_attempt_connection(self) -> bool:
+        now = time.monotonic()
+        with self._lock:
+            if self.state == ConnectionState.CIRCUIT_OPEN:
+                assert self.circuit_opened_at is not None
+                if now - self.circuit_opened_at > self.circuit_break_duration:
+                    self.state = ConnectionState.DISCONNECTED
+                    self.failure_count = 0
+                    self.retry_interval = 1
+                    return True
+                return False
+            return now - self.last_attempt >= self.retry_interval
+
+    def attempt_connection(self):
+        with self._lock:
+            current_state = self.state
+
+        if current_state == ConnectionState.CONNECTED:
+            # ensure the CONNECTED state is validated before skipping reconnection attempts
+            try:
+                healthy = self.health_check()
+            except Exception as exc:
+                logger.debug(f"health_check raised: {type_name(exc)} – {exc}")
+                healthy = False
+
+            if healthy:
+                logger.debug(f"{self.service_name} already connected and healthy")
+                return
+
+            logger.info(
+                f"{self.service_name} marked CONNECTED but health_check failed — disconnecting and reconnecting"
+            )
+            self.state = ConnectionState.DISCONNECTED
+            try:
+                # ensure the state is updated by disconnect hook (disconnect_from_service should set DISCONNECTED)
+                self.disconnect_from_service()
+            except Exception as exc:
+                if VERBOSE_DEBUG:
+                    logger.debug(f"Couldn't disconnect from {self.service_name}: {type_name(exc)} – {exc}")
+
+        with self._lock:
+            if not self.should_attempt_connection():
+                return
+            self.state = ConnectionState.CONNECTING
+            self.last_attempt = time.monotonic()
+
+        try:
+            success = self.connect_to_service()
+            with self._lock:
+                if success:
+                    self.state = ConnectionState.CONNECTED
+                    self.failure_count = 0
+                    self.retry_interval = 1
+                    logger.debug(f"Successfully connected to service {self.service_name}")
+                else:
+                    self.handle_connection_failure()
+        except Exception as exc:
+            logger.error(f"Failed to connect to service {self.service_name}: {exc}")
+            with self._lock:
+                self.handle_connection_failure()
+
+    def handle_connection_failure(self):
+        self.failure_count += 1
+        if self.failure_count >= self.max_failures_before_circuit_break:
+            self.state = ConnectionState.CIRCUIT_OPEN
+            self.circuit_opened_at = time.monotonic()
+            logger.warning(
+                f"Circuit breaker opened for service {self.service_name} after {self.failure_count} failures"
+            )
+        else:
+            self.state = ConnectionState.DISCONNECTED
+            self.retry_interval = calculate_retry_interval(
+                self.failure_count,
+                self.base_interval,
+                self.max_retry_interval,
+                self.backoff_strategy,
+                self.jitter_strategy,
+            )
+            logger.debug(f"retry_interval={self.retry_interval}")
+
+    def is_connected(self) -> bool:
+        with self._lock:
+            return self.state == ConnectionState.CONNECTED

egse/control.py

@@ -558,7 +558,8 @@ class ControlServer(metaclass=abc.ABCMeta):
                 try:
                     hk_dict = save_average_execution_time(self.device_protocol.get_housekeeping)
 
-                    self.store_housekeeping_information(hk_dict)
+                    if storage_manager:
+                        self.store_housekeeping_information(hk_dict)
                     self.propagate_metrics(hk_dict)
                 except Exception as exc:
                     logger.error(

egse/dummy.py

@@ -18,9 +18,9 @@ and stop the server with:
 
 Commands that can be used with the proxy:
 
-  * info – returns an info message from the dummy device, e.g. "Dummy Device <__version__>"
-  * get_value – returns a random float between 0.0 and 1.0
-  * division – returns the result of the division between arguments 'a' and 'b'.
+  * info - returns an info message from the dummy device, e.g. "Dummy Device <__version__>"
+  * get_value - returns a random float between 0.0 and 1.0
+  * division - returns the result of the division between arguments 'a' and 'b'.
     This can be used also to induce a ZeroDivisionError that should return a Failure
     object.
 
@@ -35,6 +35,7 @@ and stopped with:
 
 from __future__ import annotations
 
+import contextlib
 import multiprocessing
 import random
 import select
@@ -52,12 +53,14 @@ from egse.device import DeviceConnectionError
 from egse.device import DeviceConnectionInterface
 from egse.device import DeviceTimeoutError
 from egse.device import DeviceTransport
+from egse.env import bool_env
 from egse.log import logger
 from egse.protocol import CommandProtocol
 from egse.proxy import Proxy
 from egse.system import SignalCatcher
 from egse.system import attrdict
 from egse.system import format_datetime
+from egse.system import type_name
 from egse.zmq_ser import bind_address
 from egse.zmq_ser import connect_address
 
@@ -77,6 +80,9 @@ WRITE_TIMEOUT = 1.0
 CONNECT_TIMEOUT = 3.0
 """The maximum time in seconds to wait for establishing a socket connect."""
 
+
+VERBOSE_DEBUG = bool_env("VERBOSE_DEBUG", default=False)
+
 # Especially DummyCommand and DummyController need to be defined in a known module
 # because those objects are pickled and when de-pickled at the clients side the class
 # definition must be known.
@@ -116,14 +122,17 @@ def is_dummy_cs_active() -> bool:
 
 
 def is_dummy_dev_active() -> bool:
+    if VERBOSE_DEBUG:
+        logger.debug("Checking if dummy device is active...")
     try:
         dev = DummyDeviceEthernetInterface(DEV_HOST, DEV_PORT)
         dev.connect()
         rc = dev.trans("ping\n")
         dev.disconnect()
         return rc.decode().strip() == "pong"
-    except DeviceConnectionError as exc:
-        # logger.error(f"Caught {type_name(exc)}: {exc}")
+    except (DeviceConnectionError, ConnectionResetError, DeviceTimeoutError) as exc:
+        if VERBOSE_DEBUG:
+            logger.debug(f"Caught {type_name(exc)}: {exc} - returning False")
         return False
 
 
@@ -309,11 +318,10 @@ class DummyDeviceEthernetInterface(DeviceConnectionInterface, DeviceTransport):
     Args:
         hostname (str): the IP address or fully qualified hostname of the Dummy Device
             controller.
-
         port (int): the IP port number to connect to.
     """
 
-    def __init__(self, hostname: str = None, port: int = None):
+    def __init__(self, hostname: str | None = None, port: int | None = None):
         super().__init__()
 
         # Basic connection settings, loaded from the configuration YAML file
@@ -350,7 +358,8 @@ class DummyDeviceEthernetInterface(DeviceConnectionInterface, DeviceTransport):
         try:
             self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         except socket.error as exc:
-            self.sock.close()
+            if self.sock is not None:
+                self.sock.close()
             raise DeviceConnectionError("Dummy Device", "Failed to create socket.") from exc
 
         # Attempt to establish a connection to the remote host
@@ -403,7 +412,8 @@ class DummyDeviceEthernetInterface(DeviceConnectionInterface, DeviceTransport):
         """
         try:
             logger.debug(f"Disconnecting from {self.hostname}")
-            self.sock.close()
+            if self.sock is not None:
+                self.sock.close()
             self.is_connection_open = False
         except Exception as exc:
             raise DeviceConnectionError(DEV_NAME, f"Could not close socket to {self.hostname}") from exc
@@ -439,6 +449,8 @@ class DummyDeviceEthernetInterface(DeviceConnectionInterface, DeviceTransport):
         buf_size = 1024 * 10
         response = bytes()
 
+        assert self.sock is not None
+
         # Set a timeout of READ_TIMEOUT to the socket.recv
 
         saved_timeout = self.sock.gettimeout()
@@ -478,6 +490,8 @@ class DummyDeviceEthernetInterface(DeviceConnectionInterface, DeviceTransport):
                 there was a socket related error.
         """
 
+        assert self.sock is not None
+
         # logger.debug(f"{command.encode() = }")
 
         try:
@@ -505,6 +519,9 @@ class DummyDeviceEthernetInterface(DeviceConnectionInterface, DeviceTransport):
             DeviceTimeoutError: when the sendall() timed out, and a DeviceConnectionError if
                 there was a socket related error.
         """
+
+        assert self.sock is not None
+
         # logger.debug(f"{command.encode() = }")
 
         try:
@@ -580,11 +597,18 @@ def start_dev():
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
             s.bind((DEV_HOST, DEV_PORT))
             s.listen()
+            s.settimeout(CONNECT_TIMEOUT)
             logger.info(f"Ready to accept connection on {DEV_HOST}:{DEV_PORT}...")
-            conn, addr = s.accept()
+            while True:
+                with contextlib.suppress(socket.timeout):
+                    conn, addr = s.accept()
+                    break
+                if killer.term_signal_received:
+                    return
             with conn:
                 logger.info(f"Accepted connection from {addr}")
                 conn.sendall(f"Dummy Device {__version__}".encode())
+                conn.settimeout(READ_TIMEOUT)
                 try:
                     while True:
                         error_msg = ""
@@ -641,7 +665,8 @@ def process_command(command_string: str) -> str | None:
 
     try:
         action, response = COMMAND_ACTIONS_RESPONSES[command_string]
-        action and action()
+        if action:
+            action()
         if error_msg:
             return error_msg
         else:

egse/logger/__init__.py

@@ -57,7 +57,7 @@ COMMANDER_PORT = settings.get("COMMANDER_PORT", 0)  # dynamically assigned by th
 _initialised = False  # will be set to True in the setup_logging() function
 
 
-def get_log_file_name():
+def get_log_file_name() -> str:
     """
     Returns the filename of the log file as defined in the Settings or return the default name 'general.log'.
     """
@@ -315,7 +315,7 @@ def send_request(command_request: str):
     """Sends a request to the Logger Control Server and waits for a response."""
 
     if COMMANDER_PORT == 0:
-        endpoint = get_endpoint_from_registry()
+        endpoint = get_endpoint_from_registry(SERVICE_TYPE)
     else:
         endpoint = f"{PROTOCOL}://{HOSTNAME}:{COMMANDER_PORT}"
 

egse/notifyhub/server.py

@@ -23,10 +23,12 @@ from egse.notifyhub import SERVICE_TYPE
 from egse.notifyhub import STATS_INTERVAL
 from egse.notifyhub.client import AsyncNotificationHubClient
 from egse.registry import MessageType
-from egse.registry.client import AsyncRegistryClient, REQUEST_TIMEOUT
+from egse.registry.client import REQUEST_TIMEOUT
+from egse.registry.client import AsyncRegistryClient
 from egse.system import TyperAsyncCommand
 from egse.system import get_host_ip
 from egse.zmq_ser import get_port_number
+
 from .event import NotificationEvent
 
 REQUEST_POLL_TIMEOUT = 1.0

egse/procman/procman_cs.py

@@ -154,7 +154,7 @@ def stop():
 
     if COMMANDING_PORT == 0:
         with RegistryClient() as reg:
-            service = reg.discover_service(settings.SERVICE_TYPE)
+            service = reg.discover_service(SERVICE_TYPE)
             rich.print("service = ", service)
             if service:
                 hostname = service["host"]

egse/procman/procman_ui.py

@@ -44,6 +44,9 @@ from egse.zmq_ser import set_address_port
 
 MAX_SLEEP = 10
 
+DEVICE_CMD_ENTRY_POINT = "cgse.service.device_command"
+GUI_SCRIPTS_ENTRY_POINT = "gui_scripts"
+
 
 class ControlServerStatus(Enum):
     """Status of the Control Server of a device."""
@@ -78,7 +81,7 @@ def get_cgse_cmd(device_proxy: str) -> str:
 
     module_name = device_proxy[7:].rsplit(".", 1)[0]
     entry_point_values = []
-    for ep in sorted(entry_points("cgse.service"), key=lambda x: x.name):
+    for ep in sorted(entry_points(DEVICE_CMD_ENTRY_POINT), key=lambda x: x.name):
         entry_point_values.append(ep.value)
 
     similarity_scores = [
@@ -108,7 +111,7 @@ def get_cgse_ui(device_proxy: str) -> Union[str, None]:
 
     module_name = device_proxy[7:].rsplit(".", 1)[0]
     entry_point_values = []
-    for ep in sorted(entry_points("gui_scripts"), key=lambda x: x.name):
+    for ep in sorted(entry_points(GUI_SCRIPTS_ENTRY_POINT), key=lambda x: x.name):
         entry_point_values.append(ep.name)
 
     similarity_scores = [

egse/registry/client.py

@@ -639,6 +639,8 @@ class AsyncRegistryClient:
             The response from the registry as a dictionary.
         """
 
+        assert self.req_socket is not None, "REQ socket is not connected, cannot send request."
+
         timeout = timeout or self.timeout
         try:
             self.logger.debug(f"Sending request: {request}")
@@ -687,6 +689,8 @@ class AsyncRegistryClient:
             The response from the registry as a dictionary.
         """
 
+        assert self.hb_socket is not None, "HB socket is not connected, cannot send heartbeat request."
+
         try:
             self.logger.debug(f"Sending heartbeat request: {request}")
             await self.hb_socket.send_string(json.dumps(request))
@@ -867,7 +871,8 @@ class AsyncRegistryClient:
                                 await self.reregister()
 
                         else:
-                            VERBOSE_DEBUG and self.logger.debug(f"Heartbeat succeeded: {response.get('message')}")
+                            if VERBOSE_DEBUG:
+                                self.logger.debug(f"Heartbeat succeeded: {response.get('message')}")
 
                     except Exception as exc:
                         self.logger.error(f"Error in heartbeat loop: {exc}", exc_info=True)
@@ -893,13 +898,15 @@ class AsyncRegistryClient:
         """Stop the running heartbeat task."""
 
         if self._heartbeat_task is None:
-            VERBOSE_DEBUG and self.logger.debug("Couldn't stop heartbeat, heartbeat_task is None")
+            if VERBOSE_DEBUG:
+                self.logger.debug("Couldn't stop heartbeat, heartbeat_task is None")
             return
 
         self._heartbeat_task.cancel()
         try:
             await self._heartbeat_task
         except asyncio.CancelledError:
+            self.logger.info("Heartbeat task cancelled")
             pass
         self._tasks.discard(self._heartbeat_task)
         self._heartbeat_task = None
@@ -909,7 +916,8 @@ class AsyncRegistryClient:
         """Stop the running event listener task."""
 
         if self._event_listener_task is None:
-            VERBOSE_DEBUG and self.logger.debug("Couldn't stop event_listener, event_listener_task is None")
+            if VERBOSE_DEBUG:
+                self.logger.debug("Couldn't stop event_listener, event_listener_task is None")
             return
 
         self._event_listener_task.cancel()
@@ -1142,12 +1150,14 @@ class AsyncRegistryClient:
                 self.sub_socket.close()
 
             # We can not terminate the context, because we use a global instance, i.e. a singleton context.
-            # When we try to terminate it, even after checking if it was closed,
+            # When we try to terminate it, even after checking if it was closed, it raises an exception.
             if hasattr(self, "context") and self.context:
                 self.logger.info(f"{self.context = !r}")
                 self.logger.info(f"{self.context._sockets = !r}")
-                if not self.context.closed:
-                    self.context.term()
+                # The zmq context instance is the global singleton instance.
+                # Terminating it here would affect other parts of the application using zmq.
+                # if not self.context.closed:
+                #     self.context.term()
         except Exception as exc:
             self.logger.error(f"Error during cleanup: {exc}")
 

egse/registry/server.py

@@ -218,13 +218,15 @@ class AsyncRegistryServer:
         """Task that handles incoming requests."""
         self.logger.info("Started request handler task")
 
+        assert self.req_socket is not None, "REQ socket is not connected, cannot handle requests."
+
         try:
             message_parts = None
             while self._running:
                 try:
                     # Wait for a request with timeout to allow checking if still running
                     try:
-                        # self.logger.info("Waiting for a request with 1s timeout...")
+                        # self.logger.debug("Waiting for a request with 1s timeout...")
                         message_parts = await asyncio.wait_for(self.req_socket.recv_multipart(), timeout=1.0)
                     except asyncio.TimeoutError:
                         # self.logger.debug("waiting for command request...")
@@ -241,6 +243,9 @@ class AsyncRegistryServer:
                         response = await self._process_request(message_data)
 
                         await self._send_response(client_id, message_type, response)
+                    else:
+                        self.logger.warning("Request handler: message corrupted, check debug messages.")
+                        self.logger.debug(f"{message_parts=}")
 
                 except zmq.ZMQError as exc:
                     self.logger.error(f"ZMQ error: {exc}", exc_info=True)
@@ -397,6 +402,8 @@ class AsyncRegistryServer:
         """Task that handles heartbeat messages."""
         self.logger.info("Started heartbeats handler task")
 
+        assert self.hb_socket is not None, "HB socket is not connected, cannot handle heartbeat messages."
+
         try:
             message_parts = None
             while self._running:
@@ -425,7 +432,8 @@ class AsyncRegistryServer:
                         self.logger.warning("Heartbeat request: message corrupted, check debug messages.")
 
                 except asyncio.TimeoutError:
-                    VERBOSE_DEBUG and self.logger.debug("waiting for heartbeat...")
+                    if VERBOSE_DEBUG:
+                        self.logger.debug("waiting for heartbeat...")
                     continue
 
                 except Exception as exc:

egse/registry/service.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import asyncio
 import json
-import logging
 import time
 from typing import Any
 from typing import Callable
@@ -10,13 +9,14 @@ from typing import Callable
 import zmq
 import zmq.asyncio
 
+from egse.log import logging
 from egse.registry import DEFAULT_RS_PUB_PORT
 from egse.registry import DEFAULT_RS_REQ_PORT
 from egse.registry.client import AsyncRegistryClient
 from egse.system import get_host_ip
 from egse.zmq_ser import get_port_number
 
-module_module_logger_name = "async_microservice"
+module_module_logger_name = "egse.async_microservice"
 module_logger = logging.getLogger(module_module_logger_name)
 
 
@@ -64,7 +64,7 @@ class ZMQMicroservice:
         self.registry_sub_endpoint = registry_sub_endpoint or f"tcp://localhost:{DEFAULT_RS_PUB_PORT}"
         self.metadata = metadata or {}
 
-        self.host_ip = get_host_ip()
+        self.host_ip = get_host_ip() or "localhost"
 
         # Service ID will be set when registered
         self.service_id = None
@@ -164,6 +164,7 @@ class ZMQMicroservice:
 
         if not self.service_id:
             module_logger.error("Failed to register with the service registry")
+            await self._cleanup()
             return True
 
         module_logger.info(f"Registered with service ID: {self.service_id}")
@@ -175,12 +176,17 @@ class ZMQMicroservice:
         # Start request handler
         request_task = asyncio.create_task(self._handle_requests())
         self._tasks.add(request_task)
-        request_task.add_done_callback(self._tasks.discard)
+        # request_task.add_done_callback(self._tasks.discard)
 
         # Wait for shutdown signal
         await self._shutdown.wait()
 
-        # Clean shutdown
+        # request_task.cancel()
+        # try:
+        #     await request_task
+        # except asyncio.CancelledError:
+        #     module_logger.info("Request handler task cancelled during shutdown")
+
         await self._cleanup()
 
         return False